building_manager.py 19 KB
Newer Older
1
#!/usr/bin/env python
2
3
import bcrypt
import crypt
4
import docker
5
import logging
6
import os
7

8
from shutil import copy2
9
from subprocess import run
10
from PyInquirer import prompt
11
12
13

# Log level during development is info
logging.basicConfig(level=logging.WARNING)
14

15
# Directories for config generation
16
17
CUSTOM_DIR = 'custom_configs'
TEMPLATE_DIR = 'template_configs'
Dobli's avatar
Dobli committed
18
19
20
21
CONFIG_DIRS = ['mosquitto', 'nodered', 'ssh', 'traefik', 'volumerize']
TEMPLATE_FILES = [
    'mosquitto/mosquitto.conf', 'nodered/nodered_package.json',
    'nodered/nodered_settings.js', 'ssh/sshd_config', 'traefik/traefik.toml'
22
]
23
24
EDIT_FILES = {
    "mosquitto_passwords": "mosquitto/mosquitto_passwords",
25
    "sftp_users": "ssh/sftp_users.conf",
26
27
28
29
    "traefik_users": "traefik/traefik_users",
    "id_rsa": "ssh/id_rsa",
    "host_key": "ssh/ssh_host_ed25519_key",
    "known_hosts": "ssh/known_hosts"
30
}
31

32
# Default Swarm port
dobli's avatar
dobli committed
33
34
SWARM_PORT = 2377

35

36
# ******************************
dobli's avatar
dobli committed
37
# Config file functions {{{
38
# ******************************
39
def generate_config_folders(base_dir):
40
41
    """Generate folders for configuration files

42
    :base_dir: Path to add folders to
43
    """
44
45
46
47
48
49
    base_path = base_dir + '/' + CUSTOM_DIR
    if not os.path.exists(base_dir):
        os.makedirs(base_dir)

    print(f'Initialize configuration in {base_path}')

50
    # generate empty config dirs
51
52
53
54
55
    for d in CONFIG_DIRS:
        new_dir = base_path + '/' + d
        if not os.path.exists(new_dir):
            os.makedirs(new_dir)

56
57
58
59
    # copy template configs
    for template_file in TEMPLATE_FILES:
        copy_template_config(base_dir, template_file)

60

61
62
63
64
65
66
def copy_template_config(base_dir, config_path):
    """Copies template configuration files into custom folder

    :base_dir: path that contains template and custom folders
    :config_path: relative path of config to copy from template
    """
Dobli's avatar
Dobli committed
67
68
69
70
71
    custom_path = base_dir + '/' + CUSTOM_DIR + "/" + config_path
    template_path = base_dir + '/' + TEMPLATE_DIR + "/" + config_path

    logging.info(f'Copy {config_path} from {custom_path} to {template_path}')
    copy2(template_path, custom_path)
72
73


74
75
76
77
78
79
def generate_mosquitto_user_line(username, password):
    """Generates a line for a mosquitto user with a crypt hashed password

    :username: username to use
    :password: password that will be hashed (SHA512)

80
    :returns: a line as expected by mosquitto
81
82
83
84
85
86
    """
    password_hash = crypt.crypt(password, crypt.mksalt(crypt.METHOD_SHA512))
    line = f"{username}:{password_hash}"
    return line


87
88
89
90
def generate_sftp_user_line(username, password, directories=None):
    """Generates a line for a sftp user with a hashed password

    :username: username to use
91
    :password: password that will be hashed (SHA512)
92
93
    :directories: list of directories which the user should have

94
    :returns: a line as expected by sshd
95
96
97
98
99
100
101
102
103
104
105
106
    """
    # generate user line with hashed password
    password_hash = crypt.crypt(password, crypt.mksalt(crypt.METHOD_SHA512))
    line = f"{username}:{password_hash}:e"
    # add directory entries when available
    if directories:
        # create comma separated string from list
        dir_line = ','.join(d for d in directories)
        line = f"{line}:{dir_line}"
    return line


107
108
109
110
111
112
113
114
115
116
117
118
119
def generate_traefik_user_line(username, password):
    """Generates a line for a traefik user with a bcrypt hashed password

    :username: username to use
    :password: password that will be hashed (bcrypt)

    :returns: a line as expected by traefik
    """
    password_hash = bcrypt.hashpw(password.encode(), bcrypt.gensalt())
    line = f"{username}:{password_hash.decode()}"
    return line


120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
def generate_mosquitto_file(base_dir, username, password):
    """Generates a mosquitto password file using mosquitto_passwd system tool

    :base_dir: path that contains custom config folder
    :username: username to use
    :password: password that will be used
    """
    passwd_path = base_dir + '/' + CUSTOM_DIR + "/" + EDIT_FILES[
        'mosquitto_passwords']

    # ensure file exists
    if not os.path.exists(passwd_path):
        open(passwd_path, 'a').close()

    # execute mosquitto passwd
    mos_result = run(
        ['mosquitto_passwd', '-b', passwd_path, username, password],
        text=True,
        capture_output=True)
    return mos_result.returncode == 0


def generate_sftp_file(base_dir, username, password, direcories=None):
143
    """Generates a sftp password file
144
145
146
147
148
149
150
151
152
153
154
155

    :base_dir: path that contains custom config folder
    :username: username to use
    :password: password that will be used
    :directories: list of directories which the user should have
    """
    # generate line and save it into a file
    file_content = generate_sftp_user_line(username, password, direcories)
    create_or_replace_config_file(base_dir, EDIT_FILES['sftp_users'],
                                  file_content)


156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
def generate_id_rsa_files(base_dir):
    """Generates id_rsa and id_rsa.pub private/public keys using ssh-keygen

    :base_dir: path that contains custom config folder
    """
    id_path = base_dir + '/' + CUSTOM_DIR + "/" + EDIT_FILES['id_rsa']

    # execute ssh-keygen
    id_result = run(
        ['ssh-keygen', '-t', 'rsa', '-b', '4096', '-f', id_path, '-N', ''],
        text=True,
        capture_output=True)
    return id_result.returncode == 0


def generate_host_key_files(base_dir, hosts):
    """Generates ssh host keys and matching known_hosts using ssh-keygen

    :base_dir: path that contains custom config folder
    """
    key_path = base_dir + '/' + CUSTOM_DIR + "/" + EDIT_FILES['host_key']
    # ssh-keygen generates public key with .pub postfix
    pub_path = key_path + '.pub'

    # execute ssh-keygen
    id_result = run(['ssh-keygen', '-t', 'ed25519', '-f', key_path, '-N', ''],
                    text=True,
                    capture_output=True)

    # read content of public key as known line
    known_line = ""
    with open(pub_path, 'r') as pub_file:
        pub_line = pub_file.readline()
        split_line = pub_line.split()
        # delete last list element
        del split_line[-1]
        # collect hosts as comma separated string
        hosts_line = ','.join(h for h in hosts)
        split_line.insert(0, hosts_line)
        # collect parts as space separated string
        known_line = ' '.join(sp for sp in split_line)

    # write new known_line file
    create_or_replace_config_file(base_dir, EDIT_FILES['known_hosts'],
                                  known_line)

    return id_result.returncode == 0


205
206
207
208
209
210
211
212
213
214
215
216
217
def generate_traefik_file(base_dir, username, password):
    """Generates a traefik password file

    :base_dir: path that contains custom config folder
    :username: username to use
    :password: password that will be used
    """
    # generate line and save it into a file
    file_content = generate_traefik_user_line(username, password)
    create_or_replace_config_file(base_dir, EDIT_FILES['traefik_users'],
                                  file_content)


218
219
220
221
222
223
224
225
226
227
228
229
def create_or_replace_config_file(base_dir, config_path, content):
    """Creates or replaces a config file with new content

    :base_dir: path that contains custom config folder
    :config_path: relative path of config
    :content: content of the file as a string
    """
    custom_path = base_dir + '/' + CUSTOM_DIR + "/" + config_path
    with open(custom_path, 'w+') as file:
        file.write(content)


dobli's avatar
dobli committed
230
231
232
# }}}


233
# ******************************
dobli's avatar
dobli committed
234
# Docker machine functions {{{
235
# ******************************
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
def get_machine_list():
    """Get a list of docker machine names using the docker-machine system command

    :returns: a list of machine names managed by docker-machine
    """
    machine_result = run(['docker-machine', 'ls', '-q'],
                         text=True,
                         capture_output=True)
    return machine_result.stdout.splitlines()


def check_machine_exists(machine_name):
    """Checks weather a docker machine exists and is available

    :machine_name: Name of the machine to check
dobli's avatar
dobli committed
251
    :returns: True when machine is available
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
    """
    machines = get_machine_list()

    return machine_name in machines


def get_machine_env(machine_name):
    """Gets dict of env settings from a machine

    :machine_name: Name of the machine to check
    :returns: Dict of env variables for this machine
    """
    env_result = run(['docker-machine', 'env', machine_name],
                     text=True,
                     capture_output=True)

    machine_envs = {}

    lines = env_result.stdout.splitlines()
    for line in lines:
        if 'export' in line:
            assign = line.split('export ', 1)[1]
            env_entry = [a.strip('"') for a in assign.split('=', 1)]
            machine_envs[env_entry[0]] = env_entry[1]
    return machine_envs


dobli's avatar
dobli committed
279
280
281
282
283
284
285
286
def get_machine_ip(machine_name):
    """Asks for the ip of the docker machine

    :machine_name: Name of the machine to use for init
    """
    machine_result = run(['docker-machine', 'ip', machine_name],
                         text=True,
                         capture_output=True)
287
    return machine_result.stdout.strip()
dobli's avatar
dobli committed
288
289
290
291
292
293


def init_swarm_machine(machine_name):
    """Creates a new swarm with the specified machine as leader

    :machine_name: Name of the machine to use for init
294
    :return: True if swarm init was successful
dobli's avatar
dobli committed
295
296
297
    """
    machine_ip = get_machine_ip(machine_name)
    init_command = 'docker swarm init --advertise-addr ' + machine_ip
298
299
300
301
    init_result = run(['docker-machine', 'ssh', machine_name, init_command],
                      text=True,
                      capture_output=True)
    return init_result.returncode == 0
dobli's avatar
dobli committed
302
303
304
305
306
307
308


def join_swarm_machine(machine_name, leader_name):
    """Joins the swarm of the specified leader

    :machine_name: Name of the machine to join a swarm
    :leader_name: Name of the swarm leader machine
309
    :return: True if join to swarm was successful
dobli's avatar
dobli committed
310
311
312
313
314
    """
    token_command = 'docker swarm join-token manager -q'
    token_result = run(['docker-machine', 'ssh', leader_name, token_command],
                       text=True,
                       capture_output=True)
315
    token = token_result.stdout.strip()
dobli's avatar
dobli committed
316
    leader_ip = get_machine_ip(leader_name)
317
    logging.info(f"Swarm leader with ip {leader_ip} uses token {token}")
dobli's avatar
dobli committed
318

319
320
321
    join_cmd = f'docker swarm join --token {token} {leader_ip}:{SWARM_PORT}'
    logging.info(f'Machine {machine_name} joins using command {join_cmd}')
    join_result = run(['docker-machine', 'ssh', machine_name, join_cmd],
dobli's avatar
dobli committed
322
323
324
                      text=True,
                      capture_output=True)

325
    return join_result.returncode == 0
dobli's avatar
dobli committed
326
327


dobli's avatar
dobli committed
328
329
330
# }}}


331
# ******************************
dobli's avatar
dobli committed
332
# Docker client commands {{{
333
# ******************************
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
def assign_label_to_node(nodeid, label, value):
    """Assigns a label to a node (e.g. building)

    :nodeid: Id or name of the node
    :label: Label you want to add
    :value: The value to assign to the label
    """
    client = docker.from_env()

    node = client.nodes.get(nodeid)
    spec = node.attrs['Spec']
    spec['Labels'][label] = value
    node.update(spec)

    client.close()


351
352
353
354
355
356
357
358
359
360
def run_command_in_service(service, command, building=None):
    """Runs a command in a service based on its name.
    When no matching container is found or the service name is ambigous
    an error will be displayed and the function exits

    :param service: Name of the service to execute command
    :param command: Command to execute
    :param building: Optional building, make service unambigous (Default: None)
    """

361
362
363
364
365
    if building:
        building_env = get_machine_env(building)
        client = docker.from_env(environment=building_env)
    else:
        client = docker.from_env()
366
367
368
369
370
371
372

    # Find containers matching name
    service_name_filter = {"name": service}
    containers = client.containers.list(filters=service_name_filter)

    # Ensure match is unambigous
    if (len(containers) > 1):
373
        print(f'Found multiple containers matching service name {service}, '
374
375
              'ensure service is unambigous')
    elif (len(containers) < 1):
376
        print(f'Found no matching container for service name {service}')
377
378
    else:
        service_container = containers[0]
379
380
        print(f'Executing {command} in container {service_container.name}'
              f'({service_container.id}) on building {building}')
381
        print(service_container.exec_run(command))
382
    client.close()
383
384


dobli's avatar
dobli committed
385
386
387
# }}}


388
# ******************************
dobli's avatar
dobli committed
389
# CLI base commands {{{
390
# ******************************
391
392
393
394
395
396
397
398
def init_config_dirs_command(args):
    """Initialize config directories

    :args: parsed commandline arguments
    """
    base_dir = args.base_dir

    if base_dir is None:
399
        base_dir = os.getcwd()
400

Dobli's avatar
Dobli committed
401
    # generate basic config folder
402
403
404
    generate_config_folders(base_dir)


405
406
407
408
409
410
411
412
def assign_building_command(args):
    """Assigns the role of a building to a node

    :args: parsed commandline arguments
    """
    node = args.node
    building = args.building

413
    print(f'Assign role of building {building} to node {node}')
414
415
416
417

    assign_label_to_node(node, 'building', building)


418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
def execute_command(args):
    """Top level function to manage command executions from CLI

    :args: parsed commandline arguments
    """
    service = args.service
    command = " ".join(str(x) for x in args.command)  # list to string
    building = args.building

    run_command_in_service(service, command, building)


def restore_command(args):
    """Top level function to manage command executions from CLI

    :args: parsed commandline arguments
    """
435
436
437
438
    building = args.building
    target = args.target

    if not check_machine_exists(target):
439
        print(f'Machine with name {target} not found')
440
441
        return

442
    print(f'Restoring building {building} on machine {target}')
443
444

    get_machine_env(target)
445
446


447
448
449
def interactive_command(args):
    """Top level function to start the interactive mode

450
    :args: parsed command line arguments
451
    """
452
    print(main_menu(args))
453
454


dobli's avatar
dobli committed
455
456
457
# }}}


458
# ******************************
dobli's avatar
dobli committed
459
# Interactive menu entries {{{
460
# ******************************
461
def main_menu(args):
462
463
464
465
466
467
468
469
470
471
472
473
474
475
    """ Display main menu
    """
    questions = [{
        'type':
        'list',
        'name':
        'main',
        'message':
        'Public Building Manager - Main Menu',
        'choices': ['Create initial structure', 'Execute command', 'Exit']
    }]
    answers = prompt(questions)

    if 'Create' in answers['main']:
476
        init_menu(args)
477
478
479
480

    return answers


481
def init_menu(args):
482
483
    """Menu entry for initial setup and file generation
    """
484
485
486
487
488
489
490
    # Base directory for configs
    base_dir = args.base_dir

    if base_dir is None:
        base_dir = os.getcwd()

    # Prompts
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
    questions = [{
        'type': 'input',
        'name': 'stack_name',
        'message': 'Choose a name for your setup'
    },
                 {
                     'type': 'checkbox',
                     'name': 'machines',
                     'message': 'What docker machines will be used?',
                     'choices': generate_checkbox_choices(get_machine_list())
                 },
                 {
                     'type': 'input',
                     'name': 'username',
                     'message': 'Choose a username for the initial user'
                 },
                 {
                     'type': 'password',
                     'name': 'password',
                     'message': 'Choose a password for the initial user'
                 }]
512
513
    answers = prompt(questions)

dobli's avatar
dobli committed
514
515
516
517
518
519
    leader = None

    for machine in answers['machines']:
        # init swarm with first machine
        if leader is None:
            leader = machine
520
            print(f'Create initial swarm with leader {leader}')
521
522
            if init_swarm_machine(leader):
                print('Swarm init successful\n')
dobli's avatar
dobli committed
523
        else:
524
525
526
            print(f'Machine {machine} joins swarm of leader {leader}')
            if (join_swarm_machine(machine, leader)):
                print('Joining swarm successful\n')
527

528
529
    # Initialize custom configuration dirs and templates
    generate_config_folders(base_dir)
530
531
532
    # Generate config files based on input
    generate_sftp_file(base_dir, answers['username'], answers['password'])
    generate_mosquitto_file(base_dir, answers['username'], answers['password'])
533
    generate_traefik_file(base_dir, answers['username'], answers['password'])
534
535
    generate_id_rsa_files(base_dir)
    generate_host_key_files(base_dir, ["host1", "host2"])
536

537
538
539
    print(answers)


dobli's avatar
dobli committed
540
541
542
543
544
545
546
547
def generate_checkbox_choices(list):
    """Generates checkbox entries for lists of strings

    :returns: A list of dicts with name keys
    """
    return [{'name': m} for m in list]


dobli's avatar
dobli committed
548
549
# }}}

550
# ******************************
551
# Script main ( entry) {{{
552
# ******************************
553
554
555
556
557
558
if __name__ == '__main__':
    import argparse
    parser = argparse.ArgumentParser(
        prog='building_manger',
        description='Generate and manage multi'
        'building configurations of openHAB with docker swarm')
559
560
561
562
    parser.add_argument(
        '--base_dir',
        '-d',
        help='Directory to creat config folders in, default is current dir')
563
564
    subparsers = parser.add_subparsers()

565
566
567
568
569
570
    # Interactive mode
    parser_interactive = subparsers.add_parser(
        'interactive',
        help='Starts the interactive mode of the building manager')
    parser_interactive.set_defaults(func=interactive_command)

571
572
573
    # Restore command
    parser_restore = subparsers.add_parser('restore', help='Restore backups')
    parser_restore.add_argument(
574
        'building', help='Name (label) of the building that shall be restored')
575
576
577
578
    parser_restore.add_argument(
        'target', help='Name of the machine to restore to')
    parser_restore.set_defaults(func=restore_command)

579
580
581
582
583
584
585
586
587
    # Assign building command
    parser_assign_building = subparsers.add_parser(
        'assign_building', help='Assign the role of a building to a node')
    parser_assign_building.add_argument(
        'node', help='Name (or ID) of the node that gets the role assigned')
    parser_assign_building.add_argument(
        'building', help='Name of the building that will be assigned')
    parser_assign_building.set_defaults(func=assign_building_command)

588
589
590
591
592
593
594
595
596
597
598
599
600
601
    # Execute command
    parser_exec = subparsers.add_parser(
        'exec', help='Execute commands in a service container')
    parser_exec.add_argument(
        'service', help='Name of the service that will run the command')
    parser_exec.add_argument(
        'command', help='Command to be executed', nargs=argparse.REMAINDER)
    parser_exec.add_argument(
        '--building',
        '-b',
        help='Building name (label) of the service if '
        'service location is ambiguous')
    parser_exec.set_defaults(func=execute_command)

602
603
604
605
606
607
608
609
610
    # Config commands
    parser_config = subparsers.add_parser(
        'config', help='Manage configuration files')
    parser_config_subs = parser_config.add_subparsers()
    # - Config init
    parser_config_init = parser_config_subs.add_parser(
        'init', help='Initialize config file directories')
    parser_config_init.set_defaults(func=init_config_dirs_command)

611
    # Parse arguments into args dict
612
    args = parser.parse_args()
613
614
615
616
617
618

    # when no subcommand is defined show interactive menu
    try:
        args.func(args)
    except AttributeError:
        interactive_command(args)
dobli's avatar
dobli committed
619
620
621
622
# }}}

# --- vim settings ---
# vim:foldmethod=marker:foldlevel=0