From 2f7ad04a386c8c5e89a9dccbbe128dec86015ef6 Mon Sep 17 00:00:00 2001
From: dobli <dobler.alex@gmail.com>
Date: Tue, 22 Jan 2019 22:16:17 +0100
Subject: [PATCH] added generation of compose file with basic service set
---
building_manager.py | 231 +++++++++++++++++++++++++-
requirements.in | 3 +-
requirements.txt | 1 +
template_configs/docker-skeleton.yml | 67 ++++++++
template_configs/docker-templates.yml | 199 ++++++++++++++++++++++
5 files changed, 495 insertions(+), 6 deletions(-)
create mode 100644 template_configs/docker-templates.yml
diff --git a/building_manager.py b/building_manager.py
index 3b832c0..1e0ede6 100755
--- a/building_manager.py
+++ b/building_manager.py
@@ -4,10 +4,16 @@ import crypt
import docker
import logging
import os
+# import yaml
from shutil import copy2
from subprocess import run
from PyInquirer import prompt
+from ruamel.yaml import YAML
+
+# Configure YAML
+yaml = YAML()
+yaml.indent(mapping=4, sequence=4, offset=2)
# Log level during development is info
logging.basicConfig(level=logging.WARNING)
@@ -15,6 +21,9 @@ logging.basicConfig(level=logging.WARNING)
# Directories for config generation
CUSTOM_DIR = 'custom_configs'
TEMPLATE_DIR = 'template_configs'
+COMPOSE_NAME = 'docker-stack.yml'
+SKELETON_NAME = 'docker-skeleton.yml'
+TEMPLATES_NAME = 'docker-templates.yml'
CONFIG_DIRS = ['mosquitto', 'nodered', 'ssh', 'traefik', 'volumerize']
TEMPLATE_FILES = [
'mosquitto/mosquitto.conf', 'nodered/nodered_package.json',
@@ -26,11 +35,192 @@ EDIT_FILES = {
"traefik_users": "traefik/traefik_users",
"id_rsa": "ssh/id_rsa",
"host_key": "ssh/ssh_host_ed25519_key",
- "known_hosts": "ssh/known_hosts"
+ "known_hosts": "ssh/known_hosts",
+ "backup_config": "volumerize/backup_config.json"
+}
+CONSTRAINTS = {
+ "building": "node.labels.building"
+}
+SERVICES = {
+ "sftp": "sftp_X",
+ "openhab": "openhab_X",
+ "nodered": "nodered_X",
+ "mqtt": "mqtt_X"
}
# Default Swarm port
SWARM_PORT = 2377
+# UID for admin
+UID = 9001
+
+# ******************************
+# Compose file functions {{{
+# ******************************
+
+
+def generate_initial_compose(base_dir):
+ """Creates the initial compose using the skeleton
+
+ :base_dir: Folder to place configuration files into
+ """
+ base_path = base_dir + '/' + CUSTOM_DIR
+ template_path = base_dir + '/' + TEMPLATE_DIR
+ # compose file
+ compose = base_path + '/' + COMPOSE_NAME
+ # skeleton file
+ skeleton = template_path + '/' + SKELETON_NAME
+
+ with open(skeleton, 'r') as skeleton_f, open(compose, 'w+') as compose_f:
+ init_content = yaml.load(skeleton_f)
+ yaml.dump(init_content, compose_f)
+
+
+def add_sftp_service(base_dir, hostname, number=0):
+ """Generates an sftp entry and adds it to the compose file
+
+ :base_dir: base directory for configuration files
+ :hostname: names of host that the services is added to
+ :number: increment of exposed port to prevent overlaps
+ """
+ base_path = base_dir + '/' + CUSTOM_DIR
+ # compose file
+ compose_path = base_path + '/' + COMPOSE_NAME
+ # template
+ template = get_service_template(base_dir, SERVICES['sftp'])
+ # service name
+ service_name = f'sftp_{hostname}'
+
+ with open(compose_path, 'r+') as compose_f:
+ # load compose file
+ compose = yaml.load(compose_f)
+ # only label contraint is building
+ template['deploy']['placement']['constraints'][0] = (
+ f"{CONSTRAINTS['building']} == {hostname}")
+ template['ports'] = [f'{2222 + number}:22']
+ compose['services'][service_name] = template
+ # write content starting from first line
+ compose_f.seek(0)
+ # write new compose content
+ yaml.dump(compose, compose_f)
+ # reduce file to new size
+ compose_f.truncate()
+
+
+def add_openhab_service(base_dir, hostname):
+ """Generates an openhab entry and adds it to the compose file
+
+ :base_dir: base directory for configuration files
+ :hostname: names of host that the services is added to
+ """
+ base_path = base_dir + '/' + CUSTOM_DIR
+ # compose file
+ compose_path = base_path + '/' + COMPOSE_NAME
+ # template
+ template = get_service_template(base_dir, SERVICES['openhab'])
+ # service name
+ service_name = f'openhab_{hostname}'
+
+ with open(compose_path, 'r+') as compose_f:
+ # load compose file
+ compose = yaml.load(compose_f)
+ # only label contraint is building
+ template['deploy']['placement']['constraints'][0] = (
+ f"{CONSTRAINTS['building']} == {hostname}")
+ template['deploy']['labels'].append(f'traefik.backend={service_name}')
+ template['deploy']['labels'].append(f'backup={hostname}')
+ template['deploy']['labels'].append(
+ f'traefik.frontend.rule=HostRegexp:'
+ f'{hostname}.{{domain:[a-zA-z0-9-]+}}')
+ compose['services'][service_name] = template
+ # write content starting from first line
+ compose_f.seek(0)
+ # write new compose content
+ yaml.dump(compose, compose_f)
+ # reduce file to new size
+ compose_f.truncate()
+
+
+def add_nodered_service(base_dir, hostname):
+ """Generates an nodered entry and adds it to the compose file
+
+ :base_dir: base directory for configuration files
+ :hostname: names of host that the services is added to
+ """
+ base_path = base_dir + '/' + CUSTOM_DIR
+ # compose file
+ compose_path = base_path + '/' + COMPOSE_NAME
+ # template
+ template = get_service_template(base_dir, SERVICES['nodered'])
+ # service name
+ service_name = f'nodered_{hostname}'
+
+ with open(compose_path, 'r+') as compose_f:
+ # load compose file
+ compose = yaml.load(compose_f)
+ # only label contraint is building
+ template['deploy']['placement']['constraints'][0] = (
+ f"{CONSTRAINTS['building']} == {hostname}")
+ template['deploy']['labels'].append(f'traefik.backend={service_name}')
+ template['deploy']['labels'].append(f'backup={hostname}')
+ template['deploy']['labels'].append(
+ f'traefik.frontend.rule=HostRegexp:'
+ f'{service_name}.{{domain:[a-zA-z0-9-]+}}')
+ compose['services'][service_name] = template
+ # write content starting from first line
+ compose_f.seek(0)
+ # write new compose content
+ yaml.dump(compose, compose_f)
+ # reduce file to new size
+ compose_f.truncate()
+
+
+def add_mqtt_service(base_dir, hostname, number=0):
+ """Generates an mqtt entry and adds it to the compose file
+
+ :base_dir: base directory for configuration files
+ :hostname: names of host that the services is added to
+ :number: increment of exposed port to prevent overlaps
+ """
+ base_path = base_dir + '/' + CUSTOM_DIR
+ # compose file
+ compose_path = base_path + '/' + COMPOSE_NAME
+ # template
+ template = get_service_template(base_dir, SERVICES['mqtt'])
+ # service name
+ service_name = f'mqtt_{hostname}'
+
+ with open(compose_path, 'r+') as compose_f:
+ # load compose file
+ compose = yaml.load(compose_f)
+ # only label contraint is building
+ template['deploy']['placement']['constraints'][0] = (
+ f"{CONSTRAINTS['building']} == {hostname}")
+ # ports incremented by number of services
+ template['ports'] = [f'{1883 + number}:1883', f'{9001 + number}:9001']
+ # write template as service
+ compose['services'][service_name] = template
+ # write content starting from first line
+ compose_f.seek(0)
+ # write new compose content
+ yaml.dump(compose, compose_f)
+ # reduce file to new size
+ compose_f.truncate()
+
+
+def get_service_template(base_dir, service_name):
+ """Gets a service template entry from the template yaml
+
+ :return: yaml entry of a service
+ """
+ template_path = base_dir + '/' + TEMPLATE_DIR
+ templates = template_path + '/' + TEMPLATES_NAME
+
+ with open(templates, 'r') as templates_file:
+ template_content = yaml.load(templates_file)
+
+ return template_content['services'][service_name]
+
+# }}}
# ******************************
@@ -95,7 +285,7 @@ def generate_sftp_user_line(username, password, directories=None):
"""
# generate user line with hashed password
password_hash = crypt.crypt(password, crypt.mksalt(crypt.METHOD_SHA512))
- line = f"{username}:{password_hash}:e"
+ line = f"{username}:{password_hash}:e:{UID}:{UID}"
# add directory entries when available
if directories:
# create comma separated string from list
@@ -217,7 +407,26 @@ def generate_traefik_file(base_dir, username, password):
file_content)
-def create_or_replace_config_file(base_dir, config_path, content):
+def generate_volumerize_file(base_dir, hosts):
+ """Generates config for volumerize backups
+
+ :base_dir: path that contains custom config folder
+ :hosts: names of backup hosts
+ """
+ configs = []
+
+ for h in hosts:
+ host_config = {
+ 'description': f'Backup Server on {h}',
+ 'url': f'sftp://ohadmin@sftp_{h}://home/ohadmin/backup_data/{h}'
+ }
+ configs.append(host_config)
+
+ create_or_replace_config_file(
+ base_dir, EDIT_FILES['backup_config'], configs, json=True)
+
+
+def create_or_replace_config_file(base_dir, config_path, content, json=False):
"""Creates or replaces a config file with new content
:base_dir: path that contains custom config folder
@@ -226,7 +435,11 @@ def create_or_replace_config_file(base_dir, config_path, content):
"""
custom_path = base_dir + '/' + CUSTOM_DIR + "/" + config_path
with open(custom_path, 'w+') as file:
- file.write(content)
+ if json:
+ import json
+ json.dump(content, file, indent=2)
+ else:
+ file.write(content)
# }}}
@@ -333,7 +546,7 @@ def generate_swarm(machines):
:machines: list of machines in the swarm
"""
leader = None
- for machine in 'machines':
+ for machine in machines:
# init swarm with first machine
if leader is None:
leader = machine
@@ -550,6 +763,7 @@ def init_menu(args):
# Initialize custom configuration dirs and templates
generate_config_folders(base_dir)
+ generate_initial_compose(base_dir)
# Generate config files based on input
username = answers['username']
password = password_answers['password']
@@ -557,9 +771,16 @@ def init_menu(args):
generate_sftp_file(base_dir, username, password)
generate_mosquitto_file(base_dir, username, password)
generate_traefik_file(base_dir, username, password)
+ generate_volumerize_file(base_dir, hosts)
generate_id_rsa_files(base_dir)
generate_host_key_files(base_dir, hosts)
+ for i, host in enumerate(hosts):
+ add_sftp_service(base_dir, host, i)
+ add_openhab_service(base_dir, host)
+ add_nodered_service(base_dir, host)
+ add_mqtt_service(base_dir, host, i)
+
# print(answers)
print(f"Configuration files generated in {base_dir}")
diff --git a/requirements.in b/requirements.in
index 4c96d3a..c457593 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,4 +1,5 @@
+bcrypt
docker
PyInquirer
pyyaml
-bcrypt
+ruamel.yaml
diff --git a/requirements.txt b/requirements.txt
index 10a1e33..e87b88e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,6 +18,7 @@ pyinquirer==1.0.3
pyyaml==3.13
regex==2018.11.22 # via pyinquirer
requests==2.20.1 # via docker
+ruamel.yaml==0.15.86
six==1.11.0 # via bcrypt, docker, docker-pycreds, prompt-toolkit, websocket-client
urllib3==1.24.1 # via requests
wcwidth==0.1.7 # via prompt-toolkit
diff --git a/template_configs/docker-skeleton.yml b/template_configs/docker-skeleton.yml
index e69de29..e825e01 100644
--- a/template_configs/docker-skeleton.yml
+++ b/template_configs/docker-skeleton.yml
@@ -0,0 +1,67 @@
+version: "3.3"
+
+networks:
+ habnet:
+ driver: overlay
+ attachable: true
+
+configs:
+ backup_config:
+ file: ./volumerize/backup_config.json
+ sftp_config:
+ file: ./ssh/sshd_config
+ sftp_users:
+ file: ./ssh/sftp_users.conf
+ sftp_key_ed:
+ file: ./ssh/ssh_host_ed25519_key
+ sftp_id_pub:
+ file: ./ssh/id_rsa.pub
+ sftp_id_key:
+ file: ./ssh/id_rsa
+ sftp_known_hosts:
+ file: ./ssh/known_hosts
+ traefik_config:
+ file: ./traefik/traefik.toml
+ traefik_users:
+ file: ./traefik/traefik_users
+ nodered_settings:
+ file: ./nodered/nodered_settings.js
+ nodered_package:
+ file: ./nodered/nodered_package.json
+ mosquitto_passwords:
+ file: ./mosquitto/mosquitto_passwords
+ mosquitto_settings:
+ file: ./mosquitto/mosquitto.conf
+
+volumes:
+ openhab_addons:
+ openhab_conf:
+ openhab_userdata:
+ nodered_data:
+ mosquitto_data:
+ influxdb_data:
+ unison_data:
+ backup_data:
+ backup_cache:
+
+services:
+ proxy:
+ image: "traefik"
+ command: --api --docker --docker.swarmMode --logLevel="DEBUG"
+ volumes:
+ - "/var/run/docker.sock:/var/run/docker.sock"
+ ports:
+ - "8080:8080"
+ - "80:80"
+ networks:
+ - habnet
+ configs:
+ - source: traefik_config
+ target: /etc/traefik/traefik.toml
+ - source: traefik_users
+ target: /etc/traefik/traefik_users
+ deploy:
+ mode: global
+ placement:
+ constraints:
+ - node.role == manager
diff --git a/template_configs/docker-templates.yml b/template_configs/docker-templates.yml
new file mode 100644
index 0000000..ab3b62e
--- /dev/null
+++ b/template_configs/docker-templates.yml
@@ -0,0 +1,199 @@
+version: "3.3"
+
+networks:
+ habnet:
+ driver: overlay
+ attachable: true
+
+configs:
+ backup_config_X:
+ file: ./volumerize/backup_config_X.json
+ sftp_config:
+ file: ./ssh/sshd_config
+ sftp_users:
+ file: ./ssh/sftp_users.conf
+ sftp_key_ed:
+ file: ./ssh/ssh_host_ed25519_key
+ sftp_id_pub:
+ file: ./ssh/id_rsa.pub
+ sftp_id_key:
+ file: ./ssh/id_rsa
+ sftp_known_hosts:
+ file: ./ssh/known_hosts
+ traefik_config:
+ file: ./traefik/traefik.toml
+ traefik_users:
+ file: ./traefik/traefik_users
+ nodered_settings:
+ file: ./nodered/nodered_settings.js
+ nodered_package:
+ file: ./nodered/nodered_package.json
+ mosquitto_passwords:
+ file: ./mosquitto/mosquitto_passwords
+ mosquitto_settings:
+ file: ./mosquitto/mosquitto.conf
+
+volumes:
+ openhab_addons:
+ openhab_conf:
+ openhab_userdata:
+ nodered_data:
+ mosquitto_data:
+ influxdb_data:
+ backup_data:
+ backup_cache:
+
+services:
+ backup_X:
+ image: blacklabelops/volumerize
+ volumes:
+ - "openhab_userdata:/source/openhab_userdata"
+ - "openhab_conf:/source/openhab_conf"
+ - "openhab_addons:/source/openhab_addons"
+ - "nodered_data:/source/nodered_data"
+ - "influxdb_data:/source/influxdb_data"
+ - "backup_cache:/volumerize-cache"
+ - "backup_data:/backup"
+ configs:
+ - source: backup_config_X
+ target: /backup_config.json
+ - source: sftp_id_key
+ target: /root/.ssh/id_rsa
+ mode: 0400
+ - source: sftp_known_hosts
+ target: /root/.ssh/known_hosts
+ mode: 0400
+ environment:
+ - VOLUMERIZE_SOURCE=/source
+ - VOLUMERIZE_TARGET='multi:///backup_config.json?mode=mirror&onfail=abort'
+ - VOLUMERIZE_DUPLICITY_OPTIONS=--ssh-options "-oStrictHostKeyChecking=no"
+ networks:
+ - habnet
+ deploy:
+ placement:
+ constraints:
+ - node.labels.building == X
+ sftp_X:
+ image: "atmoz/sftp"
+ volumes:
+ - "openhab_userdata:/home/ohadmin/openhab_userdata"
+ - "openhab_conf:/home/ohadmin/openhab_conf"
+ - "nodered_data:/home/ohadmin/nodered_data"
+ - "backup_data:/home/ohadmin/backup_data"
+ configs:
+ - source: sftp_config
+ target: /etc/ssh/sshd_config
+ - source: sftp_users
+ target: /etc/sftp/users.conf
+ - source: sftp_key_ed
+ target: /etc/ssh/ssh_host_ed25519_key
+ mode: 0400
+ - source: sftp_id_key
+ target: /home/ohadmin/.ssh/id_rsa
+ uid: '9001'
+ mode: 0400
+ - source: sftp_id_pub
+ target: /home/ohadmin/.ssh/keys/sync.pub
+ networks:
+ - habnet
+ deploy:
+ placement:
+ constraints:
+ - node.labels.building == X
+ openhab_X:
+ image: "openhab/openhab:2.4.0-amd64-debian"
+ volumes:
+ - "/etc/localtime:/etc/localtime:ro"
+ - "/etc/timezone:/etc/timezone:ro"
+ - "openhab_addons:/openhab/addons"
+ - "openhab_conf:/openhab/conf"
+ - "openhab_userdata:/openhab/userdata"
+ environment:
+ OPENHAB_HTTP_PORT: "8181"
+ OPENHAB_HTTPS_PORT: "8443"
+ networks:
+ - habnet
+ deploy:
+ labels:
+ - "traefik.docker.network=ohpb_habnet"
+ - "traefik.port=8181"
+ placement:
+ constraints:
+ - node.labels.building == X
+ nodered_X:
+ image: "nodered/node-red-docker"
+ volumes:
+ - "nodered_data:/data"
+ networks:
+ - habnet
+ configs:
+ - source: nodered_package
+ target: /data/package.json
+ - source: nodered_settings
+ target: /data/settings.js
+ deploy:
+ labels:
+ - "traefik.port=1880"
+ - "traefik.docker.network=ohpb_habnet"
+ placement:
+ constraints:
+ - node.labels.building == X
+ mqtt_X:
+ image: "eclipse-mosquitto"
+ volumes:
+ - "mosquitto_data:/mosquitto/data"
+ ports:
+ configs:
+ - source: mosquitto_passwords
+ target: /mosquitto/config/passwd
+ - source: mosquitto_settings
+ target: /mosquitto/config/mosquitto.conf
+ networks:
+ - habnet
+ deploy:
+ placement:
+ constraints:
+ - node.labels.building == X
+ db_X:
+ image: "influxdb"
+ volumes:
+ - "influxdb_data:/var/lib/influxdb"
+ configs:
+ - source: influx_init
+ target: /init-influxdb.sh
+ mode: 0555
+ - source: influx_user
+ target: /run/secrets/influx_user
+ environment:
+ INFLUXDB_HTTP_AUTH_ENABLED: "true"
+ INFLUXDB_DB: "openhab"
+ INFLUXDB_ADMIN_USER: "ohadmin"
+ INFLUXDB_ADMIN_PASSWORD: "ohadmin"
+ INFLUXDB_USER: "ohadmin"
+ INFLUXDB_USER_PASSWORD: "ohtest"
+ networks:
+ - habnet
+ deploy:
+ placement:
+ constraints:
+ - node.labels.building == X
+ proxy:
+ image: "traefik"
+ command: --api --docker --docker.swarmMode --logLevel="DEBUG"
+ volumes:
+ - "/var/run/docker.sock:/var/run/docker.sock"
+ ports:
+ - "8080:8080"
+ - "80:80"
+ networks:
+ - habnet
+ configs:
+ - source: traefik_config
+ target: /etc/traefik/traefik.toml
+ - source: traefik_users
+ target: /etc/traefik/traefik_users
+ deploy:
+ mode: global
+ placement:
+ constraints:
+ - node.role == manager
--
GitLab