Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Sign in
Toggle navigation
Menu
Open sidebar
smartpublicbuilding
openhab-pb-stack
Commits
08d627d9
Commit
08d627d9
authored
Mar 21, 2019
by
dobli
Browse files
refactored volume management
parent
241ec383
Changes
4
Hide whitespace changes
Inline
Side-by-side
building_manager.py
View file @
08d627d9
...
@@ -2,6 +2,7 @@
...
@@ -2,6 +2,7 @@
""" Python module to assist creating and maintaining docker openHab stacks."""
""" Python module to assist creating and maintaining docker openHab stacks."""
import
crypt
import
crypt
from
enum
import
Enum
from
enum
import
Enum
from
typing
import
NamedTuple
import
logging
import
logging
import
os
import
os
import
sys
import
sys
...
@@ -79,20 +80,29 @@ USB_DEVICES = [{
...
@@ -79,20 +80,29 @@ USB_DEVICES = [{
}]
}]
class
Service
(
Enum
):
class
ServiceBody
(
NamedTuple
):
SFTP
=
(
"SFTP"
,
"sftp"
,
False
,
False
)
fullname
:
str
OPENHAB
=
(
"OpenHAB"
,
"openhab"
,
True
,
True
,
'dashboard'
)
prefix
:
str
NODERED
=
(
"Node-RED"
,
"nodered"
,
False
,
True
,
'ballot'
)
additional
:
bool
POSTGRES
=
(
"Postgre SQL"
,
"postgres"
,
True
,
False
)
frontend
:
bool
MQTT
=
(
"Mosquitto MQTT Broker"
,
"mqtt"
,
True
,
False
)
sftp
:
bool
=
False
FILES
=
(
"File Manager"
,
"files"
,
False
,
True
,
'folder'
)
icon
:
str
=
None
def
__init__
(
self
,
fullname
,
prefix
,
additional
,
frontend
,
icon
=
None
):
self
.
fullname
=
fullname
class
Service
(
ServiceBody
,
Enum
):
self
.
prefix
=
prefix
SFTP
=
ServiceBody
(
"SFTP"
,
"sftp"
,
False
,
False
)
self
.
additional
=
additional
OPENHAB
=
ServiceBody
(
"OpenHAB"
,
"openhab"
,
True
,
self
.
frontend
=
frontend
True
,
icon
=
'dashboard'
,
sftp
=
True
)
self
.
icon
=
icon
NODERED
=
ServiceBody
(
"Node-RED"
,
"nodered"
,
False
,
True
,
icon
=
'ballot'
,
sftp
=
True
)
POSTGRES
=
ServiceBody
(
"Postgre SQL"
,
"postgres"
,
True
,
False
)
MQTT
=
ServiceBody
(
"Mosquitto MQTT Broker"
,
"mqtt"
,
True
,
False
)
FILES
=
ServiceBody
(
"File Manager"
,
"files"
,
False
,
True
,
icon
=
'folder'
)
@
classmethod
def
service_by_prefix
(
cls
,
prefix
):
# cls here is the enumeration
return
next
(
service
for
service
in
cls
if
service
.
prefix
==
prefix
)
# >>>
# >>>
...
@@ -137,6 +147,11 @@ def add_sftp_service(base_dir, hostname, number=0):
...
@@ -137,6 +147,11 @@ def add_sftp_service(base_dir, hostname, number=0):
f
"
{
CONSTRAINTS
[
'building'
]
}
==
{
hostname
}
"
)
f
"
{
CONSTRAINTS
[
'building'
]
}
==
{
hostname
}
"
)
template
[
'ports'
]
=
[
f
'
{
2222
+
number
}
:22'
]
template
[
'ports'
]
=
[
f
'
{
2222
+
number
}
:22'
]
# attach volumes
volume_base
=
'/home/ohadmin/'
template
[
'volumes'
]
=
get_attachable_volume_list
(
base_dir
,
volume_base
,
hostname
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -169,6 +184,10 @@ def add_openhab_service(base_dir, hostname):
...
@@ -169,6 +184,10 @@ def add_openhab_service(base_dir, hostname):
f
'
{
service_name
}
.{{domain:[a-zA-z0-9-]+}}'
)
f
'
{
service_name
}
.{{domain:[a-zA-z0-9-]+}}'
)
template
[
'deploy'
][
'labels'
].
append
(
'traefik.sub.frontend.priority=2'
)
template
[
'deploy'
][
'labels'
].
append
(
'traefik.sub.frontend.priority=2'
)
# replace volumes with named entries in template
template
[
'volumes'
]
=
generate_named_volumes
(
template
[
'volumes'
],
service_name
,
compose_path
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -195,6 +214,10 @@ def add_nodered_service(base_dir, hostname):
...
@@ -195,6 +214,10 @@ def add_nodered_service(base_dir, hostname):
template
[
'deploy'
][
'labels'
].
extend
(
template
[
'deploy'
][
'labels'
].
extend
(
generate_traefik_subdomain_labels
(
service_name
,
segment
=
'sub'
))
generate_traefik_subdomain_labels
(
service_name
,
segment
=
'sub'
))
# replace volumes with named entries in template
template
[
'volumes'
]
=
generate_named_volumes
(
template
[
'volumes'
],
service_name
,
compose_path
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -218,6 +241,10 @@ def add_mqtt_service(base_dir, hostname, number=0):
...
@@ -218,6 +241,10 @@ def add_mqtt_service(base_dir, hostname, number=0):
# ports incremented by number of services
# ports incremented by number of services
template
[
'ports'
]
=
[
f
'
{
1883
+
number
}
:1883'
,
f
'
{
9001
+
number
}
:9001'
]
template
[
'ports'
]
=
[
f
'
{
1883
+
number
}
:1883'
,
f
'
{
9001
+
number
}
:9001'
]
# replace volumes with named entries in template
template
[
'volumes'
]
=
generate_named_volumes
(
template
[
'volumes'
],
service_name
,
compose_path
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -232,15 +259,21 @@ def add_postgres_service(base_dir, hostname, postfix=None):
...
@@ -232,15 +259,21 @@ def add_postgres_service(base_dir, hostname, postfix=None):
# compose file
# compose file
compose_path
=
base_path
+
'/'
+
COMPOSE_NAME
compose_path
=
base_path
+
'/'
+
COMPOSE_NAME
# use hostname as postfix when empty
# use hostname as postfix when empty
postfix
=
hostname
if
postfix
is
None
else
postfix
if
postfix
is
None
:
# service name
service_name
=
f
'postgres_
{
hostname
}
'
service_name
=
f
'postgres_
{
postfix
}
'
else
:
service_name
=
f
'postgres_
{
postfix
}
'
# template
# template
template
=
get_service_template
(
base_dir
,
Service
.
POSTGRES
.
prefix
)
template
=
get_service_template
(
base_dir
,
Service
.
POSTGRES
.
prefix
)
# only label constraint is building
# only label constraint is building
template
[
'deploy'
][
'placement'
][
'constraints'
][
0
]
=
(
template
[
'deploy'
][
'placement'
][
'constraints'
][
0
]
=
(
f
"
{
CONSTRAINTS
[
'building'
]
}
==
{
hostname
}
"
)
f
"
{
CONSTRAINTS
[
'building'
]
}
==
{
hostname
}
"
)
# replace volumes with named entries in template
template
[
'volumes'
]
=
generate_named_volumes
(
template
[
'volumes'
],
service_name
,
compose_path
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -267,6 +300,11 @@ def add_file_service(base_dir, hostname):
...
@@ -267,6 +300,11 @@ def add_file_service(base_dir, hostname):
generate_traefik_path_labels
(
service_name
,
segment
=
'main'
,
generate_traefik_path_labels
(
service_name
,
segment
=
'main'
,
redirect
=
False
))
redirect
=
False
))
# attach volumes
volume_base
=
'/srv/'
template
[
'volumes'
]
=
get_attachable_volume_list
(
base_dir
,
volume_base
,
hostname
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
add_or_update_compose_service
(
compose_path
,
service_name
,
template
)
...
@@ -294,7 +332,7 @@ def delete_service(base_dir, service_name):
...
@@ -294,7 +332,7 @@ def delete_service(base_dir, service_name):
# Functions to extract information
# Functions to extract information
def
get_current_services
(
base_dir
):
def
get_current_services
(
base_dir
,
placement
=
None
):
"""Gets a list of currently used services
"""Gets a list of currently used services
:base_dir: dir to find files in
:base_dir: dir to find files in
...
@@ -307,11 +345,130 @@ def get_current_services(base_dir):
...
@@ -307,11 +345,130 @@ def get_current_services(base_dir):
# load compose file
# load compose file
compose
=
yaml
.
load
(
compose_f
)
compose
=
yaml
.
load
(
compose_f
)
# generate list of names
# generate list of names
service_names
=
[
n
for
n
in
compose
[
'services'
]]
service_names
=
[]
for
(
name
,
entry
)
in
compose
[
'services'
].
items
():
if
placement
is
None
or
get_building_of_entry
(
entry
)
==
placement
:
service_names
.
append
(
name
)
return
service_names
return
service_names
def
get_building_of_entry
(
service_dict
):
"""Extract the configured building constraint from an yaml service entry
:service_dict: service dict from yaml
:returns: building that is set
"""
# get constraints
constraint_list
=
service_dict
[
'deploy'
][
'placement'
][
'constraints'
]
# convert them to dicts
label_dict
=
{
i
.
split
(
"=="
)[
0
].
strip
():
i
.
split
(
"=="
)[
1
].
strip
()
for
i
in
constraint_list
}
return
label_dict
.
get
(
'node.labels.building'
)
def
get_service_entry_info
(
service_entry
):
"""Gets service name and instance of a service entry
:service_entry: service entry name
:return: tuple with service_name and instance name
"""
entry_split
=
service_entry
.
split
(
"_"
)
name
=
entry_split
[
0
]
instance
=
entry_split
[
1
]
return
name
,
instance
def
get_service_volumes
(
base_dir
,
service_name
):
"""Gets a list of volumes of a service
:base_dir: dir to find files in
:returns: list of volumes
"""
base_path
=
base_dir
+
'/'
+
CUSTOM_DIR
# compose file
compose_path
=
base_path
+
'/'
+
COMPOSE_NAME
with
open
(
compose_path
,
'r'
)
as
compose_f
:
# load compose file
compose
=
yaml
.
load
(
compose_f
)
# load service
service
=
compose
[
'services'
].
get
(
service_name
)
# extract volume names
volume_dict
=
yaml_list_to_dict
(
service
[
'volumes'
])
volumes
=
list
(
volume_dict
.
keys
())
# filter only named volumes
named_volumes
=
[
v
for
v
in
volumes
if
'/'
not
in
v
]
return
named_volumes
# Helper functions
# Helper functions
def
get_attachable_volume_list
(
base_dir
,
volume_base
,
host
):
"""Get a list of volumes from a host that can be attatched for file acccess
:base_dir: Base config dir
:volume_base: Base path of volumes
:host: host to consider
:returns: list of attachable volume entries
"""
volume_list
=
[]
host_services
=
get_current_services
(
base_dir
,
host
)
for
host_service
in
host_services
:
name
,
instance
=
get_service_entry_info
(
host_service
)
volume_service
=
Service
.
service_by_prefix
(
name
)
if
volume_service
.
sftp
:
volumes
=
get_service_volumes
(
base_dir
,
host_service
)
vlist
=
[
f
'
{
v
}
:
{
volume_base
}{
v
}
'
for
v
in
volumes
]
volume_list
.
extend
(
vlist
)
return
volume_list
def
generate_named_volumes
(
template_volume_list
,
service_name
,
compose_path
):
"""Generates volumes including name of services and ads them to
the compose file
:template_volume_list: List of volume entries from template
:service_name: Name of the service instance
:compose_path: path to compose file
:returns: list of named entries
"""
volume_entries
=
yaml_list_to_dict
(
template_volume_list
)
# add name to entries (that are named volumes
named_volume_entries
=
{}
for
(
volume
,
target
)
in
volume_entries
.
items
():
if
"/"
not
in
volume
:
named_volume_entries
[
f
"
{
service_name
}
_
{
volume
}
"
]
=
target
else
:
named_volume_entries
[
f
"
{
volume
}
"
]
=
target
for
(
volume
,
target
)
in
named_volume_entries
.
items
():
# declare volume if it is a named one
if
"/"
not
in
volume
:
add_volume_entry
(
compose_path
,
volume
)
return
dict_to_yaml_list
(
named_volume_entries
)
def
yaml_list_to_dict
(
yaml_list
):
"""Converts a yaml list (volumes, configs etc) into a python dict
:yaml_list: list of a yaml containing colon separated entries
:return: python dict
"""
return
{
i
.
split
(
":"
)[
0
]:
i
.
split
(
":"
)[
1
]
for
i
in
yaml_list
}
def
dict_to_yaml_list
(
pdict
):
"""Converts a python dict into a yaml list (volumes, configs etc)
:pdict: python dict
:return: list of a yaml containing colon separated entries
"""
return
[
f
'
{
k
}
:
{
v
}
'
for
(
k
,
v
)
in
pdict
.
items
()]
def
get_service_template
(
base_dir
,
service_name
):
def
get_service_template
(
base_dir
,
service_name
):
"""Gets a service template entry from the template yaml
"""Gets a service template entry from the template yaml
...
@@ -410,6 +567,25 @@ def add_or_update_compose_service(compose_path, service_name, service_content):
...
@@ -410,6 +567,25 @@ def add_or_update_compose_service(compose_path, service_name, service_content):
yaml
.
dump
(
compose
,
compose_f
)
yaml
.
dump
(
compose
,
compose_f
)
# reduce file to new size
# reduce file to new size
compose_f
.
truncate
()
compose_f
.
truncate
()
def
add_volume_entry
(
compose_path
,
volume_name
):
"""Creates an additional volume entry in the stack file
:compose_path: path of the compose file to change
:volume_name: name of the additional volume
"""
with
open
(
compose_path
,
'r+'
)
as
compose_f
:
# load compose file
compose
=
yaml
.
load
(
compose_f
)
# add volume
compose
[
'volumes'
][
volume_name
]
=
None
# write content starting from first line
compose_f
.
seek
(
0
)
# write new compose content
yaml
.
dump
(
compose
,
compose_f
)
# reduce file to new size
compose_f
.
truncate
()
# >>>
# >>>
...
@@ -955,11 +1131,10 @@ def execute_command_on_machine(command, machine):
...
@@ -955,11 +1131,10 @@ def execute_command_on_machine(command, machine):
run
([
f
'docker-machine ssh
{
machine
}
{
command
}
'
],
shell
=
True
)
run
([
f
'docker-machine ssh
{
machine
}
{
command
}
'
],
shell
=
True
)
# >>>
# >>>
# ******************************
# ******************************
# Systemd functions <<<
# Systemd functions <<<
# ******************************
# ******************************
def
list_enabled_devices
():
def
list_enabled_devices
():
"""Presents a list of enabled devices (systemd services)
"""Presents a list of enabled devices (systemd services)
:returns: list of enabled devices
:returns: list of enabled devices
...
@@ -1279,8 +1454,6 @@ def init_machine_menu(base_dir, host, increment):
...
@@ -1279,8 +1454,6 @@ def init_machine_menu(base_dir, host, increment):
services
=
qust
.
checkbox
(
f
'What services shall
{
host
}
provide?'
,
services
=
qust
.
checkbox
(
f
'What services shall
{
host
}
provide?'
,
choices
=
generate_cb_service_choices
(
checked
=
True
),
choices
=
generate_cb_service_choices
(
checked
=
True
),
style
=
st
).
ask
()
style
=
st
).
ask
()
if
Service
.
SFTP
in
services
:
add_sftp_service
(
base_dir
,
host
,
increment
)
if
Service
.
OPENHAB
in
services
:
if
Service
.
OPENHAB
in
services
:
add_openhab_service
(
base_dir
,
host
)
add_openhab_service
(
base_dir
,
host
)
if
Service
.
NODERED
in
services
:
if
Service
.
NODERED
in
services
:
...
@@ -1291,6 +1464,8 @@ def init_machine_menu(base_dir, host, increment):
...
@@ -1291,6 +1464,8 @@ def init_machine_menu(base_dir, host, increment):
add_postgres_service
(
base_dir
,
host
)
add_postgres_service
(
base_dir
,
host
)
if
Service
.
FILES
in
services
:
if
Service
.
FILES
in
services
:
add_file_service
(
base_dir
,
host
)
add_file_service
(
base_dir
,
host
)
if
Service
.
SFTP
in
services
:
add_sftp_service
(
base_dir
,
host
,
increment
)
return
building
,
services
return
building
,
services
...
...
template_configs
/docker-stack.yml
→
legacy
/docker-stack.yml
View file @
08d627d9
File moved
template_configs/docker-skeleton.yml
View file @
08d627d9
...
@@ -45,13 +45,6 @@ configs:
...
@@ -45,13 +45,6 @@ configs:
file
:
./filebrowser/filebrowser.json
file
:
./filebrowser/filebrowser.json
volumes
:
volumes
:
openhab_addons
:
openhab_conf
:
openhab_userdata
:
nodered_data
:
mosquitto_data
:
influxdb_data
:
postgres_data
:
backup_data
:
backup_data
:
backup_cache
:
backup_cache
:
...
...
template_configs/docker-templates.yml
View file @
08d627d9
...
@@ -45,13 +45,6 @@ configs:
...
@@ -45,13 +45,6 @@ configs:
file
:
./filebrowser/filebrowser.json
file
:
./filebrowser/filebrowser.json
volumes
:
volumes
:
openhab_addons
:
openhab_conf
:
openhab_userdata
:
nodered_data
:
mosquitto_data
:
influxdb_data
:
postgres_data
:
backup_data
:
backup_data
:
backup_cache
:
backup_cache
:
...
@@ -88,9 +81,6 @@ services:
...
@@ -88,9 +81,6 @@ services:
sftp
:
sftp
:
image
:
"
atmoz/sftp"
image
:
"
atmoz/sftp"
volumes
:
volumes
:
-
"
openhab_userdata:/home/ohadmin/openhab_userdata"
-
"
openhab_conf:/home/ohadmin/openhab_conf"
-
"
nodered_data:/home/ohadmin/nodered_data"
-
"
backup_data:/home/ohadmin/backup_data"
-
"
backup_data:/home/ohadmin/backup_data"
configs
:
configs
:
-
source
:
sftp_config
-
source
:
sftp_config
...
@@ -117,9 +107,9 @@ services:
...
@@ -117,9 +107,9 @@ services:
volumes
:
volumes
:
-
"
/etc/localtime:/etc/localtime:ro"
-
"
/etc/localtime:/etc/localtime:ro"
-
"
/etc/timezone:/etc/timezone:ro"
-
"
/etc/timezone:/etc/timezone:ro"
-
"
openhab_
addons:/openhab/addons"
-
"
addons:/openhab/addons"
-
"
openhab_
conf:/openhab/conf"
-
"
conf:/openhab/conf"
-
"
openhab_
userdata:/openhab/userdata"
-
"
userdata:/openhab/userdata"
environment
:
environment
:
OPENHAB_HTTP_PORT
:
"
8181"
OPENHAB_HTTP_PORT
:
"
8181"
OPENHAB_HTTPS_PORT
:
"
8443"
OPENHAB_HTTPS_PORT
:
"
8443"
...
@@ -136,7 +126,7 @@ services:
...
@@ -136,7 +126,7 @@ services:
nodered
:
nodered
:
image
:
"
nodered/node-red-docker"
image
:
"
nodered/node-red-docker"
volumes
:
volumes
:
-
"
nodered_
data:/data"
-
"
data:/data"
networks
:
networks
:
-
habnet
-
habnet
configs
:
configs
:
...
@@ -154,7 +144,7 @@ services:
...
@@ -154,7 +144,7 @@ services:
mqtt
:
mqtt
:
image
:
"
eclipse-mosquitto"
image
:
"
eclipse-mosquitto"
volumes
:
volumes
:
-
"
mosquitto_
data:/mosquitto/data"
-
"
data:/mosquitto/data"
ports
:
ports
:
configs
:
configs
:
-
source
:
mosquitto_passwords
-
source
:
mosquitto_passwords
...
@@ -170,7 +160,7 @@ services:
...
@@ -170,7 +160,7 @@ services:
db
:
db
:
image
:
"
influxdb"
image
:
"
influxdb"
volumes
:
volumes
:
-
"
influxdb_
data:/var/lib/influxdb"
-
"
data:/var/lib/influxdb"
configs
:
configs
:
-
source
:
influx_init
-
source
:
influx_init
target
:
/init-influxdb.sh
target
:
/init-influxdb.sh
...
@@ -193,7 +183,7 @@ services:
...
@@ -193,7 +183,7 @@ services:
postgres
:
postgres
:
image
:
"
postgres"
image
:
"
postgres"
volumes
:
volumes
:
-
"
postgres_
data:/var/lib/postgresql/data/pgdata"
-
"
data:/var/lib/postgresql/data/pgdata"
configs
:
configs
:
-
source
:
postgres_user
-
source
:
postgres_user
target
:
/run/secrets/postgres_user
target
:
/run/secrets/postgres_user
...
@@ -233,8 +223,6 @@ services:
...
@@ -233,8 +223,6 @@ services:
files
:
files
:
image
:
filebrowser/filebrowser
image
:
filebrowser/filebrowser
volumes
:
volumes
:
-
openhab_conf:/srv/openHAB
-
nodered_data:/srv/Node-RED
configs
:
configs
:
-
source
:
filebrowser
-
source
:
filebrowser
target
:
/.filebrowser.json
target
:
/.filebrowser.json
...
@@ -247,14 +235,3 @@ services:
...
@@ -247,14 +235,3 @@ services:
placement
:
placement
:
constraints
:
constraints
:
-
node.labels.building == X
-
node.labels.building == X
zwave_oh
:
image
:
docker
command
:
"
docker
run
--rm
--name
device_oh
--network
habnet
-v
/etc/localtime:/etc/localtime:ro
-v
/etc/timezone:/etc/timezone:ro
-v
openhab_zwave_conf:/openhab/conf
-v
openhab_zwave_userdata:/openhab/userdata
-p
9898:8080
openhab/openhab:2.4.0"
volumes
:
-
"
/var/run/docker.sock:/var/run/docker.sock"
networks
:
-
habnet
deploy
:
placement
:
constraints
:
-
node.labels.device == zwave
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment