diff --git a/bundles/backup-client/items.py b/bundles/backup-client/items.py index baed1ad..a0684ef 100644 --- a/bundles/backup-client/items.py +++ b/bundles/backup-client/items.py @@ -1,5 +1,22 @@ from os.path import join +if node.has_bundle('zfs'): + wanted_paths = node.metadata.get('backups/paths', set()) + snapshot_paths = node.metadata.get('zfs/filesystems_with_backup_snapshots', {}) + backup_paths = set() + + for path in wanted_paths: + path_found = False + for zfs_paths in snapshot_paths.values(): + if path in zfs_paths: + backup_paths.add(f'/mnt/backup-snapshot{path}') + path_found = True + + if not path_found: + backup_paths.add(path) +else: + backup_paths = node.metadata.get('backups/paths', set()) + if node.metadata.get('backups/exclude_from_backups', False): files['/etc/backup.priv'] = { 'delete': True, @@ -17,7 +34,7 @@ else: 'username': node.metadata['backup-client']['user-name'], 'server': server, 'port': port, - 'paths': node.metadata.get('backups/paths', {}), + 'paths': backup_paths, }, 'mode': '0700', } diff --git a/bundles/influxdb2/items.py b/bundles/influxdb2/items.py index 666d06c..e842e7f 100644 --- a/bundles/influxdb2/items.py +++ b/bundles/influxdb2/items.py @@ -11,6 +11,17 @@ directories = { }, } +files = { + '/etc/zfs-snapshot-backup-pre.d/50-influxdb': { + 'content': '#!/bin/sh\nsystemctl stop influxdb', + 'mode': '0755', + }, + '/etc/zfs-snapshot-backup-post.d/50-influxdb': { + 'content': '#!/bin/sh\nsystemctl start influxdb', + 'mode': '0755', + }, +} + svc_systemd = { 'influxdb': { diff --git a/bundles/influxdb2/metadata.py b/bundles/influxdb2/metadata.py index a56dcc7..87f79e0 100644 --- a/bundles/influxdb2/metadata.py +++ b/bundles/influxdb2/metadata.py @@ -15,6 +15,11 @@ defaults = { }, }, }, + 'backups': { + 'paths': { + '/var/lib/influxdb', + }, + }, 'icinga2_api': { 'telegraf': { 'services': { diff --git a/bundles/postgresql/items.py b/bundles/postgresql/items.py index 6c475b7..8fa2ef2 100644 --- a/bundles/postgresql/items.py +++ b/bundles/postgresql/items.py @@ -60,9 +60,7 @@ files = { }, } -# FIXME currently we do not have a mechanism to use snapshot-backups of -# zfs datasets. -if node.has_bundle('backup-client'): # and not node.has_bundle('zfs'): +if node.has_bundle('backup-client') and not node.has_bundle('zfs'): files['/etc/backup-pre-hooks.d/90-postgresql-dump-all'] = { 'source': 'backup-pre-hook', 'content_type': 'mako', @@ -72,6 +70,10 @@ if node.has_bundle('backup-client'): # and not node.has_bundle('zfs'): 'mode': '0700', } directories['/var/tmp/postgresdumps'] = {} +else: + files['/var/tmp/postgresdumps'] = { + 'delete': True, + } postgres_roles = { 'root': { diff --git a/bundles/postgresql/metadata.py b/bundles/postgresql/metadata.py index 15c6351..6a76107 100644 --- a/bundles/postgresql/metadata.py +++ b/bundles/postgresql/metadata.py @@ -2,9 +2,6 @@ defaults = { 'backups': { 'paths': { '/var/lib/postgresql', - - # FIXME - '/var/tmp/postgresdumps', }, }, 'bash_functions': { @@ -66,9 +63,8 @@ if node.has_bundle('zfs'): }, }, } -# FIXME -#else: -# defaults['backups']['paths'].add('/var/tmp/postgresdumps') +else: + defaults['backups']['paths'].add('/var/tmp/postgresdumps') @metadata_reactor.provides( diff --git a/bundles/zfs/files/backup-pre-hook b/bundles/zfs/files/backup-pre-hook new file mode 100644 index 0000000..a5041b0 --- /dev/null +++ b/bundles/zfs/files/backup-pre-hook @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +run-parts --exit-on-error -- /etc/zfs-snapshot-backup-pre.d + + +% for dataset in sorted(node.metadata.get('zfs/filesystems_with_backup_snapshots')): +if zfs get type ${dataset}@snapshot-backup >/dev/null 2>&1 +then + zfs destroy -f ${dataset}@snapshot-backup +fi + +zfs snapshot ${dataset}@snapshot-backup + +mkdir -p /mnt/backup-snapshot${node.metadata['zfs']['datasets'][dataset]['mountpoint']} +mount -t zfs ${dataset}@snapshot-backup /mnt/backup-snapshot${node.metadata['zfs']['datasets'][dataset]['mountpoint']} + + +% endfor +run-parts --exit-on-error -- /etc/zfs-snapshot-backup-post.d diff --git a/bundles/zfs/items.py b/bundles/zfs/items.py index 1322250..da80b73 100644 --- a/bundles/zfs/items.py +++ b/bundles/zfs/items.py @@ -95,7 +95,11 @@ directories = { "/etc/zfs-snapshot-backup-post.d": { 'purge': True, }, - "/etc/zfs-snapshot-backup-final.d": { - 'purge': True, - }, } + +if node.metadata.get('zfs/filesystems_with_backup_snapshots', {}) and node.has_bundle('backup-client'): + files['/etc/backup-pre-hooks.d/99-zfs-snapshots'] = { + 'content_type': 'mako', + 'source': 'backup-pre-hook', + 'mode': '0755', + } diff --git a/bundles/zfs/metadata.py b/bundles/zfs/metadata.py index 074b14c..b2aa7a8 100644 --- a/bundles/zfs/metadata.py +++ b/bundles/zfs/metadata.py @@ -104,7 +104,9 @@ def zfs_scrub_cronjob(metadata): } -@metadata_reactor +@metadata_reactor.provides( + 'icinga2_api/zfs/services', +) def monitoring(metadata): if not node.has_bundle('sshmon'): raise DoNotRunAgain @@ -129,3 +131,25 @@ def monitoring(metadata): }, }, } + + +@metadata_reactor.provides( + 'zfs/filesystems_with_backup_snapshots', +) +def backups_with_snapshot(metadata): + backups = metadata.get('backups/paths', set()) + datasets = metadata.get('zfs/datasets', {}) + + backups_in_zfs_datasets = {} + + for path in backups: + for dname, dconfig in datasets.items(): + if 'mountpoint' in dconfig: + if path[:len(dconfig['mountpoint'])] == dconfig['mountpoint']: + backups_in_zfs_datasets.setdefault(dname, set()).add(path) + + return { + 'zfs': { + 'filesystems_with_backup_snapshots': backups_in_zfs_datasets, + }, + } diff --git a/data/backup/keys/htz-cloud.influxdb.key.vault b/data/backup/keys/htz-cloud.influxdb.key.vault new file mode 100644 index 0000000..296f65c --- /dev/null +++ b/data/backup/keys/htz-cloud.influxdb.key.vault @@ -0,0 +1 @@ +encrypt$gAAAAABgoBM4n7eEwMxfav4200d5LoGyLsM67Ps9CCPBgjZjdsOpyQulbp2l4LIhWj1kljgrXZ_LnOaRUf3S8q7-drv3yEfG1d-cyK8vP7r8wkTcawaazM5XdhR8VkgxCyBCuOZmM1vvAmOwIMi1JiQcgiJ4G32ThS085onN3T9HvEu2a9sWYuOlk-yVUBpelqP97vbO6r2n3hn-62AC7Ww-Q_EQ_kcdDdJLOawNe1anJOsOeLb1XOlMIJWI74LZXfszsRi9LmxUpzaB4Gd_nzDLO1AZHD_GOf9UOeeab8PujwhQ4UhbEHCdB-uVH88LGCw25-6eiv0yA_kRulj7InA9sKRyBZ1okSF4Xhl-htez6XwBD6BuA_ly6ulSxWuoOV_qsCNiXhJYGTuKPJS0-wpLyeLb_PV1tlYOKZv3VK5a_EpCRa4fCdX7oj9pcA8ZQkQeFAx0P4b3oYkz8YkDiBINFdOLE177lC6Kuk33sLfsZBuoR7MSjHUtZHPOXUHu8pV0o0_YxxF5fBs6hyReXvCbZB18NikeH4Ki-RR5IE4ofTngf6dIQhCxp9u-cNs-mNP0GRKiYIBUInr9Udpr-ymXRq-7OvyTM92950ePm-qoqjbpVFA8HN057WfkW6N6DASRRa2HjqQJ \ No newline at end of file diff --git a/data/backup/keys/htz-cloud.influxdb.pub b/data/backup/keys/htz-cloud.influxdb.pub new file mode 100644 index 0000000..b6016d6 --- /dev/null +++ b/data/backup/keys/htz-cloud.influxdb.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIFpAUytQ0ncucltODEr3MgcKF5U/6TS4zZG1OYJDIEQp kunsi@kunsi-t470.kunbox.net diff --git a/nodes/htz-cloud/influxdb.py b/nodes/htz-cloud/influxdb.py index af5af71..2e0e40b 100644 --- a/nodes/htz-cloud/influxdb.py +++ b/nodes/htz-cloud/influxdb.py @@ -31,10 +31,6 @@ nodes['htz-cloud.influxdb'] = { }, }, }, - 'backups': { - # TODO enable this once we have zfs-snapshot-backups - 'exclude_from_backups': True, - }, 'grafana': { 'domain': 'grafana.kunsmann.eu', 'login_max_duration': '30d',