From f866e7f59605501c918f52545aae4c097b28eb57 Mon Sep 17 00:00:00 2001 From: timeshifter Date: Sun, 18 Jul 2021 19:16:41 +0200 Subject: [PATCH] added code --- binaries.py | 3 + config.py | 30 ++++++++ zfs-backup.py | 210 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 binaries.py create mode 100644 config.py create mode 100755 zfs-backup.py diff --git a/binaries.py b/binaries.py new file mode 100644 index 0000000..7ab6f4e --- /dev/null +++ b/binaries.py @@ -0,0 +1,3 @@ +ZFS = '/usr/bin/zfs' +ZPOOL = '/usr/bin/zpool' +CRYPTSETUP = '/usr/bin/cryptsetup' diff --git a/config.py b/config.py new file mode 100644 index 0000000..1fdf042 --- /dev/null +++ b/config.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- + +# Name des lokalen Pools +local_pool = 'tank' + +# Name der Backup-Platten und UUID der Partition +backup_pool = { + 'backup1': '603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4', + 'backup2': '8632e9e4-3d6c-4627-8c28-ec72f52db281', +} + +# Datasets, die nicht gesichert werden sollen +do_not_backup = [ + 'tank/cache', + 'tank/docker', +] + +# überprüfe ob neue Snapshots mit diesen String-Teilen vorliegen +snapshot_tag = ['znap', 'monthly'] + +# wie viele Snapshots von jedem Intervall sollen beibehalten werden? +# dies sollte der Serverkonfiguration zu zfs-auto-snapshots folgen +keep = { + 'frequent' : 4, + 'hourly' : 24, + 'daily' : 31, + 'weekly' : 8, + 'monthly' : 12, +} + diff --git a/zfs-backup.py b/zfs-backup.py new file mode 100755 index 0000000..1ebee99 --- /dev/null +++ b/zfs-backup.py @@ -0,0 +1,210 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +import os +from subprocess import Popen, PIPE, check_output +from binaries import ZFS, ZPOOL, CRYPTSETUP +import config +from sys import exit, stdout +import re + + +""" +Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit +# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1 +""" + +class Command(): + def __init__(self, command=[], strict=True): + process = Popen(command, stdout=PIPE) + self.stdout, self.stderr = process.communicate() + self.exit_code = process.wait() + if strict and self.exit_code != 0: + raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') + + +def find_backup_pool(): + print('Suche nach externer Festplatte für Backups') + present_uuids = Command(['ls', '/dev/disk/by-uuid']) + for key in list(config.backup_pool.keys()): + uuid = config.backup_pool[key] + if uuid in present_uuids.stdout.decode(): + print('Gefundene Platte: ' + key) + return key + else: + raise IOError('Konnte keine Backup-Platte finden!') + + +def get_datasets_to_backup(): + datasets_str = Command([ZFS, 'list', '-H', '-d', '1', '-o', 'name', config.local_pool]).stdout.strip().decode() + # we omit the first return value, which is the pool, not the dataset + datasets_list = datasets_str.split('\n')[1:] + to_backup = [d for d in datasets_list if d not in config.do_not_backup] + print('Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:') + for tb in to_backup: + print(' ' + tb) + return to_backup + + +def get_snapshots(pool): + return Command([ZFS, 'list', '-r', '-t', 'snapshot', '-o', 'name', '-H', pool]).stdout.strip().decode().split('\n') + + +def last_snapshot(pool): + snapshots = get_snapshots(pool) + # check if the snapshot_tags appear in the snapshot name in order they are provided + relevant = [s for s in snapshots if re.search('@' + '.*'.join(config.snapshot_tag), s)] + # sort chronologically + relevant.sort() + # the most recent snapshot is the last one + try: + last_entry = relevant[-1] + # didn't find any snapshots at all + except IndexError: + return + last_snapshot = last_entry.strip().split('@')[-1] + return last_snapshot + + +def export_pool(pool): + container_name = 'crypt-' + pool + print('Exportiere Pool ' + pool) + Command([ZPOOL, 'export', pool]) + print('Schließe verschlüsselten Container') + Command([CRYPTSETUP, 'close', container_name]) + print('Fertig!') + print('Sie können die externe Festplatte jetzt entfernen.') + + +def import_pool(pool): + uuid = config.backup_pool[pool] + container_name = 'crypt-' + pool + present_mappers = Command(['ls', '/dev/mapper/']).stdout.decode().strip() + if container_name in present_mappers: + msg = 'verschlüsselter Container ' + container_name + \ + ' scheint bereits in Verwendung zu sein. Breche ab.' + raise Exception(msg) + print('Öffne verschlüsselten Container\n') + Command([CRYPTSETUP, 'open', '/dev/disk/by-uuid/' + uuid, container_name]) + print('') + print('Importiere Pool ' + pool) + # -N: no mount + # -d: directory to search the pool in + Command([ZPOOL, 'import', '-N', '-d', '/dev/mapper', pool]) + + +def present_locally(snapshot): + local_snapshots = get_snapshots(config.local_pool) + for each_snapshot in local_snapshots: + if snapshot in each_snapshot: + return True + else: + return False + + +def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool): + print('Sichere Dataset ' + dataset) + #print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot) + #return + if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool): + print(' Dieses Dataset wurde nicht zur Sicherung eingerichtet.') + print(' Verwende ein Kommando in der Art') + print(' # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN') + return + if last_local_snapshot == last_remote_snapshot: + print(' Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot') + print(' Führe kein Backup für dieses Dataset durch') + return + send = Popen([ + ZFS, 'send', '-R', + '-I', last_remote_snapshot, + dataset + '@' + last_local_snapshot], + stdout=PIPE) + # -d Discard the first element of the sent snapshot's file system name + # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. + # -u File system that is associated with the received stream is not mounted. + receive = check_output([ZFS, 'receive', '-d', '-F', '-u', backup_pool], stdin=send.stdout) + exit_code = send.wait() + if exit_code != 0: + raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') + + +def snapshot_present_in_dataset(dataset, last_snapshot, pool): + snapshots = get_snapshots(pool) + # get just the dataset name without the pool + actual_dataset = dataset.split('/')[-1] + for snapshot in snapshots: + if (actual_dataset + '@' + last_snapshot) in snapshot: + return True + else: + return False + + +def clean_old_snapshots(dataset, pool): + snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' + # get just the dataset name without the pool + actual_dataset = dataset.split('/')[-1] + #print('actual_dataset', actual_dataset) + for interval in config.keep.keys(): + #print('interval', interval) + # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... + pattern = actual_dataset + '@' + config.snapshot_tag[0] + '.*' + interval + snapshots_in_interval = [] + for snapshot in snapshots: + result = re.search(pattern, snapshot) + if result is not None: + snapshots_in_interval.append(snapshot) + snapshots_in_interval.sort() + for i in range(len(snapshots_in_interval) - config.keep[interval]): + to_delete = snapshots_in_interval[i] + print(' Lösche veralteten Snapshot {}'.format(to_delete)) + zfs_destroy(to_delete, recursive=True) + + +def zfs_destroy(full_name, recursive=False): + if config.local_pool in full_name: + msg = 'Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!' + raise Exception(msg) + else: + cmdline = [ZFS, 'destroy'] + if recursive: + cmdline.append('-r') + cmdline.append(full_name) + Command(cmdline) + + +def main(): + if os.getuid() > 0: + print('Muss als root ausgeführt werden.') + exit(1) + datasets = get_datasets_to_backup() + backup_pool = find_backup_pool() + import_pool(backup_pool) + try: + last_local_snapshot = last_snapshot(config.local_pool) + if not last_local_snapshot: + print('Es scheint keine Snapshots im Speicherpool zu geben') + print('Breche ab.') + export_pool(backup_pool) + exit() + last_remote_snapshot = last_snapshot(backup_pool) + if not last_remote_snapshot: + print('Es scheint keine Snapshots auf der Backup-Festplatte zu geben') + print('Breche ab.') + export_pool(backup_pool) + exit() + if not present_locally(last_remote_snapshot): + print(""" + Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server + keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor + zu langer Zeit das letzte mal als Backup verwendet""") + return + for dataset in datasets: + do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool) + clean_old_snapshots(dataset, backup_pool) + finally: + export_pool(backup_pool) + + +if __name__ == '__main__': + main()