From 587b677a37b58b5f353e9e7102b696a2f95103c3 Mon Sep 17 00:00:00 2001 From: timeshifter Date: Thu, 25 Nov 2021 15:30:44 +0100 Subject: [PATCH] initial cleanup of project --- binaries.py | 3 - config.py | 25 ++++---- zfs-backup.py | 164 +++++++++++++++++++++++++++++--------------------- 3 files changed, 111 insertions(+), 81 deletions(-) delete mode 100644 binaries.py diff --git a/binaries.py b/binaries.py deleted file mode 100644 index 7ab6f4e..0000000 --- a/binaries.py +++ /dev/null @@ -1,3 +0,0 @@ -ZFS = '/usr/bin/zfs' -ZPOOL = '/usr/bin/zpool' -CRYPTSETUP = '/usr/bin/cryptsetup' diff --git a/config.py b/config.py index 1fdf042..0920591 100644 --- a/config.py +++ b/config.py @@ -1,30 +1,33 @@ # -*- coding: utf-8 -*- # Name des lokalen Pools -local_pool = 'tank' +local_pool = "tank" # Name der Backup-Platten und UUID der Partition backup_pool = { - 'backup1': '603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4', - 'backup2': '8632e9e4-3d6c-4627-8c28-ec72f52db281', + "backup1": "603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4", + "backup2": "8632e9e4-3d6c-4627-8c28-ec72f52db281", } # Datasets, die nicht gesichert werden sollen do_not_backup = [ - 'tank/cache', - 'tank/docker', + "tank/cache", + "tank/docker", ] # überprüfe ob neue Snapshots mit diesen String-Teilen vorliegen -snapshot_tag = ['znap', 'monthly'] +snapshot_tag = ["znap", "monthly"] # wie viele Snapshots von jedem Intervall sollen beibehalten werden? # dies sollte der Serverkonfiguration zu zfs-auto-snapshots folgen keep = { - 'frequent' : 4, - 'hourly' : 24, - 'daily' : 31, - 'weekly' : 8, - 'monthly' : 12, + "frequent": 4, + "hourly": 24, + "daily": 31, + "weekly": 8, + "monthly": 12, } +ZFS = "/usr/bin/zfs" +ZPOOL = "/usr/bin/zpool" +CRYPTSETUP = "/usr/bin/cryptsetup" diff --git a/zfs-backup.py b/zfs-backup.py index 1ebee99..31b616c 100755 --- a/zfs-backup.py +++ b/zfs-backup.py @@ -3,9 +3,8 @@ import os from subprocess import Popen, PIPE, check_output -from binaries import ZFS, ZPOOL, CRYPTSETUP import config -from sys import exit, stdout +from sys import exit import re @@ -14,46 +13,58 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit # zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1 """ -class Command(): - def __init__(self, command=[], strict=True): + +class Command: + def __init__(self, command, strict=True): process = Popen(command, stdout=PIPE) self.stdout, self.stderr = process.communicate() self.exit_code = process.wait() if strict and self.exit_code != 0: - raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') + raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.") def find_backup_pool(): - print('Suche nach externer Festplatte für Backups') - present_uuids = Command(['ls', '/dev/disk/by-uuid']) + print("Suche nach externer Festplatte für Backups") + present_uuids = Command(["ls", "/dev/disk/by-uuid"]) for key in list(config.backup_pool.keys()): uuid = config.backup_pool[key] if uuid in present_uuids.stdout.decode(): - print('Gefundene Platte: ' + key) + print("Gefundene Platte: " + key) return key else: - raise IOError('Konnte keine Backup-Platte finden!') + raise IOError("Konnte keine Backup-Platte finden!") def get_datasets_to_backup(): - datasets_str = Command([ZFS, 'list', '-H', '-d', '1', '-o', 'name', config.local_pool]).stdout.strip().decode() + datasets_str = ( + Command([config.ZFS, "list", "-H", "-d", "1", "-o", "name", config.local_pool]) + .stdout.strip() + .decode() + ) # we omit the first return value, which is the pool, not the dataset - datasets_list = datasets_str.split('\n')[1:] + datasets_list = datasets_str.split("\n")[1:] to_backup = [d for d in datasets_list if d not in config.do_not_backup] - print('Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:') + print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:") for tb in to_backup: - print(' ' + tb) + print(" " + tb) return to_backup def get_snapshots(pool): - return Command([ZFS, 'list', '-r', '-t', 'snapshot', '-o', 'name', '-H', pool]).stdout.strip().decode().split('\n') + return ( + Command([config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool]) + .stdout.strip() + .decode() + .split("\n") + ) def last_snapshot(pool): snapshots = get_snapshots(pool) # check if the snapshot_tags appear in the snapshot name in order they are provided - relevant = [s for s in snapshots if re.search('@' + '.*'.join(config.snapshot_tag), s)] + relevant = [ + s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s) + ] # sort chronologically relevant.sort() # the most recent snapshot is the last one @@ -62,35 +73,38 @@ def last_snapshot(pool): # didn't find any snapshots at all except IndexError: return - last_snapshot = last_entry.strip().split('@')[-1] + last_snapshot = last_entry.strip().split("@")[-1] return last_snapshot def export_pool(pool): - container_name = 'crypt-' + pool - print('Exportiere Pool ' + pool) - Command([ZPOOL, 'export', pool]) - print('Schließe verschlüsselten Container') - Command([CRYPTSETUP, 'close', container_name]) - print('Fertig!') - print('Sie können die externe Festplatte jetzt entfernen.') + container_name = "crypt-" + pool + print("Exportiere Pool " + pool) + Command([config.ZPOOL, "export", pool]) + print("Schließe verschlüsselten Container") + Command([config.CRYPTSETUP, "close", container_name]) + print("Fertig!") + print("Sie können die externe Festplatte jetzt entfernen.") def import_pool(pool): uuid = config.backup_pool[pool] - container_name = 'crypt-' + pool - present_mappers = Command(['ls', '/dev/mapper/']).stdout.decode().strip() + container_name = "crypt-" + pool + present_mappers = Command(["ls", "/dev/mapper/"]).stdout.decode().strip() if container_name in present_mappers: - msg = 'verschlüsselter Container ' + container_name + \ - ' scheint bereits in Verwendung zu sein. Breche ab.' + msg = ( + "verschlüsselter Container " + + container_name + + " scheint bereits in Verwendung zu sein. Breche ab." + ) raise Exception(msg) - print('Öffne verschlüsselten Container\n') - Command([CRYPTSETUP, 'open', '/dev/disk/by-uuid/' + uuid, container_name]) - print('') - print('Importiere Pool ' + pool) + print("Öffne verschlüsselten Container\n") + Command([config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name]) + print("") + print("Importiere Pool " + pool) # -N: no mount # -d: directory to search the pool in - Command([ZPOOL, 'import', '-N', '-d', '/dev/mapper', pool]) + Command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool]) def present_locally(snapshot): @@ -103,38 +117,48 @@ def present_locally(snapshot): def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool): - print('Sichere Dataset ' + dataset) - #print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot) - #return + print("Sichere Dataset " + dataset) + # print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot) + # return if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool): - print(' Dieses Dataset wurde nicht zur Sicherung eingerichtet.') - print(' Verwende ein Kommando in der Art') - print(' # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN') + print(" Dieses Dataset wurde nicht zur Sicherung eingerichtet.") + print(" Verwende ein Kommando in der Art") + print( + " # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN" + ) return if last_local_snapshot == last_remote_snapshot: - print(' Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot') - print(' Führe kein Backup für dieses Dataset durch') + print( + " Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot" + ) + print(" Führe kein Backup für dieses Dataset durch") return - send = Popen([ - ZFS, 'send', '-R', - '-I', last_remote_snapshot, - dataset + '@' + last_local_snapshot], - stdout=PIPE) + send = Popen( + [ + config.ZFS, + "send", + "-R", + "-I", + last_remote_snapshot, + dataset + "@" + last_local_snapshot, + ], + stdout=PIPE, + ) # -d Discard the first element of the sent snapshot's file system name # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. # -u File system that is associated with the received stream is not mounted. - receive = check_output([ZFS, 'receive', '-d', '-F', '-u', backup_pool], stdin=send.stdout) + check_output([config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout) exit_code = send.wait() if exit_code != 0: - raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') + raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.") def snapshot_present_in_dataset(dataset, last_snapshot, pool): snapshots = get_snapshots(pool) # get just the dataset name without the pool - actual_dataset = dataset.split('/')[-1] + actual_dataset = dataset.split("/")[-1] for snapshot in snapshots: - if (actual_dataset + '@' + last_snapshot) in snapshot: + if (actual_dataset + "@" + last_snapshot) in snapshot: return True else: return False @@ -143,12 +167,12 @@ def snapshot_present_in_dataset(dataset, last_snapshot, pool): def clean_old_snapshots(dataset, pool): snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' # get just the dataset name without the pool - actual_dataset = dataset.split('/')[-1] - #print('actual_dataset', actual_dataset) + actual_dataset = dataset.split("/")[-1] + # print('actual_dataset', actual_dataset) for interval in config.keep.keys(): - #print('interval', interval) + # print('interval', interval) # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... - pattern = actual_dataset + '@' + config.snapshot_tag[0] + '.*' + interval + pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval snapshots_in_interval = [] for snapshot in snapshots: result = re.search(pattern, snapshot) @@ -157,47 +181,47 @@ def clean_old_snapshots(dataset, pool): snapshots_in_interval.sort() for i in range(len(snapshots_in_interval) - config.keep[interval]): to_delete = snapshots_in_interval[i] - print(' Lösche veralteten Snapshot {}'.format(to_delete)) + print(" Lösche veralteten Snapshot {}".format(to_delete)) zfs_destroy(to_delete, recursive=True) def zfs_destroy(full_name, recursive=False): if config.local_pool in full_name: - msg = 'Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!' + msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!" raise Exception(msg) else: - cmdline = [ZFS, 'destroy'] + cmdline = [config.ZFS, "destroy"] if recursive: - cmdline.append('-r') + cmdline.append("-r") cmdline.append(full_name) Command(cmdline) def main(): - if os.getuid() > 0: - print('Muss als root ausgeführt werden.') - exit(1) + verify_running_as_root() datasets = get_datasets_to_backup() backup_pool = find_backup_pool() import_pool(backup_pool) try: last_local_snapshot = last_snapshot(config.local_pool) if not last_local_snapshot: - print('Es scheint keine Snapshots im Speicherpool zu geben') - print('Breche ab.') + print("Es scheint keine Snapshots im Speicherpool zu geben") + print("Breche ab.") export_pool(backup_pool) exit() last_remote_snapshot = last_snapshot(backup_pool) if not last_remote_snapshot: - print('Es scheint keine Snapshots auf der Backup-Festplatte zu geben') - print('Breche ab.') + print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben") + print("Breche ab.") export_pool(backup_pool) exit() if not present_locally(last_remote_snapshot): - print(""" + print( + """ Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor - zu langer Zeit das letzte mal als Backup verwendet""") + zu langer Zeit das letzte mal als Backup verwendet""" + ) return for dataset in datasets: do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool) @@ -206,5 +230,11 @@ def main(): export_pool(backup_pool) -if __name__ == '__main__': +def verify_running_as_root(): + if os.getuid() > 0: + print("Muss als root ausgeführt werden.") + exit(1) + + +if __name__ == "__main__": main()