diff --git a/config.py b/config.py index 9a07c5b..e01cfdb 100644 --- a/config.py +++ b/config.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from uuid import UUID # Name des lokalen Pools local_pool_to_backup = "tank" # Name der Backup-Platten und UUID der Partition -backup_pool = { - "backup1": "603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4", - "backup2": "8632e9e4-3d6c-4627-8c28-ec72f52db281", +backup_pools = { + "backup1": UUID("603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4"), + "backup2": UUID("8632e9e4-3d6c-4627-8c28-ec72f52db281"), } # Datasets, die nicht gesichert werden sollen diff --git a/zfs-backup.py b/zfs-backup.py index 8548c4c..28e019f 100755 --- a/zfs-backup.py +++ b/zfs-backup.py @@ -7,11 +7,16 @@ import os import re import subprocess from functools import cache +from pathlib import Path from subprocess import PIPE, Popen, check_output from sys import exit +from uuid import UUID import config +DISK_BY_UUID = Path("/dev/disk/by-uuid") +MAPPER_PATH = Path("/dev/mapper") + """ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit # zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1 @@ -21,244 +26,303 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit def main(): verify_running_as_root() local_pool = Pool(config.local_pool_to_backup) - datasets_to_backup = local_pool.get_datasets_not_in_list(config.do_not_backup) - backup_pool = Pool.find_from_dict(config.backup_pool) - import_pool(backup_pool) + external_pool = ExternalPool.find_from_dict(config.backup_pools) + external_pool.import_() try: - send_datasets_to_backup_pool(backup_pool, datasets_to_backup) + local_pool.backup_to(external_pool) + clean_old_snapshots(external_pool) finally: - export_pool(backup_pool) + external_pool.export() + + +class ZFS: + @staticmethod + def get_datasets(name: str) -> list[Dataset]: + datasets_str = get_output_of_command( + [ + config.ZFS, + "list", + "-H", + "-d", + "1", + "-o", + "name", + "-t", + "filesystem", + name, + ] + ) + # we omit the first return value, which is the pool, not the dataset + datasets_list = datasets_str.split("\n")[1:] + return [Dataset(d) for d in datasets_list] + + @staticmethod + def import_pool_from_directory(name, directory) -> None: + # -N: no mount + # -d: directory to search the pool in + run_command([config.ZPOOL, "import", "-N", "-d", directory, name]) + + @staticmethod + def export_pool(name) -> None: + run_command([config.ZPOOL, "export", name]) + + @staticmethod + def get_snapshots(name) -> list[Snapshot]: + return [ + Snapshot(name) + for name in get_output_of_command( + [ + config.ZFS, + "list", + "-r", + "-t", + "snapshot", + "-o", + "name", + "-H", + "-d", + "1", + name, + ] + ).split() + ] + + @staticmethod + def destroy(full_name, *, recursive=False): + if config.local_pool_to_backup in full_name: + msg = "ALERT! Tried to delete in local pool!" + raise Exception(msg) + cmdline = [config.ZFS, "destroy"] + if recursive: + cmdline.append("-r") + cmdline.append(full_name) + run_command(cmdline) + + @staticmethod + def send(old_snap: Snapshot, new_snap: Snapshot) -> Popen[str]: + sender = Popen( + [config.ZFS, "send", "-R", "-I", old_snap, new_snap], stdout=PIPE + ) + return sender + + @staticmethod + def receive(sender: Popen[str], pool_name): + # -d Discard the first element of the sent snapshot's file system name + # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. + # -u File system that is associated with the received stream is not mounted. + check_output( + [config.ZFS, "receive", "-d", "-F", "-u", pool_name], + stdin=sender.stdout, + ) + + +class Disk: + def __init__(self, uuid: UUID, name: str): + self.uuid = uuid + self._decrypted = False + self._name = name + + def decrypt(self) -> None: + self._verify_mapper_entry_not_in_use() + self._decrypt_with_cryptsetup() + self._decrypted = True + + def _decrypt_with_cryptsetup(self): + run_command([config.CRYPTSETUP, "open", self._path, self._mapper_entry]) + + def _verify_mapper_entry_not_in_use(self): + for item in MAPPER_PATH.iterdir(): + if self._mapper_entry == item.name: + raise IOError(f"mapper entry {self._mapper_entry} already exists") + + @property + def _path(self) -> Path: + return DISK_BY_UUID / str(self.uuid) + + @property + def _mapper_entry(self) -> str: + return f"crypt-{self._name}" + + def encrypt(self): + get_output_of_command([config.CRYPTSETUP, "close", self._name]) + self._decrypted = False + + +def get_last_common_snapshot(ds1: Dataset, ds2: Dataset) -> Snapshot: + common_snapshots = [ + local_snap + for local_snap in ds1.snapshots_matching_backup_tags + for remote_snap in ds2.snapshots_matching_backup_tags + if local_snap.snapname == remote_snap.snapname + ] + common_snapshots.sort() + return common_snapshots[-1] class Pool: def __init__(self, name): self.name = name - @cache @property - def datasets(self) -> list[Dataset]: - datasets_str = get_output_of_command( - [config.ZFS, "list", "-H", "-d", "1", "-o", "name", self.name] - ) - # we omit the first return value, which is the pool, not the dataset - datasets_list = datasets_str.split("\n")[1:] - to_backup = [Dataset(d) for d in datasets_list] - return to_backup - @cache - def get_datasets_not_in_list(self, do_not_backup) -> list[Dataset]: - return [d for d in self.datasets if d.name not in do_not_backup] + def datasets(self) -> list[Dataset]: + return ZFS.get_datasets(self.name) + + @property + @cache + def datasets_to_backup(self) -> list[Dataset]: + return [ + d for d in self.datasets if d.qualified_name not in config.do_not_backup + ] + + def backup_to(self, external_pool: ExternalPool): + for local_dataset in self.datasets_to_backup: + remote_dataset = external_pool.search_dataset( + local_dataset.name_without_pool + ) + last_common_snapshot = get_last_common_snapshot( + remote_dataset, local_dataset + ) + last_local_snapshot: Snapshot = ( + local_dataset.snapshots_matching_backup_tags[-1] + ) + sender = ZFS.send(last_common_snapshot, last_local_snapshot) + ZFS.receive(sender, external_pool.name) + + +class ExternalPool(Pool): + def __init__(self, name: str, disk: Disk): + super().__init__(name) + self._imported: bool = False + self._disk = disk + + def import_(self) -> None: + self._disk.decrypt() + self._import_from_directory(MAPPER_PATH) + + def export(self) -> None: + self._export_pool() + self._disk.encrypt() + + def _import_from_directory(self, directory: Path) -> None: + ZFS.import_pool_from_directory(self.name, directory) + + def _export_pool(self) -> None: + ZFS.export_pool(self.name) @classmethod - def find_from_dict(cls, backup_pools: dict[str, str]) -> Pool: - print("Suche nach externer Festplatte für Backups") - present_uuids = get_output_of_command(["ls", "/dev/disk/by-uuid"]) - for pool, uuid in backup_pools.items(): - if uuid in present_uuids: - print("Gefundene Platte: " + pool) - return cls(pool) + def find_from_dict(cls, backup_pools: dict[str, UUID]) -> ExternalPool: + present_uuids = list(DISK_BY_UUID.iterdir()) + for pool_name, disk_uuid in backup_pools.items(): + if str(disk_uuid) in present_uuids: + disk = Disk(disk_uuid, pool_name) + return cls(pool_name, disk) else: - raise IOError("Konnte keine Backup-Platte finden!") + print("Could not find a backup drive") + exit(1) + + def search_dataset(self, name_without_pool: str) -> Dataset: + qualified_name = self.name + "/" + name_without_pool + for dataset in self.datasets_to_backup: + if qualified_name == dataset.qualified_name: + return dataset class Dataset: - def __init__(self, name: str): - self.name = name + def __init__(self, qualified_name: str): + self.qualified_name = qualified_name def __repr__(self): - return self.name + return self.qualified_name + + @property + def name_without_pool(self) -> str: + return self.qualified_name.split("/", maxsplit=1)[1] + + @property + @cache + def snapshots(self) -> list[Snapshot]: + """get qualified snapshot names that are direct children of the dataset""" + return ZFS.get_snapshots(self.qualified_name) + + @property + @cache + def snapshots_matching_backup_tags(self) -> list[Snapshot]: + snapshots = [ + snapshot for snapshot in self.snapshots if snapshot.matches_backup_tags + ] + snapshots.sort() + return snapshots + + @property + def last_snapshot(self) -> Snapshot: + for snapshot in self.snapshots[::-1]: + if snapshot.matches_backup_tags: + return snapshot -def get_snapshots(pool): - return get_output_of_command( - [config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool] - ).split() +class Snapshot: + def __init__(self, qualified_name: str): + self.qualified_name = qualified_name + def __lt__(self, other: Snapshot) -> bool: + return self.snapname < other.snapname -def last_snapshot(pool): - snapshots = get_snapshots(pool) - # check if the snapshot_tags appear in the snapshot name in order they are provided - relevant = [ - s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s) - ] - # sort chronologically - relevant.sort() - # the most recent snapshot is the last one - try: - last_entry = relevant[-1] - # didn't find any snapshots at all - except IndexError: - return - last_snapshot = last_entry.strip().split("@")[-1] - return last_snapshot + @property + def snapname(self) -> str: + return self.qualified_name.split("@")[1] - -def export_pool(pool): - container_name = "crypt-" + pool - print("Exportiere Pool " + pool) - get_output_of_command([config.ZPOOL, "export", pool]) - print("Schließe verschlüsselten Container") - get_output_of_command([config.CRYPTSETUP, "close", container_name]) - print("Fertig!") - print("Sie können die externe Festplatte jetzt entfernen.") - - -def import_pool(pool): - uuid = config.backup_pool[pool] - container_name = "crypt-" + pool - present_mappers = get_output_of_command(["ls", "/dev/mapper/"]) - if container_name in present_mappers: - msg = ( - "verschlüsselter Container " - + container_name - + " scheint bereits in Verwendung zu sein. Breche ab." + @property + def matches_backup_tags(self) -> bool: + return bool( + re.search("@" + ".*".join(config.snapshot_tag), self.qualified_name) ) - raise Exception(msg) - print("Öffne verschlüsselten Container\n") - get_output_of_command( - [config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name] - ) - print("") - print("Importiere Pool " + pool) - # -N: no mount - # -d: directory to search the pool in - get_output_of_command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool]) + + @property + def name_without_pool(self) -> str: + return self.qualified_name.split("/", maxsplit=1)[1] -def present_locally(snapshot): - local_snapshots = get_snapshots(config.local_pool_to_backup) - for each_snapshot in local_snapshots: - if snapshot in each_snapshot: - return True - else: - return False +# def clean_old_snapshots(dataset, pool): +# snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' +# # get just the dataset name without the pool +# actual_dataset = dataset.split("/")[-1] +# # print('actual_dataset', actual_dataset) +# for interval in config.keep.keys(): +# # print('interval', interval) +# # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... +# pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval +# snapshots_in_interval = [] +# for snapshot in snapshots: +# result = re.search(pattern, snapshot) +# if result is not None: +# snapshots_in_interval.append(snapshot) +# snapshots_in_interval.sort() +# for i in range(len(snapshots_in_interval) - config.keep[interval]): +# to_delete = snapshots_in_interval[i] +# print(" Lösche veralteten Snapshot {}".format(to_delete)) +# zfs_destroy(to_delete, recursive=True) +# -def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool): - print("Sichere Dataset " + dataset) - # print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot) - # return - if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool): - print(" Dieses Dataset wurde nicht zur Sicherung eingerichtet.") - print(" Verwende ein Kommando in der Art") - print( - " # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN" - ) - return - if last_local_snapshot == last_remote_snapshot: - print( - " Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot" - ) - print(" Führe kein Backup für dieses Dataset durch") - return - send = Popen( - [ - config.ZFS, - "send", - "-R", - "-I", - last_remote_snapshot, - dataset + "@" + last_local_snapshot, - ], - stdout=PIPE, - ) - # -d Discard the first element of the sent snapshot's file system name - # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. - # -u File system that is associated with the received stream is not mounted. - check_output( - [config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout - ) - exit_code = send.wait() - if exit_code != 0: - raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.") +def run_command(cmdline: list[str]): + subprocess.call(cmdline) -def snapshot_present_in_dataset(dataset, last_snapshot, pool): - snapshots = get_snapshots(pool) - # get just the dataset name without the pool - actual_dataset = dataset.split("/")[-1] - for snapshot in snapshots: - if (actual_dataset + "@" + last_snapshot) in snapshot: - return True - else: - return False - - -def clean_old_snapshots(dataset, pool): - snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' - # get just the dataset name without the pool - actual_dataset = dataset.split("/")[-1] - # print('actual_dataset', actual_dataset) - for interval in config.keep.keys(): - # print('interval', interval) - # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... - pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval - snapshots_in_interval = [] - for snapshot in snapshots: - result = re.search(pattern, snapshot) - if result is not None: - snapshots_in_interval.append(snapshot) - snapshots_in_interval.sort() - for i in range(len(snapshots_in_interval) - config.keep[interval]): - to_delete = snapshots_in_interval[i] - print(" Lösche veralteten Snapshot {}".format(to_delete)) - zfs_destroy(to_delete, recursive=True) - - -def get_output_of_command(cmdline: str | list[str]): +def get_output_of_command(cmdline: str | list[str]) -> str: if isinstance(cmdline, str): cmdline = [cmdline] return subprocess.check_output(cmdline).strip().decode() -def zfs_destroy(full_name, recursive=False): - if config.local_pool_to_backup in full_name: - msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!" - raise Exception(msg) - else: - cmdline = [config.ZFS, "destroy"] - if recursive: - cmdline.append("-r") - cmdline.append(full_name) - get_output_of_command(cmdline) - - -def send_datasets_to_backup_pool(backup_pool, datasets): - last_local_snapshot = last_snapshot(config.local_pool_to_backup) - if not last_local_snapshot: - print("Es scheint keine Snapshots im Speicherpool zu geben") - print("Breche ab.") - export_pool(backup_pool) - exit() - last_remote_snapshot = last_snapshot(backup_pool) - if not last_remote_snapshot: - print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben") - print("Breche ab.") - export_pool(backup_pool) - exit() - if not present_locally(last_remote_snapshot): - print( - """ -Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server -keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor -zu langer Zeit das letzte mal als Backup verwendet""" - ) - return - for dataset in datasets: - do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool) - clean_old_snapshots(dataset, backup_pool) - - def verify_running_as_root(): if os.getuid() > 0: - print("Muss als root ausgeführt werden.") + print("Run as root.") exit(1) -def show_actions_to_execute(datasets: list[Dataset]): - print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:") - for ds in datasets: - print(" " + str(ds)) +def clean_old_snapshots(external_pool: ExternalPool) -> None: # TODO + ... if __name__ == "__main__":