major rewrite

config.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
+from uuid import UUID
 
 # Name des lokalen Pools
 local_pool_to_backup = "tank"
 
 # Name der Backup-Platten und UUID der Partition
-backup_pool = {
+backup_pools = {
-    "backup1": "603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4",
+    "backup1": UUID("603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4"),
-    "backup2": "8632e9e4-3d6c-4627-8c28-ec72f52db281",
+    "backup2": UUID("8632e9e4-3d6c-4627-8c28-ec72f52db281"),
 }
 
 # Datasets, die nicht gesichert werden sollen

zfs-backup.py

@@ -7,11 +7,16 @@ import os
 import re
 import subprocess
 from functools import cache
+from pathlib import Path
 from subprocess import PIPE, Popen, check_output
 from sys import exit
+from uuid import UUID
 
 import config
 
+DISK_BY_UUID = Path("/dev/disk/by-uuid")
+MAPPER_PATH = Path("/dev/mapper")
+
 """
 Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
 # zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
@@ -21,244 +26,303 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
 def main():
     verify_running_as_root()
     local_pool = Pool(config.local_pool_to_backup)
-    datasets_to_backup = local_pool.get_datasets_not_in_list(config.do_not_backup)
+    external_pool = ExternalPool.find_from_dict(config.backup_pools)
-    backup_pool = Pool.find_from_dict(config.backup_pool)
+    external_pool.import_()
-    import_pool(backup_pool)
     try:
-        send_datasets_to_backup_pool(backup_pool, datasets_to_backup)
+        local_pool.backup_to(external_pool)
+        clean_old_snapshots(external_pool)
     finally:
-        export_pool(backup_pool)
+        external_pool.export()
+
+
+class ZFS:
+    @staticmethod
+    def get_datasets(name: str) -> list[Dataset]:
+        datasets_str = get_output_of_command(
+            [
+                config.ZFS,
+                "list",
+                "-H",
+                "-d",
+                "1",
+                "-o",
+                "name",
+                "-t",
+                "filesystem",
+                name,
+            ]
+        )
+        # we omit the first return value, which is the pool, not the dataset
+        datasets_list = datasets_str.split("\n")[1:]
+        return [Dataset(d) for d in datasets_list]
+
+    @staticmethod
+    def import_pool_from_directory(name, directory) -> None:
+        # -N: no mount
+        # -d: directory to search the pool in
+        run_command([config.ZPOOL, "import", "-N", "-d", directory, name])
+
+    @staticmethod
+    def export_pool(name) -> None:
+        run_command([config.ZPOOL, "export", name])
+
+    @staticmethod
+    def get_snapshots(name) -> list[Snapshot]:
+        return [
+            Snapshot(name)
+            for name in get_output_of_command(
+                [
+                    config.ZFS,
+                    "list",
+                    "-r",
+                    "-t",
+                    "snapshot",
+                    "-o",
+                    "name",
+                    "-H",
+                    "-d",
+                    "1",
+                    name,
+                ]
+            ).split()
+        ]
+
+    @staticmethod
+    def destroy(full_name, *, recursive=False):
+        if config.local_pool_to_backup in full_name:
+            msg = "ALERT! Tried to delete in local pool!"
+            raise Exception(msg)
+        cmdline = [config.ZFS, "destroy"]
+        if recursive:
+            cmdline.append("-r")
+        cmdline.append(full_name)
+        run_command(cmdline)
+
+    @staticmethod
+    def send(old_snap: Snapshot, new_snap: Snapshot) -> Popen[str]:
+        sender = Popen(
+            [config.ZFS, "send", "-R", "-I", old_snap, new_snap], stdout=PIPE
+        )
+        return sender
+
+    @staticmethod
+    def receive(sender: Popen[str], pool_name):
+        # -d  Discard the first element of the sent snapshot's file system name
+        # -F  Force a rollback of the file system to the most recent snapshot before performing the receive operation.
+        # -u  File system that is associated with the received stream is not mounted.
+        check_output(
+            [config.ZFS, "receive", "-d", "-F", "-u", pool_name],
+            stdin=sender.stdout,
+        )
+
+
+class Disk:
+    def __init__(self, uuid: UUID, name: str):
+        self.uuid = uuid
+        self._decrypted = False
+        self._name = name
+
+    def decrypt(self) -> None:
+        self._verify_mapper_entry_not_in_use()
+        self._decrypt_with_cryptsetup()
+        self._decrypted = True
+
+    def _decrypt_with_cryptsetup(self):
+        run_command([config.CRYPTSETUP, "open", self._path, self._mapper_entry])
+
+    def _verify_mapper_entry_not_in_use(self):
+        for item in MAPPER_PATH.iterdir():
+            if self._mapper_entry == item.name:
+                raise IOError(f"mapper entry {self._mapper_entry} already exists")
+
+    @property
+    def _path(self) -> Path:
+        return DISK_BY_UUID / str(self.uuid)
+
+    @property
+    def _mapper_entry(self) -> str:
+        return f"crypt-{self._name}"
+
+    def encrypt(self):
+        get_output_of_command([config.CRYPTSETUP, "close", self._name])
+        self._decrypted = False
+
+
+def get_last_common_snapshot(ds1: Dataset, ds2: Dataset) -> Snapshot:
+    common_snapshots = [
+        local_snap
+        for local_snap in ds1.snapshots_matching_backup_tags
+        for remote_snap in ds2.snapshots_matching_backup_tags
+        if local_snap.snapname == remote_snap.snapname
+    ]
+    common_snapshots.sort()
+    return common_snapshots[-1]
 
 
 class Pool:
     def __init__(self, name):
         self.name = name
 
-    @cache
     @property
-    def datasets(self) -> list[Dataset]:
-        datasets_str = get_output_of_command(
-            [config.ZFS, "list", "-H", "-d", "1", "-o", "name", self.name]
-        )
-        # we omit the first return value, which is the pool, not the dataset
-        datasets_list = datasets_str.split("\n")[1:]
-        to_backup = [Dataset(d) for d in datasets_list]
-        return to_backup
-
     @cache
-    def get_datasets_not_in_list(self, do_not_backup) -> list[Dataset]:
+    def datasets(self) -> list[Dataset]:
-        return [d for d in self.datasets if d.name not in do_not_backup]
+        return ZFS.get_datasets(self.name)
+
+    @property
+    @cache
+    def datasets_to_backup(self) -> list[Dataset]:
+        return [
+            d for d in self.datasets if d.qualified_name not in config.do_not_backup
+        ]
+
+    def backup_to(self, external_pool: ExternalPool):
+        for local_dataset in self.datasets_to_backup:
+            remote_dataset = external_pool.search_dataset(
+                local_dataset.name_without_pool
+            )
+            last_common_snapshot = get_last_common_snapshot(
+                remote_dataset, local_dataset
+            )
+            last_local_snapshot: Snapshot = (
+                local_dataset.snapshots_matching_backup_tags[-1]
+            )
+            sender = ZFS.send(last_common_snapshot, last_local_snapshot)
+            ZFS.receive(sender, external_pool.name)
+
+
+class ExternalPool(Pool):
+    def __init__(self, name: str, disk: Disk):
+        super().__init__(name)
+        self._imported: bool = False
+        self._disk = disk
+
+    def import_(self) -> None:
+        self._disk.decrypt()
+        self._import_from_directory(MAPPER_PATH)
+
+    def export(self) -> None:
+        self._export_pool()
+        self._disk.encrypt()
+
+    def _import_from_directory(self, directory: Path) -> None:
+        ZFS.import_pool_from_directory(self.name, directory)
+
+    def _export_pool(self) -> None:
+        ZFS.export_pool(self.name)
 
     @classmethod
-    def find_from_dict(cls, backup_pools: dict[str, str]) -> Pool:
+    def find_from_dict(cls, backup_pools: dict[str, UUID]) -> ExternalPool:
-        print("Suche nach externer Festplatte für Backups")
+        present_uuids = list(DISK_BY_UUID.iterdir())
-        present_uuids = get_output_of_command(["ls", "/dev/disk/by-uuid"])
+        for pool_name, disk_uuid in backup_pools.items():
-        for pool, uuid in backup_pools.items():
+            if str(disk_uuid) in present_uuids:
-            if uuid in present_uuids:
+                disk = Disk(disk_uuid, pool_name)
-                print("Gefundene Platte: " + pool)
+                return cls(pool_name, disk)
-                return cls(pool)
         else:
-            raise IOError("Konnte keine Backup-Platte finden!")
+            print("Could not find a backup drive")
+            exit(1)
+
+    def search_dataset(self, name_without_pool: str) -> Dataset:
+        qualified_name = self.name + "/" + name_without_pool
+        for dataset in self.datasets_to_backup:
+            if qualified_name == dataset.qualified_name:
+                return dataset
 
 
 class Dataset:
-    def __init__(self, name: str):
+    def __init__(self, qualified_name: str):
-        self.name = name
+        self.qualified_name = qualified_name
 
     def __repr__(self):
-        return self.name
+        return self.qualified_name
+
+    @property
+    def name_without_pool(self) -> str:
+        return self.qualified_name.split("/", maxsplit=1)[1]
+
+    @property
+    @cache
+    def snapshots(self) -> list[Snapshot]:
+        """get qualified snapshot names that are direct children of the dataset"""
+        return ZFS.get_snapshots(self.qualified_name)
+
+    @property
+    @cache
+    def snapshots_matching_backup_tags(self) -> list[Snapshot]:
+        snapshots = [
+            snapshot for snapshot in self.snapshots if snapshot.matches_backup_tags
+        ]
+        snapshots.sort()
+        return snapshots
+
+    @property
+    def last_snapshot(self) -> Snapshot:
+        for snapshot in self.snapshots[::-1]:
+            if snapshot.matches_backup_tags:
+                return snapshot
 
 
-def get_snapshots(pool):
+class Snapshot:
-    return get_output_of_command(
+    def __init__(self, qualified_name: str):
-        [config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool]
+        self.qualified_name = qualified_name
-    ).split()
 
+    def __lt__(self, other: Snapshot) -> bool:
+        return self.snapname < other.snapname
 
-def last_snapshot(pool):
+    @property
-    snapshots = get_snapshots(pool)
+    def snapname(self) -> str:
-    # check if the snapshot_tags appear in the snapshot name in order they are provided
+        return self.qualified_name.split("@")[1]
-    relevant = [
-        s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s)
-    ]
-    # sort chronologically
-    relevant.sort()
-    # the most recent snapshot is the last one
-    try:
-        last_entry = relevant[-1]
-    # didn't find any snapshots at all
-    except IndexError:
-        return
-    last_snapshot = last_entry.strip().split("@")[-1]
-    return last_snapshot
 
-
+    @property
-def export_pool(pool):
+    def matches_backup_tags(self) -> bool:
-    container_name = "crypt-" + pool
+        return bool(
-    print("Exportiere Pool " + pool)
+            re.search("@" + ".*".join(config.snapshot_tag), self.qualified_name)
-    get_output_of_command([config.ZPOOL, "export", pool])
-    print("Schließe verschlüsselten Container")
-    get_output_of_command([config.CRYPTSETUP, "close", container_name])
-    print("Fertig!")
-    print("Sie können die externe Festplatte jetzt entfernen.")
-
-
-def import_pool(pool):
-    uuid = config.backup_pool[pool]
-    container_name = "crypt-" + pool
-    present_mappers = get_output_of_command(["ls", "/dev/mapper/"])
-    if container_name in present_mappers:
-        msg = (
-            "verschlüsselter Container "
-            + container_name
-            + " scheint bereits in Verwendung zu sein. Breche ab."
         )
-        raise Exception(msg)
+
-    print("Öffne verschlüsselten Container\n")
+    @property
-    get_output_of_command(
+    def name_without_pool(self) -> str:
-        [config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name]
+        return self.qualified_name.split("/", maxsplit=1)[1]
-    )
-    print("")
-    print("Importiere Pool " + pool)
-    # -N: no mount
-    # -d: directory to search the pool in
-    get_output_of_command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool])
 
 
-def present_locally(snapshot):
+# def clean_old_snapshots(dataset, pool):
-    local_snapshots = get_snapshots(config.local_pool_to_backup)
+#    snapshots = get_snapshots(pool)  # full name like 'pool/dataset@snapshot'
-    for each_snapshot in local_snapshots:
+#    # get just the dataset name without the pool
-        if snapshot in each_snapshot:
+#    actual_dataset = dataset.split("/")[-1]
-            return True
+#    # print('actual_dataset', actual_dataset)
-    else:
+#    for interval in config.keep.keys():
-        return False
+#        # print('interval', interval)
+#        # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
+#        pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
+#        snapshots_in_interval = []
+#        for snapshot in snapshots:
+#            result = re.search(pattern, snapshot)
+#            if result is not None:
+#                snapshots_in_interval.append(snapshot)
+#        snapshots_in_interval.sort()
+#        for i in range(len(snapshots_in_interval) - config.keep[interval]):
+#            to_delete = snapshots_in_interval[i]
+#            print("  Lösche veralteten Snapshot {}".format(to_delete))
+#            zfs_destroy(to_delete, recursive=True)
+#
 
 
-def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
+def run_command(cmdline: list[str]):
-    print("Sichere Dataset " + dataset)
+    subprocess.call(cmdline)
-    # print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
-    # return
-    if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
-        print("  Dieses Dataset wurde nicht zur Sicherung eingerichtet.")
-        print("  Verwende ein Kommando in der Art")
-        print(
-            "  # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN"
-        )
-        return
-    if last_local_snapshot == last_remote_snapshot:
-        print(
-            "  Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot"
-        )
-        print("  Führe kein Backup für dieses Dataset durch")
-        return
-    send = Popen(
-        [
-            config.ZFS,
-            "send",
-            "-R",
-            "-I",
-            last_remote_snapshot,
-            dataset + "@" + last_local_snapshot,
-        ],
-        stdout=PIPE,
-    )
-    # -d  Discard the first element of the sent snapshot's file system name
-    # -F  Force a rollback of the file system to the most recent snapshot before performing the receive operation.
-    # -u  File system that is associated with the received stream is not mounted.
-    check_output(
-        [config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout
-    )
-    exit_code = send.wait()
-    if exit_code != 0:
-        raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")
 
 
-def snapshot_present_in_dataset(dataset, last_snapshot, pool):
+def get_output_of_command(cmdline: str | list[str]) -> str:
-    snapshots = get_snapshots(pool)
-    # get just the dataset name without the pool
-    actual_dataset = dataset.split("/")[-1]
-    for snapshot in snapshots:
-        if (actual_dataset + "@" + last_snapshot) in snapshot:
-            return True
-    else:
-        return False
-
-
-def clean_old_snapshots(dataset, pool):
-    snapshots = get_snapshots(pool)  # full name like 'pool/dataset@snapshot'
-    # get just the dataset name without the pool
-    actual_dataset = dataset.split("/")[-1]
-    # print('actual_dataset', actual_dataset)
-    for interval in config.keep.keys():
-        # print('interval', interval)
-        # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
-        pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
-        snapshots_in_interval = []
-        for snapshot in snapshots:
-            result = re.search(pattern, snapshot)
-            if result is not None:
-                snapshots_in_interval.append(snapshot)
-        snapshots_in_interval.sort()
-        for i in range(len(snapshots_in_interval) - config.keep[interval]):
-            to_delete = snapshots_in_interval[i]
-            print("  Lösche veralteten Snapshot {}".format(to_delete))
-            zfs_destroy(to_delete, recursive=True)
-
-
-def get_output_of_command(cmdline: str | list[str]):
     if isinstance(cmdline, str):
         cmdline = [cmdline]
     return subprocess.check_output(cmdline).strip().decode()
 
 
-def zfs_destroy(full_name, recursive=False):
-    if config.local_pool_to_backup in full_name:
-        msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!"
-        raise Exception(msg)
-    else:
-        cmdline = [config.ZFS, "destroy"]
-        if recursive:
-            cmdline.append("-r")
-        cmdline.append(full_name)
-        get_output_of_command(cmdline)
-
-
-def send_datasets_to_backup_pool(backup_pool, datasets):
-    last_local_snapshot = last_snapshot(config.local_pool_to_backup)
-    if not last_local_snapshot:
-        print("Es scheint keine Snapshots im Speicherpool zu geben")
-        print("Breche ab.")
-        export_pool(backup_pool)
-        exit()
-    last_remote_snapshot = last_snapshot(backup_pool)
-    if not last_remote_snapshot:
-        print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben")
-        print("Breche ab.")
-        export_pool(backup_pool)
-        exit()
-    if not present_locally(last_remote_snapshot):
-        print(
-            """
-Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
-keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
-zu langer Zeit das letzte mal als Backup verwendet"""
-        )
-        return
-    for dataset in datasets:
-        do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
-        clean_old_snapshots(dataset, backup_pool)
-
-
 def verify_running_as_root():
     if os.getuid() > 0:
-        print("Muss als root ausgeführt werden.")
+        print("Run as root.")
         exit(1)
 
 
-def show_actions_to_execute(datasets: list[Dataset]):
+def clean_old_snapshots(external_pool: ExternalPool) -> None:  # TODO
-    print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:")
+    ...
-    for ds in datasets:
-        print("  " + str(ds))
 
 
 if __name__ == "__main__":