From fef8a3f3ea8c7d242a2fb49dbb7dbbe09292dbb0 Mon Sep 17 00:00:00 2001 From: timeshifter Date: Fri, 3 Dec 2021 20:10:17 +0100 Subject: [PATCH] refactoring --- zfs-backup.py | 181 +++++++++++++++++++++++++++----------------------- 1 file changed, 97 insertions(+), 84 deletions(-) diff --git a/zfs-backup.py b/zfs-backup.py index dad62cf..89f2a83 100755 --- a/zfs-backup.py +++ b/zfs-backup.py @@ -51,6 +51,7 @@ def main(): # TODO --skip-import # TODO --skip-post-backup-scrub # TODO print time estimation while scrub is in progress +# TODO logging class Manager: @@ -125,6 +126,7 @@ class Manager: return common_snapshots[-1] +# TODO this is accessed from both Manager and Dataset. Refactor? def _get_regex_matching_snapshots_with_tags(tags: list[str]): return "@" + ".*".join(tags) @@ -191,27 +193,25 @@ class ZFSPath: raise ValueError(NOT_VALID_ZFS_PATH) -class ZFSBase: - @staticmethod - def _assert_type(instance: Any, object_type: Any, message: str) -> None: - if not isinstance(instance, object_type): - raise TypeError(message) +def _assert_type(instance: Any, object_type: Any, message: str) -> None: + if not isinstance(instance, object_type): + raise TypeError(message) -class ZPOOL(ZFSBase): +class ZPOOL: _ZPOOL = Path("/usr/bin/zpool") @classmethod def import_from_directory(cls, pool: Pool, directory: Path) -> None: - cls._assert_type(pool, Pool, "Can only import Pool objects") + _assert_type(pool, Pool, "Can only import Pool objects") # -N: no mount # -d: directory to search the pool in - Command.run([cls._ZPOOL, "import", "-N", "-d", directory, pool.name]) + CommandRunner.run([cls._ZPOOL, "import", "-N", "-d", directory, pool.name]) @classmethod def export(cls, pool: Pool) -> None: - cls._assert_type(pool, Pool, "Can only export Pool objects") - Command.run([cls._ZPOOL, "export", pool.name]) + _assert_type(pool, Pool, "Can only export Pool objects") + CommandRunner.run([cls._ZPOOL, "export", pool.name]) @classmethod def scrub( @@ -221,10 +221,10 @@ class ZPOOL(ZFSBase): skip_if_recently_scrubbed: bool, wait_for_finish: bool = True, ) -> None: - cls._assert_type(pool, Pool, "Can only scrub pools.") + _assert_type(pool, Pool, "Can only scrub pools.") cls._scrub_if_necessary(pool, skip_if_recently_scrubbed) if wait_for_finish: - Command.run([cls._ZPOOL, "wait", "-t", "scrub", pool.name]) + CommandRunner.run([cls._ZPOOL, "wait", "-t", "scrub", pool.name]) if not cls._healthy(pool): raise IOError(f"Pool {pool.name} is not healthy.") @@ -233,10 +233,10 @@ class ZPOOL(ZFSBase): if cls._scrub_in_progress(pool): return if not skip_if_recently_scrubbed: - Command.run([cls._ZPOOL, "scrub", pool.name]) + CommandRunner.run([cls._ZPOOL, "scrub", pool.name]) else: if not cls._recently_scrubbed(pool): - Command.run([cls._ZPOOL, "scrub", pool.name]) + CommandRunner.run([cls._ZPOOL, "scrub", pool.name]) @classmethod def _recently_scrubbed(cls, pool: Pool) -> bool: @@ -252,88 +252,80 @@ class ZPOOL(ZFSBase): @staticmethod def _get_pool_status_output(pool: Pool) -> str: - return Command.get_output([ZFS, "status", pool.name]) + return CommandRunner.get_output([ZFS, "status", pool.name]) @classmethod def _last_scrub(cls, pool: Pool) -> datetime: return Time.get_last_scrub_from_status_output(cls._get_pool_status_output(pool)) -class ZFS(ZFSBase): +class ZFS: + """Interface of `zfs` commands""" + + class _Subcommands: + class List: + # -H Scripting mode, omit headers + # -d1 maximum of 1 + # -o name output column + # -t object type + # -r recursive + _base = ["list", "-H", "-d1", "-o", "name"] + dataset = _base + ["-t", "filesystem"] + snapshot = _base + ["-t", "snapshot", "-r"] + + destroy = ["destroy"] + # -R Replicate filesystem + # -I send all intermediary snapshots + send = ["send", "-R", "-I"] + # -d Discard the first element of the sent snapshot's file system name + # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. + # -u File system that is associated with the received stream is not mounted. + receive = ["receive", "-d", "-F", "-u"] + _ZFS = Path("/usr/bin/zfs") - # TODO introduce command constants @classmethod def get_first_level_datasets(cls, pool: Pool) -> list[Dataset]: - cls._assert_type(pool, Pool, "Can only get datasets from Pool objects") - datasets_str = Command.get_output( - [ - ZFS, - "list", - "-H", - "-d", - "1", - "-o", - "name", - "-t", - "filesystem", - pool.name, - ] - ) + _assert_type(pool, Pool, "Can only get datasets from Pool objects") + command = [str(cls._ZFS), *cls._Subcommands.List.dataset, pool.name] + datasets_str = CommandRunner.get_output(command) # we omit the first return value, which is the pool, not the dataset datasets_list = datasets_str.split("\n")[1:] return [Dataset(ZFSPath.from_string(d)) for d in datasets_list] @classmethod def get_snapshots(cls, dataset: Dataset) -> list[Snapshot]: - cls._assert_type( - dataset, Dataset, "Can only get snapshots from Dataset objects" - ) - return [ - Snapshot(ZFSPath.from_string(name)) - for name in Command.get_output( - [ - ZFS, - "list", - "-r", - "-t", - "snapshot", - "-o", - "name", - "-H", - "-d", - "1", - dataset.qualified_name, - ] - ).split() + _assert_type(dataset, Dataset, "Can only get snapshots from Dataset object") + command = [ + str(cls._ZFS), + *cls._Subcommands.List.snapshot, + dataset.qualified_name, ] + snapshot_names: list[str] = CommandRunner.get_output(command).split() + return [Snapshot(ZFSPath.from_string(name)) for name in snapshot_names] @classmethod - def destroy_snapshot(cls, snapshot: Snapshot, *, recursive=False): - cls._assert_type(snapshot, Snapshot, "Can only destroy snapshots.") + def destroy_snapshot(cls, snapshot: Snapshot): + _assert_type(snapshot, Snapshot, "Can only destroy snapshots.") if config.local_pool_to_backup == snapshot.pool: raise Exception(DELETE_IN_LOCAL_POOL) - cmdline = [ZFS, "destroy"] - if recursive: - cmdline.append("-r") - cmdline.append(str(snapshot)) - Command.run(cmdline) + command = [str(cls._ZFS), *cls._Subcommands.destroy, str(snapshot)] + CommandRunner.run(command) @classmethod def send_incremental( cls, old_snapshot: Snapshot, new_snapshot: Snapshot ) -> Popen[str]: - cls._send_sanity_checks(old_snapshot, new_snapshot) + cls._pre_send_sanity_checks(old_snapshot, new_snapshot) print(f" {old_snapshot} -> {new_snapshot}") - stream = Popen( - [ZFS, "send", "-R", "-I", str(old_snapshot), str(new_snapshot)], stdout=PIPE + return CommandRunner.send_stream( + [cls._ZFS, *cls._Subcommands.send, str(old_snapshot), str(new_snapshot)] ) - return stream @classmethod - def _send_sanity_checks(cls, old_snapshot: Snapshot, new_snapshot: Snapshot): + def _pre_send_sanity_checks(cls, old_snapshot: Snapshot, new_snapshot: Snapshot): for snapshot in [old_snapshot, new_snapshot]: - cls._assert_type( + _assert_type( snapshot, Snapshot, f"Cannot send stream, object is not a Snapshot: {snapshot}", @@ -347,14 +339,9 @@ class ZFS(ZFSBase): @classmethod def receive(cls, stream: Popen[str], pool: Pool): - # -d Discard the first element of the sent snapshot's file system name - # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. - # -u File system that is associated with the received stream is not mounted. - cls._assert_type(pool, Pool, "Can only receive into pools.") - Command.receive_from_stream( - [ZFS, "receive", "-d", "-F", "-u", pool.name], - stream, - ) + _assert_type(pool, Pool, "Can only receive into pools.") + command = [str(cls._ZFS), *cls._Subcommands.receive, pool.name] + CommandRunner.receive_from_stream(command, stream) class Time: @@ -421,15 +408,19 @@ class Disk: class Cryptsetup: + class _Subcommands: + close = "close" + status = "status" + open = "open" _CRYPTSETUP = Path("/usr/bin/cryptsetup") @classmethod def encrypt(cls, mapper_entry: str) -> None: - Command.run([cls._CRYPTSETUP, "close", mapper_entry]) + CommandRunner.run([cls._CRYPTSETUP, "close", mapper_entry]) @classmethod def _status(cls, name: str) -> str: - return Command.get_output([cls._CRYPTSETUP, "status", name]) + return CommandRunner.get_output([cls._CRYPTSETUP, "status", name]) @classmethod def decrypt(cls, path: Path, mapper_entry: str) -> bool: @@ -438,7 +429,7 @@ class Cryptsetup: return True print(MAPPER_ENTRY_ALREADY_EXISTS) exit(EXIT_ERROR) - Command.run([cls._CRYPTSETUP, "open", str(path), mapper_entry]) + CommandRunner.run([cls._CRYPTSETUP, "open", str(path), mapper_entry]) return False @classmethod @@ -610,21 +601,43 @@ class Snapshot: return Dataset(self._path.dataset) -class Command: +class CommandRunner: @staticmethod - def run(cmdline: list[str]) -> None: + def _sanity_check(cmdline: list[str]) -> None: + _assert_type(cmdline, list, "Command line must be a list") + for item in cmdline: + _assert_type(item, str, "Only strings allowed in command line") + + @staticmethod + def _to_list(command: str | list[str]) -> list[str]: + if isinstance(command, str): + return [command] + return command + + @classmethod + def run(cls, cmdline: str | list[str]) -> None: + cmdline = cls._to_list(cmdline) + cls._sanity_check(cmdline) call(cmdline) - @staticmethod - def get_output(cmdline: str | list[str]) -> str: - if isinstance(cmdline, str): - cmdline = [cmdline] + @classmethod + def get_output(cls, cmdline: str | list[str]) -> str: + cmdline = cls._to_list(cmdline) + cls._sanity_check(cmdline) return check_output(cmdline).strip().decode() - @staticmethod - def receive_from_stream(cmdline: str | list[str], stream: Popen) -> None: + @classmethod + def receive_from_stream(cls, cmdline: str | list[str], stream: Popen) -> None: + cmdline = cls._to_list(cmdline) + cls._sanity_check(cmdline) check_output(cmdline, stdin=stream.stdout) + @classmethod + def send_stream(cls, cmdline: str | list[str]) -> Popen: + cmdline = cls._to_list(cmdline) + cls._sanity_check(cmdline) + return Popen(cmdline, stdout=PIPE) + def verify_running_as_root(): if getuid() > 0: