#!/usr/bin/python # -*- coding: utf-8 -*- from __future__ import annotations import os import re import subprocess from functools import cache from pathlib import Path from subprocess import PIPE, Popen, check_output from sys import exit from uuid import UUID import config DISK_BY_UUID = Path("/dev/disk/by-uuid") MAPPER_PATH = Path("/dev/mapper") """ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit # zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1 """ def main(): verify_running_as_root() local_pool = Pool(config.local_pool_to_backup) external_pool = ExternalPool.find_from_dict(config.backup_pools) external_pool.import_() try: local_pool.backup_to(external_pool) clean_old_snapshots(external_pool) finally: external_pool.export() class ZFS: @staticmethod def get_datasets(name: str) -> list[Dataset]: datasets_str = get_output_of_command( [ config.ZFS, "list", "-H", "-d", "1", "-o", "name", "-t", "filesystem", name, ] ) # we omit the first return value, which is the pool, not the dataset datasets_list = datasets_str.split("\n")[1:] return [Dataset(d) for d in datasets_list] @staticmethod def import_pool_from_directory(name, directory) -> None: # -N: no mount # -d: directory to search the pool in run_command([config.ZPOOL, "import", "-N", "-d", directory, name]) @staticmethod def export_pool(name) -> None: run_command([config.ZPOOL, "export", name]) @staticmethod def get_snapshots(name) -> list[Snapshot]: return [ Snapshot(name) for name in get_output_of_command( [ config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", "-d", "1", name, ] ).split() ] @staticmethod def destroy(full_name, *, recursive=False): if config.local_pool_to_backup in full_name: msg = "ALERT! Tried to delete in local pool!" raise Exception(msg) cmdline = [config.ZFS, "destroy"] if recursive: cmdline.append("-r") cmdline.append(full_name) run_command(cmdline) @staticmethod def send(old_snap: Snapshot, new_snap: Snapshot) -> Popen[str]: sender = Popen( [config.ZFS, "send", "-R", "-I", old_snap, new_snap], stdout=PIPE ) return sender @staticmethod def receive(sender: Popen[str], pool_name): # -d Discard the first element of the sent snapshot's file system name # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. # -u File system that is associated with the received stream is not mounted. check_output( [config.ZFS, "receive", "-d", "-F", "-u", pool_name], stdin=sender.stdout, ) class Disk: def __init__(self, uuid: UUID, name: str): self.uuid = uuid self._decrypted = False self._name = name def decrypt(self) -> None: self._verify_mapper_entry_not_in_use() self._decrypt_with_cryptsetup() self._decrypted = True def _decrypt_with_cryptsetup(self): run_command([config.CRYPTSETUP, "open", self._path, self._mapper_entry]) def _verify_mapper_entry_not_in_use(self): for item in MAPPER_PATH.iterdir(): if self._mapper_entry == item.name: raise IOError(f"mapper entry {self._mapper_entry} already exists") @property def _path(self) -> Path: return DISK_BY_UUID / str(self.uuid) @property def _mapper_entry(self) -> str: return f"crypt-{self._name}" def encrypt(self): get_output_of_command([config.CRYPTSETUP, "close", self._name]) self._decrypted = False def get_last_common_snapshot(ds1: Dataset, ds2: Dataset) -> Snapshot: common_snapshots = [ local_snap for local_snap in ds1.snapshots_matching_backup_tags for remote_snap in ds2.snapshots_matching_backup_tags if local_snap.snapname == remote_snap.snapname ] common_snapshots.sort() return common_snapshots[-1] class Pool: def __init__(self, name): self.name = name @property @cache def datasets(self) -> list[Dataset]: return ZFS.get_datasets(self.name) @property @cache def datasets_to_backup(self) -> list[Dataset]: return [ d for d in self.datasets if d.qualified_name not in config.do_not_backup ] def backup_to(self, external_pool: ExternalPool): for local_dataset in self.datasets_to_backup: remote_dataset = external_pool.search_dataset( local_dataset.name_without_pool ) last_common_snapshot = get_last_common_snapshot( remote_dataset, local_dataset ) last_local_snapshot: Snapshot = ( local_dataset.snapshots_matching_backup_tags[-1] ) sender = ZFS.send(last_common_snapshot, last_local_snapshot) ZFS.receive(sender, external_pool.name) class ExternalPool(Pool): def __init__(self, name: str, disk: Disk): super().__init__(name) self._imported: bool = False self._disk = disk def import_(self) -> None: self._disk.decrypt() self._import_from_directory(MAPPER_PATH) def export(self) -> None: self._export_pool() self._disk.encrypt() def _import_from_directory(self, directory: Path) -> None: ZFS.import_pool_from_directory(self.name, directory) def _export_pool(self) -> None: ZFS.export_pool(self.name) @classmethod def find_from_dict(cls, backup_pools: dict[str, UUID]) -> ExternalPool: present_uuids = [item.name for item in DISK_BY_UUID.iterdir()] for pool_name, disk_uuid in backup_pools.items(): if str(disk_uuid) in present_uuids: disk = Disk(disk_uuid, pool_name) return cls(pool_name, disk) else: print("Could not find a backup drive") exit(1) def search_dataset(self, name_without_pool: str) -> Dataset: qualified_name = self.name + "/" + name_without_pool for dataset in self.datasets_to_backup: if qualified_name == dataset.qualified_name: return dataset class Dataset: def __init__(self, qualified_name: str): self.qualified_name = qualified_name def __repr__(self): return self.qualified_name @property def name_without_pool(self) -> str: return self.qualified_name.split("/", maxsplit=1)[1] @property @cache def snapshots(self) -> list[Snapshot]: """get qualified snapshot names that are direct children of the dataset""" return ZFS.get_snapshots(self.qualified_name) @property @cache def snapshots_matching_backup_tags(self) -> list[Snapshot]: snapshots = [ snapshot for snapshot in self.snapshots if snapshot.matches_backup_tags ] snapshots.sort() return snapshots @property def last_snapshot(self) -> Snapshot: for snapshot in self.snapshots[::-1]: if snapshot.matches_backup_tags: return snapshot class Snapshot: def __init__(self, qualified_name: str): self.qualified_name = qualified_name def __lt__(self, other: Snapshot) -> bool: return self.snapname < other.snapname @property def snapname(self) -> str: return self.qualified_name.split("@")[1] @property def matches_backup_tags(self) -> bool: return bool( re.search("@" + ".*".join(config.snapshot_tag), self.qualified_name) ) @property def name_without_pool(self) -> str: return self.qualified_name.split("/", maxsplit=1)[1] # def clean_old_snapshots(dataset, pool): # snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' # # get just the dataset name without the pool # actual_dataset = dataset.split("/")[-1] # # print('actual_dataset', actual_dataset) # for interval in config.keep.keys(): # # print('interval', interval) # # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... # pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval # snapshots_in_interval = [] # for snapshot in snapshots: # result = re.search(pattern, snapshot) # if result is not None: # snapshots_in_interval.append(snapshot) # snapshots_in_interval.sort() # for i in range(len(snapshots_in_interval) - config.keep[interval]): # to_delete = snapshots_in_interval[i] # print(" Lösche veralteten Snapshot {}".format(to_delete)) # zfs_destroy(to_delete, recursive=True) # def run_command(cmdline: list[str]): subprocess.call(cmdline) def get_output_of_command(cmdline: str | list[str]) -> str: if isinstance(cmdline, str): cmdline = [cmdline] return subprocess.check_output(cmdline).strip().decode() def verify_running_as_root(): if os.getuid() > 0: print("Run as root.") exit(1) def clean_old_snapshots(external_pool: ExternalPool) -> None: # TODO ... if __name__ == "__main__": main()