zfs-backup/zfs-backup.py

#!/usr/bin/python
# -*- coding: utf-8 -*-

from __future__ import annotations

import os
import re
import subprocess
from functools import cache
from subprocess import PIPE, Popen, check_output
from sys import exit

import config

"""
Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
"""


def main():
    verify_running_as_root()
    local_pool = Pool(config.local_pool_to_backup)
    datasets_to_backup = local_pool.get_datasets_not_in_list(config.do_not_backup)
    backup_pool = Pool.find_from_dict(config.backup_pool)
    import_pool(backup_pool)
    try:
        send_datasets_to_backup_pool(backup_pool, datasets_to_backup)
    finally:
        export_pool(backup_pool)


class Pool:
    def __init__(self, name):
        self.name = name

    @cache
    @property
    def datasets(self) -> list[Dataset]:
        datasets_str = get_output_of_command(
            [config.ZFS, "list", "-H", "-d", "1", "-o", "name", self.name]
        )
        # we omit the first return value, which is the pool, not the dataset
        datasets_list = datasets_str.split("\n")[1:]
        to_backup = [Dataset(d) for d in datasets_list]
        return to_backup

    @cache
    def get_datasets_not_in_list(self, do_not_backup) -> list[Dataset]:
        return [d for d in self.datasets if d.name not in do_not_backup]

    @classmethod
    def find_from_dict(cls, backup_pools: dict[str, str]) -> Pool:
        print("Suche nach externer Festplatte für Backups")
        present_uuids = get_output_of_command(["ls", "/dev/disk/by-uuid"])
        for pool, uuid in backup_pools.items():
            if uuid in present_uuids:
                print("Gefundene Platte: " + pool)
                return cls(pool)
        else:
            raise IOError("Konnte keine Backup-Platte finden!")


class Dataset:
    def __init__(self, name: str):
        self.name = name

    def __repr__(self):
        return self.name


def get_snapshots(pool):
    return get_output_of_command(
        [config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool]
    ).split()


def last_snapshot(pool):
    snapshots = get_snapshots(pool)
    # check if the snapshot_tags appear in the snapshot name in order they are provided
    relevant = [
        s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s)
    ]
    # sort chronologically
    relevant.sort()
    # the most recent snapshot is the last one
    try:
        last_entry = relevant[-1]
    # didn't find any snapshots at all
    except IndexError:
        return
    last_snapshot = last_entry.strip().split("@")[-1]
    return last_snapshot


def export_pool(pool):
    container_name = "crypt-" + pool
    print("Exportiere Pool " + pool)
    get_output_of_command([config.ZPOOL, "export", pool])
    print("Schließe verschlüsselten Container")
    get_output_of_command([config.CRYPTSETUP, "close", container_name])
    print("Fertig!")
    print("Sie können die externe Festplatte jetzt entfernen.")


def import_pool(pool):
    uuid = config.backup_pool[pool]
    container_name = "crypt-" + pool
    present_mappers = get_output_of_command(["ls", "/dev/mapper/"])
    if container_name in present_mappers:
        msg = (
            "verschlüsselter Container "
            + container_name
            + " scheint bereits in Verwendung zu sein. Breche ab."
        )
        raise Exception(msg)
    print("Öffne verschlüsselten Container\n")
    get_output_of_command(
        [config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name]
    )
    print("")
    print("Importiere Pool " + pool)
    # -N: no mount
    # -d: directory to search the pool in
    get_output_of_command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool])


def present_locally(snapshot):
    local_snapshots = get_snapshots(config.local_pool_to_backup)
    for each_snapshot in local_snapshots:
        if snapshot in each_snapshot:
            return True
    else:
        return False


def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
    print("Sichere Dataset " + dataset)
    # print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
    # return
    if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
        print("  Dieses Dataset wurde nicht zur Sicherung eingerichtet.")
        print("  Verwende ein Kommando in der Art")
        print(
            "  # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN"
        )
        return
    if last_local_snapshot == last_remote_snapshot:
        print(
            "  Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot"
        )
        print("  Führe kein Backup für dieses Dataset durch")
        return
    send = Popen(
        [
            config.ZFS,
            "send",
            "-R",
            "-I",
            last_remote_snapshot,
            dataset + "@" + last_local_snapshot,
        ],
        stdout=PIPE,
    )
    # -d  Discard the first element of the sent snapshot's file system name
    # -F  Force a rollback of the file system to the most recent snapshot before performing the receive operation.
    # -u  File system that is associated with the received stream is not mounted.
    check_output(
        [config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout
    )
    exit_code = send.wait()
    if exit_code != 0:
        raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")


def snapshot_present_in_dataset(dataset, last_snapshot, pool):
    snapshots = get_snapshots(pool)
    # get just the dataset name without the pool
    actual_dataset = dataset.split("/")[-1]
    for snapshot in snapshots:
        if (actual_dataset + "@" + last_snapshot) in snapshot:
            return True
    else:
        return False


def clean_old_snapshots(dataset, pool):
    snapshots = get_snapshots(pool)  # full name like 'pool/dataset@snapshot'
    # get just the dataset name without the pool
    actual_dataset = dataset.split("/")[-1]
    # print('actual_dataset', actual_dataset)
    for interval in config.keep.keys():
        # print('interval', interval)
        # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
        pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
        snapshots_in_interval = []
        for snapshot in snapshots:
            result = re.search(pattern, snapshot)
            if result is not None:
                snapshots_in_interval.append(snapshot)
        snapshots_in_interval.sort()
        for i in range(len(snapshots_in_interval) - config.keep[interval]):
            to_delete = snapshots_in_interval[i]
            print("  Lösche veralteten Snapshot {}".format(to_delete))
            zfs_destroy(to_delete, recursive=True)


def get_output_of_command(cmdline: str | list[str]):
    if isinstance(cmdline, str):
        cmdline = [cmdline]
    return subprocess.check_output(cmdline).strip().decode()


def zfs_destroy(full_name, recursive=False):
    if config.local_pool_to_backup in full_name:
        msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!"
        raise Exception(msg)
    else:
        cmdline = [config.ZFS, "destroy"]
        if recursive:
            cmdline.append("-r")
        cmdline.append(full_name)
        get_output_of_command(cmdline)


def send_datasets_to_backup_pool(backup_pool, datasets):
    last_local_snapshot = last_snapshot(config.local_pool_to_backup)
    if not last_local_snapshot:
        print("Es scheint keine Snapshots im Speicherpool zu geben")
        print("Breche ab.")
        export_pool(backup_pool)
        exit()
    last_remote_snapshot = last_snapshot(backup_pool)
    if not last_remote_snapshot:
        print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben")
        print("Breche ab.")
        export_pool(backup_pool)
        exit()
    if not present_locally(last_remote_snapshot):
        print(
            """
Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
zu langer Zeit das letzte mal als Backup verwendet"""
        )
        return
    for dataset in datasets:
        do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
        clean_old_snapshots(dataset, backup_pool)


def verify_running_as_root():
    if os.getuid() > 0:
        print("Muss als root ausgeführt werden.")
        exit(1)


def show_actions_to_execute(datasets: list[Dataset]):
    print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:")
    for ds in datasets:
        print("  " + str(ds))


if __name__ == "__main__":
    main()