major rewrite
This commit is contained in:
timeshifter
+262
-198
@@ -7,11 +7,16 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
from subprocess import PIPE, Popen, check_output
|
||||
from sys import exit
|
||||
from uuid import UUID
|
||||
|
||||
import config
|
||||
|
||||
DISK_BY_UUID = Path("/dev/disk/by-uuid")
|
||||
MAPPER_PATH = Path("/dev/mapper")
|
||||
|
||||
"""
|
||||
Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
|
||||
# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
|
||||
@@ -21,244 +26,303 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
|
||||
def main():
|
||||
verify_running_as_root()
|
||||
local_pool = Pool(config.local_pool_to_backup)
|
||||
datasets_to_backup = local_pool.get_datasets_not_in_list(config.do_not_backup)
|
||||
backup_pool = Pool.find_from_dict(config.backup_pool)
|
||||
import_pool(backup_pool)
|
||||
external_pool = ExternalPool.find_from_dict(config.backup_pools)
|
||||
external_pool.import_()
|
||||
try:
|
||||
send_datasets_to_backup_pool(backup_pool, datasets_to_backup)
|
||||
local_pool.backup_to(external_pool)
|
||||
clean_old_snapshots(external_pool)
|
||||
finally:
|
||||
export_pool(backup_pool)
|
||||
external_pool.export()
|
||||
|
||||
|
||||
class ZFS:
|
||||
@staticmethod
|
||||
def get_datasets(name: str) -> list[Dataset]:
|
||||
datasets_str = get_output_of_command(
|
||||
[
|
||||
config.ZFS,
|
||||
"list",
|
||||
"-H",
|
||||
"-d",
|
||||
"1",
|
||||
"-o",
|
||||
"name",
|
||||
"-t",
|
||||
"filesystem",
|
||||
name,
|
||||
]
|
||||
)
|
||||
# we omit the first return value, which is the pool, not the dataset
|
||||
datasets_list = datasets_str.split("\n")[1:]
|
||||
return [Dataset(d) for d in datasets_list]
|
||||
|
||||
@staticmethod
|
||||
def import_pool_from_directory(name, directory) -> None:
|
||||
# -N: no mount
|
||||
# -d: directory to search the pool in
|
||||
run_command([config.ZPOOL, "import", "-N", "-d", directory, name])
|
||||
|
||||
@staticmethod
|
||||
def export_pool(name) -> None:
|
||||
run_command([config.ZPOOL, "export", name])
|
||||
|
||||
@staticmethod
|
||||
def get_snapshots(name) -> list[Snapshot]:
|
||||
return [
|
||||
Snapshot(name)
|
||||
for name in get_output_of_command(
|
||||
[
|
||||
config.ZFS,
|
||||
"list",
|
||||
"-r",
|
||||
"-t",
|
||||
"snapshot",
|
||||
"-o",
|
||||
"name",
|
||||
"-H",
|
||||
"-d",
|
||||
"1",
|
||||
name,
|
||||
]
|
||||
).split()
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def destroy(full_name, *, recursive=False):
|
||||
if config.local_pool_to_backup in full_name:
|
||||
msg = "ALERT! Tried to delete in local pool!"
|
||||
raise Exception(msg)
|
||||
cmdline = [config.ZFS, "destroy"]
|
||||
if recursive:
|
||||
cmdline.append("-r")
|
||||
cmdline.append(full_name)
|
||||
run_command(cmdline)
|
||||
|
||||
@staticmethod
|
||||
def send(old_snap: Snapshot, new_snap: Snapshot) -> Popen[str]:
|
||||
sender = Popen(
|
||||
[config.ZFS, "send", "-R", "-I", old_snap, new_snap], stdout=PIPE
|
||||
)
|
||||
return sender
|
||||
|
||||
@staticmethod
|
||||
def receive(sender: Popen[str], pool_name):
|
||||
# -d Discard the first element of the sent snapshot's file system name
|
||||
# -F Force a rollback of the file system to the most recent snapshot before performing the receive operation.
|
||||
# -u File system that is associated with the received stream is not mounted.
|
||||
check_output(
|
||||
[config.ZFS, "receive", "-d", "-F", "-u", pool_name],
|
||||
stdin=sender.stdout,
|
||||
)
|
||||
|
||||
|
||||
class Disk:
|
||||
def __init__(self, uuid: UUID, name: str):
|
||||
self.uuid = uuid
|
||||
self._decrypted = False
|
||||
self._name = name
|
||||
|
||||
def decrypt(self) -> None:
|
||||
self._verify_mapper_entry_not_in_use()
|
||||
self._decrypt_with_cryptsetup()
|
||||
self._decrypted = True
|
||||
|
||||
def _decrypt_with_cryptsetup(self):
|
||||
run_command([config.CRYPTSETUP, "open", self._path, self._mapper_entry])
|
||||
|
||||
def _verify_mapper_entry_not_in_use(self):
|
||||
for item in MAPPER_PATH.iterdir():
|
||||
if self._mapper_entry == item.name:
|
||||
raise IOError(f"mapper entry {self._mapper_entry} already exists")
|
||||
|
||||
@property
|
||||
def _path(self) -> Path:
|
||||
return DISK_BY_UUID / str(self.uuid)
|
||||
|
||||
@property
|
||||
def _mapper_entry(self) -> str:
|
||||
return f"crypt-{self._name}"
|
||||
|
||||
def encrypt(self):
|
||||
get_output_of_command([config.CRYPTSETUP, "close", self._name])
|
||||
self._decrypted = False
|
||||
|
||||
|
||||
def get_last_common_snapshot(ds1: Dataset, ds2: Dataset) -> Snapshot:
|
||||
common_snapshots = [
|
||||
local_snap
|
||||
for local_snap in ds1.snapshots_matching_backup_tags
|
||||
for remote_snap in ds2.snapshots_matching_backup_tags
|
||||
if local_snap.snapname == remote_snap.snapname
|
||||
]
|
||||
common_snapshots.sort()
|
||||
return common_snapshots[-1]
|
||||
|
||||
|
||||
class Pool:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
|
||||
@cache
|
||||
@property
|
||||
def datasets(self) -> list[Dataset]:
|
||||
datasets_str = get_output_of_command(
|
||||
[config.ZFS, "list", "-H", "-d", "1", "-o", "name", self.name]
|
||||
)
|
||||
# we omit the first return value, which is the pool, not the dataset
|
||||
datasets_list = datasets_str.split("\n")[1:]
|
||||
to_backup = [Dataset(d) for d in datasets_list]
|
||||
return to_backup
|
||||
|
||||
@cache
|
||||
def get_datasets_not_in_list(self, do_not_backup) -> list[Dataset]:
|
||||
return [d for d in self.datasets if d.name not in do_not_backup]
|
||||
def datasets(self) -> list[Dataset]:
|
||||
return ZFS.get_datasets(self.name)
|
||||
|
||||
@property
|
||||
@cache
|
||||
def datasets_to_backup(self) -> list[Dataset]:
|
||||
return [
|
||||
d for d in self.datasets if d.qualified_name not in config.do_not_backup
|
||||
]
|
||||
|
||||
def backup_to(self, external_pool: ExternalPool):
|
||||
for local_dataset in self.datasets_to_backup:
|
||||
remote_dataset = external_pool.search_dataset(
|
||||
local_dataset.name_without_pool
|
||||
)
|
||||
last_common_snapshot = get_last_common_snapshot(
|
||||
remote_dataset, local_dataset
|
||||
)
|
||||
last_local_snapshot: Snapshot = (
|
||||
local_dataset.snapshots_matching_backup_tags[-1]
|
||||
)
|
||||
sender = ZFS.send(last_common_snapshot, last_local_snapshot)
|
||||
ZFS.receive(sender, external_pool.name)
|
||||
|
||||
|
||||
class ExternalPool(Pool):
|
||||
def __init__(self, name: str, disk: Disk):
|
||||
super().__init__(name)
|
||||
self._imported: bool = False
|
||||
self._disk = disk
|
||||
|
||||
def import_(self) -> None:
|
||||
self._disk.decrypt()
|
||||
self._import_from_directory(MAPPER_PATH)
|
||||
|
||||
def export(self) -> None:
|
||||
self._export_pool()
|
||||
self._disk.encrypt()
|
||||
|
||||
def _import_from_directory(self, directory: Path) -> None:
|
||||
ZFS.import_pool_from_directory(self.name, directory)
|
||||
|
||||
def _export_pool(self) -> None:
|
||||
ZFS.export_pool(self.name)
|
||||
|
||||
@classmethod
|
||||
def find_from_dict(cls, backup_pools: dict[str, str]) -> Pool:
|
||||
print("Suche nach externer Festplatte für Backups")
|
||||
present_uuids = get_output_of_command(["ls", "/dev/disk/by-uuid"])
|
||||
for pool, uuid in backup_pools.items():
|
||||
if uuid in present_uuids:
|
||||
print("Gefundene Platte: " + pool)
|
||||
return cls(pool)
|
||||
def find_from_dict(cls, backup_pools: dict[str, UUID]) -> ExternalPool:
|
||||
present_uuids = list(DISK_BY_UUID.iterdir())
|
||||
for pool_name, disk_uuid in backup_pools.items():
|
||||
if str(disk_uuid) in present_uuids:
|
||||
disk = Disk(disk_uuid, pool_name)
|
||||
return cls(pool_name, disk)
|
||||
else:
|
||||
raise IOError("Konnte keine Backup-Platte finden!")
|
||||
print("Could not find a backup drive")
|
||||
exit(1)
|
||||
|
||||
def search_dataset(self, name_without_pool: str) -> Dataset:
|
||||
qualified_name = self.name + "/" + name_without_pool
|
||||
for dataset in self.datasets_to_backup:
|
||||
if qualified_name == dataset.qualified_name:
|
||||
return dataset
|
||||
|
||||
|
||||
class Dataset:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
def __init__(self, qualified_name: str):
|
||||
self.qualified_name = qualified_name
|
||||
|
||||
def __repr__(self):
|
||||
return self.name
|
||||
return self.qualified_name
|
||||
|
||||
@property
|
||||
def name_without_pool(self) -> str:
|
||||
return self.qualified_name.split("/", maxsplit=1)[1]
|
||||
|
||||
@property
|
||||
@cache
|
||||
def snapshots(self) -> list[Snapshot]:
|
||||
"""get qualified snapshot names that are direct children of the dataset"""
|
||||
return ZFS.get_snapshots(self.qualified_name)
|
||||
|
||||
@property
|
||||
@cache
|
||||
def snapshots_matching_backup_tags(self) -> list[Snapshot]:
|
||||
snapshots = [
|
||||
snapshot for snapshot in self.snapshots if snapshot.matches_backup_tags
|
||||
]
|
||||
snapshots.sort()
|
||||
return snapshots
|
||||
|
||||
@property
|
||||
def last_snapshot(self) -> Snapshot:
|
||||
for snapshot in self.snapshots[::-1]:
|
||||
if snapshot.matches_backup_tags:
|
||||
return snapshot
|
||||
|
||||
|
||||
def get_snapshots(pool):
|
||||
return get_output_of_command(
|
||||
[config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool]
|
||||
).split()
|
||||
class Snapshot:
|
||||
def __init__(self, qualified_name: str):
|
||||
self.qualified_name = qualified_name
|
||||
|
||||
def __lt__(self, other: Snapshot) -> bool:
|
||||
return self.snapname < other.snapname
|
||||
|
||||
def last_snapshot(pool):
|
||||
snapshots = get_snapshots(pool)
|
||||
# check if the snapshot_tags appear in the snapshot name in order they are provided
|
||||
relevant = [
|
||||
s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s)
|
||||
]
|
||||
# sort chronologically
|
||||
relevant.sort()
|
||||
# the most recent snapshot is the last one
|
||||
try:
|
||||
last_entry = relevant[-1]
|
||||
# didn't find any snapshots at all
|
||||
except IndexError:
|
||||
return
|
||||
last_snapshot = last_entry.strip().split("@")[-1]
|
||||
return last_snapshot
|
||||
@property
|
||||
def snapname(self) -> str:
|
||||
return self.qualified_name.split("@")[1]
|
||||
|
||||
|
||||
def export_pool(pool):
|
||||
container_name = "crypt-" + pool
|
||||
print("Exportiere Pool " + pool)
|
||||
get_output_of_command([config.ZPOOL, "export", pool])
|
||||
print("Schließe verschlüsselten Container")
|
||||
get_output_of_command([config.CRYPTSETUP, "close", container_name])
|
||||
print("Fertig!")
|
||||
print("Sie können die externe Festplatte jetzt entfernen.")
|
||||
|
||||
|
||||
def import_pool(pool):
|
||||
uuid = config.backup_pool[pool]
|
||||
container_name = "crypt-" + pool
|
||||
present_mappers = get_output_of_command(["ls", "/dev/mapper/"])
|
||||
if container_name in present_mappers:
|
||||
msg = (
|
||||
"verschlüsselter Container "
|
||||
+ container_name
|
||||
+ " scheint bereits in Verwendung zu sein. Breche ab."
|
||||
@property
|
||||
def matches_backup_tags(self) -> bool:
|
||||
return bool(
|
||||
re.search("@" + ".*".join(config.snapshot_tag), self.qualified_name)
|
||||
)
|
||||
raise Exception(msg)
|
||||
print("Öffne verschlüsselten Container\n")
|
||||
get_output_of_command(
|
||||
[config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name]
|
||||
)
|
||||
print("")
|
||||
print("Importiere Pool " + pool)
|
||||
# -N: no mount
|
||||
# -d: directory to search the pool in
|
||||
get_output_of_command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool])
|
||||
|
||||
@property
|
||||
def name_without_pool(self) -> str:
|
||||
return self.qualified_name.split("/", maxsplit=1)[1]
|
||||
|
||||
|
||||
def present_locally(snapshot):
|
||||
local_snapshots = get_snapshots(config.local_pool_to_backup)
|
||||
for each_snapshot in local_snapshots:
|
||||
if snapshot in each_snapshot:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
# def clean_old_snapshots(dataset, pool):
|
||||
# snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot'
|
||||
# # get just the dataset name without the pool
|
||||
# actual_dataset = dataset.split("/")[-1]
|
||||
# # print('actual_dataset', actual_dataset)
|
||||
# for interval in config.keep.keys():
|
||||
# # print('interval', interval)
|
||||
# # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
|
||||
# pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
|
||||
# snapshots_in_interval = []
|
||||
# for snapshot in snapshots:
|
||||
# result = re.search(pattern, snapshot)
|
||||
# if result is not None:
|
||||
# snapshots_in_interval.append(snapshot)
|
||||
# snapshots_in_interval.sort()
|
||||
# for i in range(len(snapshots_in_interval) - config.keep[interval]):
|
||||
# to_delete = snapshots_in_interval[i]
|
||||
# print(" Lösche veralteten Snapshot {}".format(to_delete))
|
||||
# zfs_destroy(to_delete, recursive=True)
|
||||
#
|
||||
|
||||
|
||||
def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
|
||||
print("Sichere Dataset " + dataset)
|
||||
# print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
|
||||
# return
|
||||
if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
|
||||
print(" Dieses Dataset wurde nicht zur Sicherung eingerichtet.")
|
||||
print(" Verwende ein Kommando in der Art")
|
||||
print(
|
||||
" # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN"
|
||||
)
|
||||
return
|
||||
if last_local_snapshot == last_remote_snapshot:
|
||||
print(
|
||||
" Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot"
|
||||
)
|
||||
print(" Führe kein Backup für dieses Dataset durch")
|
||||
return
|
||||
send = Popen(
|
||||
[
|
||||
config.ZFS,
|
||||
"send",
|
||||
"-R",
|
||||
"-I",
|
||||
last_remote_snapshot,
|
||||
dataset + "@" + last_local_snapshot,
|
||||
],
|
||||
stdout=PIPE,
|
||||
)
|
||||
# -d Discard the first element of the sent snapshot's file system name
|
||||
# -F Force a rollback of the file system to the most recent snapshot before performing the receive operation.
|
||||
# -u File system that is associated with the received stream is not mounted.
|
||||
check_output(
|
||||
[config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout
|
||||
)
|
||||
exit_code = send.wait()
|
||||
if exit_code != 0:
|
||||
raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")
|
||||
def run_command(cmdline: list[str]):
|
||||
subprocess.call(cmdline)
|
||||
|
||||
|
||||
def snapshot_present_in_dataset(dataset, last_snapshot, pool):
|
||||
snapshots = get_snapshots(pool)
|
||||
# get just the dataset name without the pool
|
||||
actual_dataset = dataset.split("/")[-1]
|
||||
for snapshot in snapshots:
|
||||
if (actual_dataset + "@" + last_snapshot) in snapshot:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def clean_old_snapshots(dataset, pool):
|
||||
snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot'
|
||||
# get just the dataset name without the pool
|
||||
actual_dataset = dataset.split("/")[-1]
|
||||
# print('actual_dataset', actual_dataset)
|
||||
for interval in config.keep.keys():
|
||||
# print('interval', interval)
|
||||
# e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
|
||||
pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
|
||||
snapshots_in_interval = []
|
||||
for snapshot in snapshots:
|
||||
result = re.search(pattern, snapshot)
|
||||
if result is not None:
|
||||
snapshots_in_interval.append(snapshot)
|
||||
snapshots_in_interval.sort()
|
||||
for i in range(len(snapshots_in_interval) - config.keep[interval]):
|
||||
to_delete = snapshots_in_interval[i]
|
||||
print(" Lösche veralteten Snapshot {}".format(to_delete))
|
||||
zfs_destroy(to_delete, recursive=True)
|
||||
|
||||
|
||||
def get_output_of_command(cmdline: str | list[str]):
|
||||
def get_output_of_command(cmdline: str | list[str]) -> str:
|
||||
if isinstance(cmdline, str):
|
||||
cmdline = [cmdline]
|
||||
return subprocess.check_output(cmdline).strip().decode()
|
||||
|
||||
|
||||
def zfs_destroy(full_name, recursive=False):
|
||||
if config.local_pool_to_backup in full_name:
|
||||
msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!"
|
||||
raise Exception(msg)
|
||||
else:
|
||||
cmdline = [config.ZFS, "destroy"]
|
||||
if recursive:
|
||||
cmdline.append("-r")
|
||||
cmdline.append(full_name)
|
||||
get_output_of_command(cmdline)
|
||||
|
||||
|
||||
def send_datasets_to_backup_pool(backup_pool, datasets):
|
||||
last_local_snapshot = last_snapshot(config.local_pool_to_backup)
|
||||
if not last_local_snapshot:
|
||||
print("Es scheint keine Snapshots im Speicherpool zu geben")
|
||||
print("Breche ab.")
|
||||
export_pool(backup_pool)
|
||||
exit()
|
||||
last_remote_snapshot = last_snapshot(backup_pool)
|
||||
if not last_remote_snapshot:
|
||||
print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben")
|
||||
print("Breche ab.")
|
||||
export_pool(backup_pool)
|
||||
exit()
|
||||
if not present_locally(last_remote_snapshot):
|
||||
print(
|
||||
"""
|
||||
Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
|
||||
keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
|
||||
zu langer Zeit das letzte mal als Backup verwendet"""
|
||||
)
|
||||
return
|
||||
for dataset in datasets:
|
||||
do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
|
||||
clean_old_snapshots(dataset, backup_pool)
|
||||
|
||||
|
||||
def verify_running_as_root():
|
||||
if os.getuid() > 0:
|
||||
print("Muss als root ausgeführt werden.")
|
||||
print("Run as root.")
|
||||
exit(1)
|
||||
|
||||
|
||||
def show_actions_to_execute(datasets: list[Dataset]):
|
||||
print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:")
|
||||
for ds in datasets:
|
||||
print(" " + str(ds))
|
||||
def clean_old_snapshots(external_pool: ExternalPool) -> None: # TODO
|
||||
...
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user