timeshifter/zfs-backup
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

major rewrite

Browse Source
This commit is contained in:
timeshifter
2021-11-25 19:47:36 +01:00
parent de795cdba3
commit c3889fdd7c
2 changed files with 266 additions and 201 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.py
+4 -3
View File
@@ -1,12 +1,13 @@
# -*- coding: utf-8 -*-
from uuid import UUID
# Name des lokalen Pools
local_pool_to_backup = "tank"
# Name der Backup-Platten und UUID der Partition
backup_pool = {
"backup1": "603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4",
"backup2": "8632e9e4-3d6c-4627-8c28-ec72f52db281",
backup_pools = {
"backup1": UUID("603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4"),
"backup2": UUID("8632e9e4-3d6c-4627-8c28-ec72f52db281"),
}
# Datasets, die nicht gesichert werden sollen
zfs-backup.py
+259 -195
View File
@@ -7,11 +7,16 @@ import os
import re
import subprocess
from functools import cache
from pathlib import Path
from subprocess import PIPE, Popen, check_output
from sys import exit
from uuid import UUID
import config
DISK_BY_UUID = Path("/dev/disk/by-uuid")
MAPPER_PATH = Path("/dev/mapper")
"""
Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
@@ -21,244 +26,303 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
def main():
verify_running_as_root()
local_pool = Pool(config.local_pool_to_backup)
datasets_to_backup = local_pool.get_datasets_not_in_list(config.do_not_backup)
backup_pool = Pool.find_from_dict(config.backup_pool)
import_pool(backup_pool)
external_pool = ExternalPool.find_from_dict(config.backup_pools)
external_pool.import_()
try:
send_datasets_to_backup_pool(backup_pool, datasets_to_backup)
local_pool.backup_to(external_pool)
clean_old_snapshots(external_pool)
finally:
export_pool(backup_pool)
external_pool.export()
class ZFS:
@staticmethod
def get_datasets(name: str) -> list[Dataset]:
datasets_str = get_output_of_command(
[
config.ZFS,
"list",
"-H",
"-d",
"1",
"-o",
"name",
"-t",
"filesystem",
name,
]
)
# we omit the first return value, which is the pool, not the dataset
datasets_list = datasets_str.split("\n")[1:]
return [Dataset(d) for d in datasets_list]
@staticmethod
def import_pool_from_directory(name, directory) -> None:
# -N: no mount
# -d: directory to search the pool in
run_command([config.ZPOOL, "import", "-N", "-d", directory, name])
@staticmethod
def export_pool(name) -> None:
run_command([config.ZPOOL, "export", name])
@staticmethod
def get_snapshots(name) -> list[Snapshot]:
return [
Snapshot(name)
for name in get_output_of_command(
[
config.ZFS,
"list",
"-r",
"-t",
"snapshot",
"-o",
"name",
"-H",
"-d",
"1",
name,
]
).split()
]
@staticmethod
def destroy(full_name, *, recursive=False):
if config.local_pool_to_backup in full_name:
msg = "ALERT! Tried to delete in local pool!"
raise Exception(msg)
cmdline = [config.ZFS, "destroy"]
if recursive:
cmdline.append("-r")
cmdline.append(full_name)
run_command(cmdline)
@staticmethod
def send(old_snap: Snapshot, new_snap: Snapshot) -> Popen[str]:
sender = Popen(
[config.ZFS, "send", "-R", "-I", old_snap, new_snap], stdout=PIPE
)
return sender
@staticmethod
def receive(sender: Popen[str], pool_name):
# -d Discard the first element of the sent snapshot's file system name
# -F Force a rollback of the file system to the most recent snapshot before performing the receive operation.
# -u File system that is associated with the received stream is not mounted.
check_output(
[config.ZFS, "receive", "-d", "-F", "-u", pool_name],
stdin=sender.stdout,
)
class Disk:
def __init__(self, uuid: UUID, name: str):
self.uuid = uuid
self._decrypted = False
self._name = name
def decrypt(self) -> None:
self._verify_mapper_entry_not_in_use()
self._decrypt_with_cryptsetup()
self._decrypted = True
def _decrypt_with_cryptsetup(self):
run_command([config.CRYPTSETUP, "open", self._path, self._mapper_entry])
def _verify_mapper_entry_not_in_use(self):
for item in MAPPER_PATH.iterdir():
if self._mapper_entry == item.name:
raise IOError(f"mapper entry {self._mapper_entry} already exists")
@property
def _path(self) -> Path:
return DISK_BY_UUID / str(self.uuid)
@property
def _mapper_entry(self) -> str:
return f"crypt-{self._name}"
def encrypt(self):
get_output_of_command([config.CRYPTSETUP, "close", self._name])
self._decrypted = False
def get_last_common_snapshot(ds1: Dataset, ds2: Dataset) -> Snapshot:
common_snapshots = [
local_snap
for local_snap in ds1.snapshots_matching_backup_tags
for remote_snap in ds2.snapshots_matching_backup_tags
if local_snap.snapname == remote_snap.snapname
]
common_snapshots.sort()
return common_snapshots[-1]
class Pool:
def __init__(self, name):
self.name = name
@cache
@property
def datasets(self) -> list[Dataset]:
datasets_str = get_output_of_command(
[config.ZFS, "list", "-H", "-d", "1", "-o", "name", self.name]
)
# we omit the first return value, which is the pool, not the dataset
datasets_list = datasets_str.split("\n")[1:]
to_backup = [Dataset(d) for d in datasets_list]
return to_backup
@cache
def get_datasets_not_in_list(self, do_not_backup) -> list[Dataset]:
return [d for d in self.datasets if d.name not in do_not_backup]
def datasets(self) -> list[Dataset]:
return ZFS.get_datasets(self.name)
@property
@cache
def datasets_to_backup(self) -> list[Dataset]:
return [
d for d in self.datasets if d.qualified_name not in config.do_not_backup
]
def backup_to(self, external_pool: ExternalPool):
for local_dataset in self.datasets_to_backup:
remote_dataset = external_pool.search_dataset(
local_dataset.name_without_pool
)
last_common_snapshot = get_last_common_snapshot(
remote_dataset, local_dataset
)
last_local_snapshot: Snapshot = (
local_dataset.snapshots_matching_backup_tags[-1]
)
sender = ZFS.send(last_common_snapshot, last_local_snapshot)
ZFS.receive(sender, external_pool.name)
class ExternalPool(Pool):
def __init__(self, name: str, disk: Disk):
super().__init__(name)
self._imported: bool = False
self._disk = disk
def import_(self) -> None:
self._disk.decrypt()
self._import_from_directory(MAPPER_PATH)
def export(self) -> None:
self._export_pool()
self._disk.encrypt()
def _import_from_directory(self, directory: Path) -> None:
ZFS.import_pool_from_directory(self.name, directory)
def _export_pool(self) -> None:
ZFS.export_pool(self.name)
@classmethod
def find_from_dict(cls, backup_pools: dict[str, str]) -> Pool:
print("Suche nach externer Festplatte für Backups")
present_uuids = get_output_of_command(["ls", "/dev/disk/by-uuid"])
for pool, uuid in backup_pools.items():
if uuid in present_uuids:
print("Gefundene Platte: " + pool)
return cls(pool)
def find_from_dict(cls, backup_pools: dict[str, UUID]) -> ExternalPool:
present_uuids = list(DISK_BY_UUID.iterdir())
for pool_name, disk_uuid in backup_pools.items():
if str(disk_uuid) in present_uuids:
disk = Disk(disk_uuid, pool_name)
return cls(pool_name, disk)
else:
raise IOError("Konnte keine Backup-Platte finden!")
print("Could not find a backup drive")
exit(1)
def search_dataset(self, name_without_pool: str) -> Dataset:
qualified_name = self.name + "/" + name_without_pool
for dataset in self.datasets_to_backup:
if qualified_name == dataset.qualified_name:
return dataset
class Dataset:
def __init__(self, name: str):
self.name = name
def __init__(self, qualified_name: str):
self.qualified_name = qualified_name
def __repr__(self):
return self.name
return self.qualified_name
@property
def name_without_pool(self) -> str:
return self.qualified_name.split("/", maxsplit=1)[1]
def get_snapshots(pool):
return get_output_of_command(
[config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool]
).split()
@property
@cache
def snapshots(self) -> list[Snapshot]:
"""get qualified snapshot names that are direct children of the dataset"""
return ZFS.get_snapshots(self.qualified_name)
def last_snapshot(pool):
snapshots = get_snapshots(pool)
# check if the snapshot_tags appear in the snapshot name in order they are provided
relevant = [
s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s)
@property
@cache
def snapshots_matching_backup_tags(self) -> list[Snapshot]:
snapshots = [
snapshot for snapshot in self.snapshots if snapshot.matches_backup_tags
]
# sort chronologically
relevant.sort()
# the most recent snapshot is the last one
try:
last_entry = relevant[-1]
# didn't find any snapshots at all
except IndexError:
return
last_snapshot = last_entry.strip().split("@")[-1]
return last_snapshot
snapshots.sort()
return snapshots
@property
def last_snapshot(self) -> Snapshot:
for snapshot in self.snapshots[::-1]:
if snapshot.matches_backup_tags:
return snapshot
def export_pool(pool):
container_name = "crypt-" + pool
print("Exportiere Pool " + pool)
get_output_of_command([config.ZPOOL, "export", pool])
print("Schließe verschlüsselten Container")
get_output_of_command([config.CRYPTSETUP, "close", container_name])
print("Fertig!")
print("Sie können die externe Festplatte jetzt entfernen.")
class Snapshot:
def __init__(self, qualified_name: str):
self.qualified_name = qualified_name
def __lt__(self, other: Snapshot) -> bool:
return self.snapname < other.snapname
def import_pool(pool):
uuid = config.backup_pool[pool]
container_name = "crypt-" + pool
present_mappers = get_output_of_command(["ls", "/dev/mapper/"])
if container_name in present_mappers:
msg = (
"verschlüsselter Container "
+ container_name
+ " scheint bereits in Verwendung zu sein. Breche ab."
@property
def snapname(self) -> str:
return self.qualified_name.split("@")[1]
@property
def matches_backup_tags(self) -> bool:
return bool(
re.search("@" + ".*".join(config.snapshot_tag), self.qualified_name)
)
raise Exception(msg)
print("Öffne verschlüsselten Container\n")
get_output_of_command(
[config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name]
)
print("")
print("Importiere Pool " + pool)
# -N: no mount
# -d: directory to search the pool in
get_output_of_command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool])
@property
def name_without_pool(self) -> str:
return self.qualified_name.split("/", maxsplit=1)[1]
def present_locally(snapshot):
local_snapshots = get_snapshots(config.local_pool_to_backup)
for each_snapshot in local_snapshots:
if snapshot in each_snapshot:
return True
else:
return False
# def clean_old_snapshots(dataset, pool):
# snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot'
# # get just the dataset name without the pool
# actual_dataset = dataset.split("/")[-1]
# # print('actual_dataset', actual_dataset)
# for interval in config.keep.keys():
# # print('interval', interval)
# # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
# pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
# snapshots_in_interval = []
# for snapshot in snapshots:
# result = re.search(pattern, snapshot)
# if result is not None:
# snapshots_in_interval.append(snapshot)
# snapshots_in_interval.sort()
# for i in range(len(snapshots_in_interval) - config.keep[interval]):
# to_delete = snapshots_in_interval[i]
# print(" Lösche veralteten Snapshot {}".format(to_delete))
# zfs_destroy(to_delete, recursive=True)
#
def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
print("Sichere Dataset " + dataset)
# print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
# return
if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
print(" Dieses Dataset wurde nicht zur Sicherung eingerichtet.")
print(" Verwende ein Kommando in der Art")
print(
" # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN"
)
return
if last_local_snapshot == last_remote_snapshot:
print(
" Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot"
)
print(" Führe kein Backup für dieses Dataset durch")
return
send = Popen(
[
config.ZFS,
"send",
"-R",
"-I",
last_remote_snapshot,
dataset + "@" + last_local_snapshot,
],
stdout=PIPE,
)
# -d Discard the first element of the sent snapshot's file system name
# -F Force a rollback of the file system to the most recent snapshot before performing the receive operation.
# -u File system that is associated with the received stream is not mounted.
check_output(
[config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout
)
exit_code = send.wait()
if exit_code != 0:
raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")
def run_command(cmdline: list[str]):
subprocess.call(cmdline)
def snapshot_present_in_dataset(dataset, last_snapshot, pool):
snapshots = get_snapshots(pool)
# get just the dataset name without the pool
actual_dataset = dataset.split("/")[-1]
for snapshot in snapshots:
if (actual_dataset + "@" + last_snapshot) in snapshot:
return True
else:
return False
def clean_old_snapshots(dataset, pool):
snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot'
# get just the dataset name without the pool
actual_dataset = dataset.split("/")[-1]
# print('actual_dataset', actual_dataset)
for interval in config.keep.keys():
# print('interval', interval)
# e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
snapshots_in_interval = []
for snapshot in snapshots:
result = re.search(pattern, snapshot)
if result is not None:
snapshots_in_interval.append(snapshot)
snapshots_in_interval.sort()
for i in range(len(snapshots_in_interval) - config.keep[interval]):
to_delete = snapshots_in_interval[i]
print(" Lösche veralteten Snapshot {}".format(to_delete))
zfs_destroy(to_delete, recursive=True)
def get_output_of_command(cmdline: str | list[str]):
def get_output_of_command(cmdline: str | list[str]) -> str:
if isinstance(cmdline, str):
cmdline = [cmdline]
return subprocess.check_output(cmdline).strip().decode()
def zfs_destroy(full_name, recursive=False):
if config.local_pool_to_backup in full_name:
msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!"
raise Exception(msg)
else:
cmdline = [config.ZFS, "destroy"]
if recursive:
cmdline.append("-r")
cmdline.append(full_name)
get_output_of_command(cmdline)
def send_datasets_to_backup_pool(backup_pool, datasets):
last_local_snapshot = last_snapshot(config.local_pool_to_backup)
if not last_local_snapshot:
print("Es scheint keine Snapshots im Speicherpool zu geben")
print("Breche ab.")
export_pool(backup_pool)
exit()
last_remote_snapshot = last_snapshot(backup_pool)
if not last_remote_snapshot:
print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben")
print("Breche ab.")
export_pool(backup_pool)
exit()
if not present_locally(last_remote_snapshot):
print(
"""
Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
zu langer Zeit das letzte mal als Backup verwendet"""
)
return
for dataset in datasets:
do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
clean_old_snapshots(dataset, backup_pool)
def verify_running_as_root():
if os.getuid() > 0:
print("Muss als root ausgeführt werden.")
print("Run as root.")
exit(1)
def show_actions_to_execute(datasets: list[Dataset]):
print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:")
for ds in datasets:
print(" " + str(ds))
def clean_old_snapshots(external_pool: ExternalPool) -> None: # TODO
...
if __name__ == "__main__":