timeshifter/zfs-backup
1
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

initial cleanup of project

Browse Source
This commit is contained in:
timeshifter
2021-11-25 15:30:44 +01:00
parent f866e7f596
commit 587b677a37
3 changed files with 111 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files
binaries.py
-3
View File
@@ -1,3 +0,0 @@
ZFS = '/usr/bin/zfs'
ZPOOL = '/usr/bin/zpool'
CRYPTSETUP = '/usr/bin/cryptsetup'
config.py
+14 -11
View File
@@ -1,30 +1,33 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# Name des lokalen Pools # Name des lokalen Pools
local_pool = 'tank' local_pool = "tank"
# Name der Backup-Platten und UUID der Partition # Name der Backup-Platten und UUID der Partition
backup_pool = { backup_pool = {
'backup1': '603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4', "backup1": "603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4",
'backup2': '8632e9e4-3d6c-4627-8c28-ec72f52db281', "backup2": "8632e9e4-3d6c-4627-8c28-ec72f52db281",
} }
# Datasets, die nicht gesichert werden sollen # Datasets, die nicht gesichert werden sollen
do_not_backup = [ do_not_backup = [
'tank/cache', "tank/cache",
'tank/docker', "tank/docker",
] ]
# überprüfe ob neue Snapshots mit diesen String-Teilen vorliegen # überprüfe ob neue Snapshots mit diesen String-Teilen vorliegen
snapshot_tag = ['znap', 'monthly'] snapshot_tag = ["znap", "monthly"]
# wie viele Snapshots von jedem Intervall sollen beibehalten werden? # wie viele Snapshots von jedem Intervall sollen beibehalten werden?
# dies sollte der Serverkonfiguration zu zfs-auto-snapshots folgen # dies sollte der Serverkonfiguration zu zfs-auto-snapshots folgen
keep = { keep = {
'frequent' : 4, "frequent": 4,
'hourly' : 24, "hourly": 24,
'daily' : 31, "daily": 31,
'weekly' : 8, "weekly": 8,
'monthly' : 12, "monthly": 12,
} }
ZFS = "/usr/bin/zfs"
ZPOOL = "/usr/bin/zpool"
CRYPTSETUP = "/usr/bin/cryptsetup"
zfs-backup.py
+97 -67
View File
@@ -3,9 +3,8 @@
import os import os
from subprocess import Popen, PIPE, check_output from subprocess import Popen, PIPE, check_output
from binaries import ZFS, ZPOOL, CRYPTSETUP
import config import config
from sys import exit, stdout from sys import exit
import re import re
@@ -14,46 +13,58 @@ Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1 # zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
""" """
class Command():
def __init__(self, command=[], strict=True): class Command:
def __init__(self, command, strict=True):
process = Popen(command, stdout=PIPE) process = Popen(command, stdout=PIPE)
self.stdout, self.stderr = process.communicate() self.stdout, self.stderr = process.communicate()
self.exit_code = process.wait() self.exit_code = process.wait()
if strict and self.exit_code != 0: if strict and self.exit_code != 0:
raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")
def find_backup_pool(): def find_backup_pool():
print('Suche nach externer Festplatte für Backups') print("Suche nach externer Festplatte für Backups")
present_uuids = Command(['ls', '/dev/disk/by-uuid']) present_uuids = Command(["ls", "/dev/disk/by-uuid"])
for key in list(config.backup_pool.keys()): for key in list(config.backup_pool.keys()):
uuid = config.backup_pool[key] uuid = config.backup_pool[key]
if uuid in present_uuids.stdout.decode(): if uuid in present_uuids.stdout.decode():
print('Gefundene Platte: ' + key) print("Gefundene Platte: " + key)
return key return key
else: else:
raise IOError('Konnte keine Backup-Platte finden!') raise IOError("Konnte keine Backup-Platte finden!")
def get_datasets_to_backup(): def get_datasets_to_backup():
datasets_str = Command([ZFS, 'list', '-H', '-d', '1', '-o', 'name', config.local_pool]).stdout.strip().decode() datasets_str = (
Command([config.ZFS, "list", "-H", "-d", "1", "-o", "name", config.local_pool])
.stdout.strip()
.decode()
)
# we omit the first return value, which is the pool, not the dataset # we omit the first return value, which is the pool, not the dataset
datasets_list = datasets_str.split('\n')[1:] datasets_list = datasets_str.split("\n")[1:]
to_backup = [d for d in datasets_list if d not in config.do_not_backup] to_backup = [d for d in datasets_list if d not in config.do_not_backup]
print('Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:') print("Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:")
for tb in to_backup: for tb in to_backup:
print(' ' + tb) print(" " + tb)
return to_backup return to_backup
def get_snapshots(pool): def get_snapshots(pool):
return Command([ZFS, 'list', '-r', '-t', 'snapshot', '-o', 'name', '-H', pool]).stdout.strip().decode().split('\n') return (
Command([config.ZFS, "list", "-r", "-t", "snapshot", "-o", "name", "-H", pool])
.stdout.strip()
.decode()
.split("\n")
)
def last_snapshot(pool): def last_snapshot(pool):
snapshots = get_snapshots(pool) snapshots = get_snapshots(pool)
# check if the snapshot_tags appear in the snapshot name in order they are provided # check if the snapshot_tags appear in the snapshot name in order they are provided
relevant = [s for s in snapshots if re.search('@' + '.*'.join(config.snapshot_tag), s)] relevant = [
s for s in snapshots if re.search("@" + ".*".join(config.snapshot_tag), s)
]
# sort chronologically # sort chronologically
relevant.sort() relevant.sort()
# the most recent snapshot is the last one # the most recent snapshot is the last one
@@ -62,35 +73,38 @@ def last_snapshot(pool):
# didn't find any snapshots at all # didn't find any snapshots at all
except IndexError: except IndexError:
return return
last_snapshot = last_entry.strip().split('@')[-1] last_snapshot = last_entry.strip().split("@")[-1]
return last_snapshot return last_snapshot
def export_pool(pool): def export_pool(pool):
container_name = 'crypt-' + pool container_name = "crypt-" + pool
print('Exportiere Pool ' + pool) print("Exportiere Pool " + pool)
Command([ZPOOL, 'export', pool]) Command([config.ZPOOL, "export", pool])
print('Schließe verschlüsselten Container') print("Schließe verschlüsselten Container")
Command([CRYPTSETUP, 'close', container_name]) Command([config.CRYPTSETUP, "close", container_name])
print('Fertig!') print("Fertig!")
print('Sie können die externe Festplatte jetzt entfernen.') print("Sie können die externe Festplatte jetzt entfernen.")
def import_pool(pool): def import_pool(pool):
uuid = config.backup_pool[pool] uuid = config.backup_pool[pool]
container_name = 'crypt-' + pool container_name = "crypt-" + pool
present_mappers = Command(['ls', '/dev/mapper/']).stdout.decode().strip() present_mappers = Command(["ls", "/dev/mapper/"]).stdout.decode().strip()
if container_name in present_mappers: if container_name in present_mappers:
msg = 'verschlüsselter Container ' + container_name + \ msg = (
' scheint bereits in Verwendung zu sein. Breche ab.' "verschlüsselter Container "
+ container_name
+ " scheint bereits in Verwendung zu sein. Breche ab."
)
raise Exception(msg) raise Exception(msg)
print('Öffne verschlüsselten Container\n') print("Öffne verschlüsselten Container\n")
Command([CRYPTSETUP, 'open', '/dev/disk/by-uuid/' + uuid, container_name]) Command([config.CRYPTSETUP, "open", "/dev/disk/by-uuid/" + uuid, container_name])
print('') print("")
print('Importiere Pool ' + pool) print("Importiere Pool " + pool)
# -N: no mount # -N: no mount
# -d: directory to search the pool in # -d: directory to search the pool in
Command([ZPOOL, 'import', '-N', '-d', '/dev/mapper', pool]) Command([config.ZPOOL, "import", "-N", "-d", "/dev/mapper", pool])
def present_locally(snapshot): def present_locally(snapshot):
@@ -103,38 +117,48 @@ def present_locally(snapshot):
def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool): def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
print('Sichere Dataset ' + dataset) print("Sichere Dataset " + dataset)
#print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot) # print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
#return # return
if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool): if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
print(' Dieses Dataset wurde nicht zur Sicherung eingerichtet.') print(" Dieses Dataset wurde nicht zur Sicherung eingerichtet.")
print(' Verwende ein Kommando in der Art') print(" Verwende ein Kommando in der Art")
print(' # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN') print(
" # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN"
)
return return
if last_local_snapshot == last_remote_snapshot: if last_local_snapshot == last_remote_snapshot:
print(' Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot') print(
print(' Führe kein Backup für dieses Dataset durch') " Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot"
)
print(" Führe kein Backup für dieses Dataset durch")
return return
send = Popen([ send = Popen(
ZFS, 'send', '-R', [
'-I', last_remote_snapshot, config.ZFS,
dataset + '@' + last_local_snapshot], "send",
stdout=PIPE) "-R",
"-I",
last_remote_snapshot,
dataset + "@" + last_local_snapshot,
],
stdout=PIPE,
)
# -d Discard the first element of the sent snapshot's file system name # -d Discard the first element of the sent snapshot's file system name
# -F Force a rollback of the file system to the most recent snapshot before performing the receive operation. # -F Force a rollback of the file system to the most recent snapshot before performing the receive operation.
# -u File system that is associated with the received stream is not mounted. # -u File system that is associated with the received stream is not mounted.
receive = check_output([ZFS, 'receive', '-d', '-F', '-u', backup_pool], stdin=send.stdout) check_output([config.ZFS, "receive", "-d", "-F", "-u", backup_pool], stdin=send.stdout)
exit_code = send.wait() exit_code = send.wait()
if exit_code != 0: if exit_code != 0:
raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.') raise Exception("Kritisches Kommando fehlgeschlagen. Breche ab.")
def snapshot_present_in_dataset(dataset, last_snapshot, pool): def snapshot_present_in_dataset(dataset, last_snapshot, pool):
snapshots = get_snapshots(pool) snapshots = get_snapshots(pool)
# get just the dataset name without the pool # get just the dataset name without the pool
actual_dataset = dataset.split('/')[-1] actual_dataset = dataset.split("/")[-1]
for snapshot in snapshots: for snapshot in snapshots:
if (actual_dataset + '@' + last_snapshot) in snapshot: if (actual_dataset + "@" + last_snapshot) in snapshot:
return True return True
else: else:
return False return False
@@ -143,12 +167,12 @@ def snapshot_present_in_dataset(dataset, last_snapshot, pool):
def clean_old_snapshots(dataset, pool): def clean_old_snapshots(dataset, pool):
snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot' snapshots = get_snapshots(pool) # full name like 'pool/dataset@snapshot'
# get just the dataset name without the pool # get just the dataset name without the pool
actual_dataset = dataset.split('/')[-1] actual_dataset = dataset.split("/")[-1]
#print('actual_dataset', actual_dataset) # print('actual_dataset', actual_dataset)
for interval in config.keep.keys(): for interval in config.keep.keys():
#print('interval', interval) # print('interval', interval)
# e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ... # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
pattern = actual_dataset + '@' + config.snapshot_tag[0] + '.*' + interval pattern = actual_dataset + "@" + config.snapshot_tag[0] + ".*" + interval
snapshots_in_interval = [] snapshots_in_interval = []
for snapshot in snapshots: for snapshot in snapshots:
result = re.search(pattern, snapshot) result = re.search(pattern, snapshot)
@@ -157,47 +181,47 @@ def clean_old_snapshots(dataset, pool):
snapshots_in_interval.sort() snapshots_in_interval.sort()
for i in range(len(snapshots_in_interval) - config.keep[interval]): for i in range(len(snapshots_in_interval) - config.keep[interval]):
to_delete = snapshots_in_interval[i] to_delete = snapshots_in_interval[i]
print(' Lösche veralteten Snapshot {}'.format(to_delete)) print(" Lösche veralteten Snapshot {}".format(to_delete))
zfs_destroy(to_delete, recursive=True) zfs_destroy(to_delete, recursive=True)
def zfs_destroy(full_name, recursive=False): def zfs_destroy(full_name, recursive=False):
if config.local_pool in full_name: if config.local_pool in full_name:
msg = 'Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!' msg = "Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!"
raise Exception(msg) raise Exception(msg)
else: else:
cmdline = [ZFS, 'destroy'] cmdline = [config.ZFS, "destroy"]
if recursive: if recursive:
cmdline.append('-r') cmdline.append("-r")
cmdline.append(full_name) cmdline.append(full_name)
Command(cmdline) Command(cmdline)
def main(): def main():
if os.getuid() > 0: verify_running_as_root()
print('Muss als root ausgeführt werden.')
exit(1)
datasets = get_datasets_to_backup() datasets = get_datasets_to_backup()
backup_pool = find_backup_pool() backup_pool = find_backup_pool()
import_pool(backup_pool) import_pool(backup_pool)
try: try:
last_local_snapshot = last_snapshot(config.local_pool) last_local_snapshot = last_snapshot(config.local_pool)
if not last_local_snapshot: if not last_local_snapshot:
print('Es scheint keine Snapshots im Speicherpool zu geben') print("Es scheint keine Snapshots im Speicherpool zu geben")
print('Breche ab.') print("Breche ab.")
export_pool(backup_pool) export_pool(backup_pool)
exit() exit()
last_remote_snapshot = last_snapshot(backup_pool) last_remote_snapshot = last_snapshot(backup_pool)
if not last_remote_snapshot: if not last_remote_snapshot:
print('Es scheint keine Snapshots auf der Backup-Festplatte zu geben') print("Es scheint keine Snapshots auf der Backup-Festplatte zu geben")
print('Breche ab.') print("Breche ab.")
export_pool(backup_pool) export_pool(backup_pool)
exit() exit()
if not present_locally(last_remote_snapshot): if not present_locally(last_remote_snapshot):
print(""" print(
"""
Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
zu langer Zeit das letzte mal als Backup verwendet""") zu langer Zeit das letzte mal als Backup verwendet"""
)
return return
for dataset in datasets: for dataset in datasets:
do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool) do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
@@ -206,5 +230,11 @@ def main():
export_pool(backup_pool) export_pool(backup_pool)
if __name__ == '__main__': def verify_running_as_root():
if os.getuid() > 0:
print("Muss als root ausgeführt werden.")
exit(1)
if __name__ == "__main__":
main() main()