added code

binaries.py

@@ -0,0 +1,3 @@
+ZFS = '/usr/bin/zfs'
+ZPOOL = '/usr/bin/zpool'
+CRYPTSETUP = '/usr/bin/cryptsetup'

config.py

@@ -0,0 +1,30 @@
+# -*- coding: utf-8 -*-
+
+# Name des lokalen Pools
+local_pool = 'tank'
+
+# Name der Backup-Platten und UUID der Partition
+backup_pool = {
+    'backup1': '603eeae5-1f16-4d1f-8dc5-8b5368c0cdd4',
+    'backup2': '8632e9e4-3d6c-4627-8c28-ec72f52db281',
+}
+
+# Datasets, die nicht gesichert werden sollen
+do_not_backup = [
+    'tank/cache',
+    'tank/docker',
+]
+
+# überprüfe ob neue Snapshots mit diesen String-Teilen vorliegen
+snapshot_tag = ['znap', 'monthly']
+
+# wie viele Snapshots von jedem Intervall sollen beibehalten werden?
+# dies sollte der Serverkonfiguration zu zfs-auto-snapshots folgen
+keep = {
+    'frequent' : 4,
+    'hourly' : 24,
+    'daily' : 31,
+    'weekly' : 8,
+    'monthly' : 12,
+}
+

zfs-backup.py

@@ -0,0 +1,210 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+
+import os
+from subprocess import Popen, PIPE, check_output
+from binaries import ZFS, ZPOOL, CRYPTSETUP
+import config
+from sys import exit, stdout
+import re
+
+
+"""
+Erstes Backup muss für alle notwendigen Datasets durchgeführt werden mit
+# zfs send -Rv rpool/ROOT@zfs-auto-snap_daily-2018-06-14-1648 | zfs receive -dvF backup1
+"""
+
+class Command():
+    def __init__(self, command=[], strict=True):
+        process = Popen(command, stdout=PIPE)
+        self.stdout, self.stderr = process.communicate()
+        self.exit_code = process.wait()
+        if strict and self.exit_code != 0:
+            raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.')
+
+
+def find_backup_pool():
+    print('Suche nach externer Festplatte für Backups')
+    present_uuids = Command(['ls', '/dev/disk/by-uuid'])
+    for key in list(config.backup_pool.keys()):
+        uuid = config.backup_pool[key]
+        if uuid in present_uuids.stdout.decode():
+            print('Gefundene Platte: ' + key)
+            return key
+    else:
+        raise IOError('Konnte keine Backup-Platte finden!')
+
+
+def get_datasets_to_backup():
+    datasets_str = Command([ZFS, 'list', '-H', '-d', '1', '-o', 'name', config.local_pool]).stdout.strip().decode()
+    # we omit the first return value, which is the pool, not the dataset
+    datasets_list = datasets_str.split('\n')[1:]
+    to_backup = [d for d in datasets_list if d not in config.do_not_backup]
+    print('Folgende Datasets und alle darunter befindlichen Kinder werden gesichert:')
+    for tb in to_backup:
+        print('  ' + tb)
+    return to_backup
+
+
+def get_snapshots(pool):
+    return Command([ZFS, 'list', '-r', '-t', 'snapshot', '-o', 'name', '-H', pool]).stdout.strip().decode().split('\n')
+
+
+def last_snapshot(pool):
+    snapshots = get_snapshots(pool)
+    # check if the snapshot_tags appear in the snapshot name in order they are provided
+    relevant = [s for s in snapshots if re.search('@' + '.*'.join(config.snapshot_tag), s)]
+    # sort chronologically
+    relevant.sort()
+    # the most recent snapshot is the last one
+    try:
+        last_entry = relevant[-1]
+    # didn't find any snapshots at all
+    except IndexError:
+        return
+    last_snapshot = last_entry.strip().split('@')[-1]
+    return last_snapshot
+
+
+def export_pool(pool):
+    container_name = 'crypt-' + pool
+    print('Exportiere Pool ' + pool)
+    Command([ZPOOL, 'export', pool])
+    print('Schließe verschlüsselten Container')
+    Command([CRYPTSETUP, 'close', container_name])
+    print('Fertig!')
+    print('Sie können die externe Festplatte jetzt entfernen.')
+
+
+def import_pool(pool):
+    uuid = config.backup_pool[pool]
+    container_name = 'crypt-' + pool
+    present_mappers = Command(['ls', '/dev/mapper/']).stdout.decode().strip()
+    if container_name in present_mappers:
+        msg = 'verschlüsselter Container ' + container_name + \
+                ' scheint bereits in Verwendung zu sein. Breche ab.'
+        raise Exception(msg)
+    print('Öffne verschlüsselten Container\n')
+    Command([CRYPTSETUP, 'open', '/dev/disk/by-uuid/' + uuid, container_name])
+    print('')
+    print('Importiere Pool ' + pool)
+    # -N: no mount
+    # -d: directory to search the pool in
+    Command([ZPOOL, 'import', '-N', '-d', '/dev/mapper', pool])
+
+
+def present_locally(snapshot):
+    local_snapshots = get_snapshots(config.local_pool)
+    for each_snapshot in local_snapshots:
+        if snapshot in each_snapshot:
+            return True
+    else:
+        return False
+
+
+def do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool):
+    print('Sichere Dataset ' + dataset)
+    #print(dataset, last_local_snapshot, backup_pool, last_remote_snapshot)
+    #return
+    if not snapshot_present_in_dataset(dataset, last_remote_snapshot, backup_pool):
+        print('  Dieses Dataset wurde nicht zur Sicherung eingerichtet.')
+        print('  Verwende ein Kommando in der Art')
+        print('  # zfs send -Rv rpool/[DATASET]@zfs-auto-snap_daily-yyyy-mm-dd-hhmm | zfs receive -dvF backupN')
+        return
+    if last_local_snapshot == last_remote_snapshot:
+        print('  Letzter lokaler Snapshot ist identisch mit letzten entferntem Snapshot')
+        print('  Führe kein Backup für dieses Dataset durch')
+        return
+    send = Popen([
+        ZFS, 'send', '-R',
+         '-I', last_remote_snapshot,
+         dataset + '@' + last_local_snapshot],
+        stdout=PIPE)
+    # -d  Discard the first element of the sent snapshot's file system name
+    # -F  Force a rollback of the file system to the most recent snapshot before performing the receive operation.
+    # -u  File system that is associated with the received stream is not mounted.
+    receive = check_output([ZFS, 'receive', '-d', '-F', '-u', backup_pool], stdin=send.stdout)
+    exit_code = send.wait()
+    if exit_code != 0:
+        raise Exception('Kritisches Kommando fehlgeschlagen. Breche ab.')
+
+
+def snapshot_present_in_dataset(dataset, last_snapshot, pool):
+    snapshots = get_snapshots(pool)
+    # get just the dataset name without the pool
+    actual_dataset = dataset.split('/')[-1]
+    for snapshot in snapshots:
+        if (actual_dataset + '@' + last_snapshot) in snapshot:
+            return True
+    else:
+        return False
+
+
+def clean_old_snapshots(dataset, pool):
+    snapshots = get_snapshots(pool)  # full name like 'pool/dataset@snapshot'
+    # get just the dataset name without the pool
+    actual_dataset = dataset.split('/')[-1]
+    #print('actual_dataset', actual_dataset)
+    for interval in config.keep.keys():
+        #print('interval', interval)
+        # e.g. 'data@zfs-auto-snap_daily', 'ubuntu@zfs-auto-snap_hourly', ...
+        pattern = actual_dataset + '@' + config.snapshot_tag[0] + '.*' + interval
+        snapshots_in_interval = []
+        for snapshot in snapshots:
+            result = re.search(pattern, snapshot)
+            if result is not None:
+                snapshots_in_interval.append(snapshot)
+        snapshots_in_interval.sort()
+        for i in range(len(snapshots_in_interval) - config.keep[interval]):
+            to_delete = snapshots_in_interval[i]
+            print('  Lösche veralteten Snapshot {}'.format(to_delete))
+            zfs_destroy(to_delete, recursive=True)
+
+
+def zfs_destroy(full_name, recursive=False):
+    if config.local_pool in full_name:
+        msg = 'Wollte in falschem Pool löschen! Dies sollte niemals passieren! Abbruch!'
+        raise Exception(msg)
+    else:
+        cmdline = [ZFS, 'destroy']
+        if recursive:
+            cmdline.append('-r')
+        cmdline.append(full_name)
+        Command(cmdline)
+
+
+def main():
+    if os.getuid() > 0:
+        print('Muss als root ausgeführt werden.')
+        exit(1)
+    datasets = get_datasets_to_backup()
+    backup_pool = find_backup_pool()
+    import_pool(backup_pool)
+    try:
+        last_local_snapshot = last_snapshot(config.local_pool)
+        if not last_local_snapshot:
+            print('Es scheint keine Snapshots im Speicherpool zu geben')
+            print('Breche ab.')
+            export_pool(backup_pool)
+            exit()
+        last_remote_snapshot = last_snapshot(backup_pool)
+        if not last_remote_snapshot:
+            print('Es scheint keine Snapshots auf der Backup-Festplatte zu geben')
+            print('Breche ab.')
+            export_pool(backup_pool)
+            exit()
+        if not present_locally(last_remote_snapshot):
+            print("""
+    Kann keine inkrementelle Sicherung durchführen, da es im Backup und auf dem Server
+    keinen gemeinsamen Snapshot gibt. Vermutlich wurde diese externe Festplatte vor
+    zu langer Zeit das letzte mal als Backup verwendet""")
+            return
+        for dataset in datasets:
+            do_backup(dataset, last_local_snapshot, last_remote_snapshot, backup_pool)
+            clean_old_snapshots(dataset, backup_pool)
+    finally:
+        export_pool(backup_pool)
+
+
+if __name__ == '__main__':
+    main()