Implement snapshotting when changed

tests/test_keep.py

@@ -1,15 +1,17 @@
 from datetime import datetime
-from zfs_smart_snapshot.main import keep_year, keep_month, keep_day, keep_hour, keep_frequent
+from zfs_smart_snapshot.main import keep_year, keep_month, keep_day, keep_hour, keep_frequent, snapshots_to_keep
 
 
-dates = [datetime(2018, 2, 3), datetime(2019, 3, 4), datetime(2020, 4, 5), datetime(2021, 1, 2),
+dates = [datetime(2018, 2, 3, 3, 4), datetime(2019, 3, 4, 4, 5), datetime(2020, 4, 5, 3, 4),
-         datetime(2021, 2, 2), datetime(2021, 2, 3), datetime(2021, 3, 3), datetime(2021, 3, 4),
+         datetime(2021, 1, 2, 3, 4), datetime(2021, 2, 2, 12, 59), datetime(2021, 2, 3, 1, 0), datetime(2021, 3, 3, 1, 0), datetime(2021, 3, 4, 23, 59),
-         datetime(2021, 3, 5)]
+         datetime(2021, 3, 5, 0, 0), datetime(2021, 3, 5, 17, 0, 0), datetime(2021, 3, 5, 18, 0, 0), datetime(2021, 3, 5, 19, 0, 0),
+         datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
+         datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]
 now = datetime(2021, 3, 5, 22, 45, 0)
 
 
 def test_keep_year():
-    assert keep_year(now, dates, 2) == {datetime(2020, 4, 5), datetime(2021, 1, 2)}
+    assert keep_year(now, dates, 2) == {datetime(2020, 4, 5, 3, 4), datetime(2021, 1, 2, 3, 4)}
     assert keep_year(now, dates, 0) == set()
     dates_gap = [datetime(2018, 2, 3), datetime(2019, 3, 4), datetime(2021, 5, 6)]
     assert keep_year(datetime(2021, 5, 6), dates_gap, 3) == {datetime(2019, 3, 4), datetime(2021, 5, 6)}
@@ -17,23 +19,27 @@ def test_keep_year():
 
 
 def test_keep_month():
-    assert keep_month(now, dates, 3) == {datetime(2021, 1, 2), datetime(2021, 2, 2), datetime(2021, 3, 3)}
+    assert keep_month(now, dates, 3) == {datetime(2021, 1, 2, 3, 4), datetime(2021, 2, 2, 12, 59), datetime(2021, 3, 3, 1, 0)}
 
 
 def test_keep_day():
-    assert keep_day(now, dates, 3) == {datetime(2021, 3, 3), datetime(2021, 3, 4), datetime(2021, 3, 5)}
+    assert keep_day(now, dates, 3) == {datetime(2021, 3, 3, 1, 0), datetime(2021, 3, 4, 23, 59), datetime(2021, 3, 5, 0, 0)}
-
-
-datetimes = [datetime(2021, 3, 5, 17, 0, 0), datetime(2021, 3, 5, 18, 0, 0), datetime(2021, 3, 5, 19, 0, 0),
-             datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
-             datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]
 
 
 def test_keep_hour():
-    assert keep_hour(now, datetimes, 3) == {datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0),
+    assert keep_hour(now, dates, 3) == {datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0),
                                             datetime(2021, 3, 5, 22, 0, 0)}
 
 
 def test_keep_frequent():
-    assert keep_frequent(now, datetimes, 3) == {datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0),
+    assert keep_frequent(now, dates, 3) == {datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0),
                                                 datetime(2021, 3, 5, 22, 45, 0)}
+
+
+def test_keep():
+    assert snapshots_to_keep(now, dates, yearly=2, monthly=2, daily=3, hourly=2, frequently=4) == \
+        [datetime(2020, 4, 5, 3, 4), datetime(2021, 1, 2, 3, 4),
+         datetime(2021, 2, 2, 12, 59), datetime(2021, 3, 3, 1, 0),
+         datetime(2021, 3, 4, 23, 59), datetime(2021, 3, 5, 0, 0),
+         datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
+         datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]

zfs_smart_snapshot/main.py

@@ -1,31 +1,8 @@
-from datetime import datetime
+from datetime import datetime, timezone
 from dateutil.relativedelta import relativedelta
-
+from pprint import pprint
-
+import subprocess
-def keep_date(now, snapshots, date_convert, delta, count):
+import sys
-    keep = set()
-    if count < 1:
-        return keep
-    start_date = date_convert(now) - delta*(count-1)
-    for snapshot in snapshots:
-        if snapshot.date() >= start_date:
-            keep.add(snapshot)
-            start_date = date_convert(snapshot) + delta
-        if start_date > now.date():
-            break
-    return keep
-
-
-def keep_year(now, snapshots, count):
-    return keep_date(now, snapshots, lambda d: d.date().replace(day=1, month=1), relativedelta(years=1), count)
-
-
-def keep_month(now, snapshots, count):
-    return keep_date(now, snapshots, lambda d: d.date().replace(day=1), relativedelta(months=1), count)
-
-
-def keep_day(now, snapshots, count):
-    return keep_date(now, snapshots, lambda d: d.date(), relativedelta(days=1), count)
 
 
 def keep_time(now, snapshots, convert, delta, count):
@@ -42,6 +19,18 @@ def keep_time(now, snapshots, convert, delta, count):
     return keep
 
 
+def keep_year(now, snapshots, count):
+    return keep_time(now, snapshots, lambda d: d.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0), relativedelta(years=1), count)
+
+
+def keep_month(now, snapshots, count):
+    return keep_time(now, snapshots, lambda d: d.replace(day=1, hour=0, minute=0, second=0, microsecond=0), relativedelta(months=1), count)
+
+
+def keep_day(now, snapshots, count):
+    return keep_time(now, snapshots, lambda d: d.replace(hour=0, minute=0, second=0, microsecond=0),  relativedelta(days=1), count)
+
+
 def keep_hour(now, snapshots, count):
     return keep_time(now, snapshots, lambda d: d.replace(minute=0, second=0, microsecond=0), relativedelta(hours=1),
                      count)
@@ -52,14 +41,98 @@ def keep_frequent(now, snapshots, count):
                      relativedelta(minutes=15), count)
 
 
-def snapshots_to_keep(now, snapshots, yearly=0, monthly=0, weekly=0, daily=0, hourly=0, frequently=0):
+def snapshots_to_keep(now, snapshots, yearly=0, monthly=0, daily=0, hourly=0, frequently=0):
     snapshots.sort()
     keep = set()
-    keep.union(keep_year(now, snapshots, yearly))
+    keep.update(keep_year(now, snapshots, yearly))
-    keep.union(keep_month(now, snapshots, monthly))
+    keep.update(keep_month(now, snapshots, monthly))
-    keep.union(keep_day(now, snapshots, daily))
+    keep.update(keep_day(now, snapshots, daily))
-    return keep
+    keep.update(keep_hour(now, snapshots, hourly))
+    keep.update(keep_frequent(now, snapshots, frequently))
+    return sorted(list(keep))
+
+
+def parse_int(fields, name, fieldname):
+    if fields[fieldname] == '-':
+        return 0
+    try:
+        return int(fields[fieldname])
+    except ValueError:
+        print(f"Warning: Could not parse value {fields[fieldname]} for fs/zvol {name} prop {fieldname}",
+              output=sys.stderr)
+        return -1
+
+
+def zfs_list(fieldnames, args):
+    proc = subprocess.run(['/usr/sbin/zfs', 'list', '-H', '-o', ','.join(['name'] + fieldnames)] + args,
+                          capture_output=True, check=True, encoding='UTF-8')
+    pprint(proc.args)
+    if not fieldnames:
+        result = list()
+    else:
+        result = dict()
+    for line in proc.stdout.splitlines():
+        fields = line.split('\t')
+        if not fieldnames:
+            result.append(fields[0])
+        else:
+            result[fields[0]] = {n: v for n, v in zip(fieldnames, fields[1:])}
+    return result
+
+
+def read_snapshot_props():
+    periods = ['yearly', 'monthly', 'daily', 'hourly', 'frequently']
+    fieldnames = ['org.blankertz:snapshot'] + ['org.blankertz:snapshot:' + period for period in periods]
+    raw_list = zfs_list(fieldnames, ['-t', 'filesystem,volume', '-r'])
+    result = dict()
+    for name, fields in raw_list.items():
+        result[name] = {'snapshot': True if fields['org.blankertz:snapshot'].lower() == 'true' else False}
+        result[name].update({period: parse_int(fields, name, 'org.blankertz:snapshot:' + period) for period in periods})
+    return result
+
+
+def read_snapshots(fs_to_snap):
+    raw_list = zfs_list([], ['-t', 'snapshot'] + fs_to_snap) if fs_to_snap else []
+    result = dict()
+    for name in raw_list:
+        fs, snap = name.split('@')
+        print(f'{fs}, {snap}')
+        if snap.startswith('zfs-smart-snap'):
+            result[fs] = result.get(fs, []) + [snap]
+    return result
+
+
+def find_latest_snapshots(existing_snaps):
+    result = dict()
+    for fs, snaps in existing_snaps.items():
+        snaps.sort()
+        result[fs] = snaps[-1]
+    return result
+
+
+def written_since_last(fs, snap):
+    proc = subprocess.run(['/usr/sbin/zfs', 'list', '-Hp', '-o', f'written@{snap}', fs],
+                          check=True, capture_output=True, encoding="UTF-8")
+    return int(proc.stdout) > 0
 
 
 def main():
-    print("Hello, world!")
+    now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
+    props = read_snapshot_props()
+    pprint(props)
+    fs_to_snap = [k for k, v in props.items() if v['snapshot']]
+    if fs_to_snap:
+        existing_snaps = read_snapshots(fs_to_snap)
+        pprint(existing_snaps)
+        latest_snaps = find_latest_snapshots(existing_snaps)
+        fs_to_really_snap = [fs for fs in fs_to_snap if fs not in latest_snaps or
+                             written_since_last(fs, latest_snaps[fs])]
+        if fs_to_really_snap:
+            snapname = 'zfs-smart-snap-' + now.strftime('%Y-%m-%d-%H%M')
+            subprocess.run(['/usr/sbin/zfs', 'snapshot'] + [f'{fs}@{snapname}' for fs in fs_to_really_snap], check=True)
+
+    # TODO: Cleanup expired snapshots
+
+
+if __name__ == "__main__":
+    main()