matthias/zfs-smart-snapshot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement snapshotting when changed

Browse Source
This commit is contained in:
Matthias Blankertz
2021-09-29 23:45:01 +02:00
parent b8edec0076
commit 0e1d8aeb8b
2 changed files with 126 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
tests/test_keep.py
View File

@@ -1,15 +1,17 @@
from datetime import datetime
from zfs_smart_snapshot.main import keep_year, keep_month, keep_day, keep_hour, keep_frequent
from zfs_smart_snapshot.main import keep_year, keep_month, keep_day, keep_hour, keep_frequent, snapshots_to_keep
dates = [datetime(2018, 2, 3), datetime(2019, 3, 4), datetime(2020, 4, 5), datetime(2021, 1, 2),
datetime(2021, 2, 2), datetime(2021, 2, 3), datetime(2021, 3, 3), datetime(2021, 3, 4),
datetime(2021, 3, 5)]
dates = [datetime(2018, 2, 3, 3, 4), datetime(2019, 3, 4, 4, 5), datetime(2020, 4, 5, 3, 4),
datetime(2021, 1, 2, 3, 4), datetime(2021, 2, 2, 12, 59), datetime(2021, 2, 3, 1, 0), datetime(2021, 3, 3, 1, 0), datetime(2021, 3, 4, 23, 59),
datetime(2021, 3, 5, 0, 0), datetime(2021, 3, 5, 17, 0, 0), datetime(2021, 3, 5, 18, 0, 0), datetime(2021, 3, 5, 19, 0, 0),
datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]
now = datetime(2021, 3, 5, 22, 45, 0)
def test_keep_year():
assert keep_year(now, dates, 2) == {datetime(2020, 4, 5), datetime(2021, 1, 2)}
assert keep_year(now, dates, 2) == {datetime(2020, 4, 5, 3, 4), datetime(2021, 1, 2, 3, 4)}
assert keep_year(now, dates, 0) == set()
dates_gap = [datetime(2018, 2, 3), datetime(2019, 3, 4), datetime(2021, 5, 6)]
assert keep_year(datetime(2021, 5, 6), dates_gap, 3) == {datetime(2019, 3, 4), datetime(2021, 5, 6)}
@@ -17,23 +19,27 @@ def test_keep_year():
def test_keep_month():
assert keep_month(now, dates, 3) == {datetime(2021, 1, 2), datetime(2021, 2, 2), datetime(2021, 3, 3)}
assert keep_month(now, dates, 3) == {datetime(2021, 1, 2, 3, 4), datetime(2021, 2, 2, 12, 59), datetime(2021, 3, 3, 1, 0)}
def test_keep_day():
assert keep_day(now, dates, 3) == {datetime(2021, 3, 3), datetime(2021, 3, 4), datetime(2021, 3, 5)}
datetimes = [datetime(2021, 3, 5, 17, 0, 0), datetime(2021, 3, 5, 18, 0, 0), datetime(2021, 3, 5, 19, 0, 0),
datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]
assert keep_day(now, dates, 3) == {datetime(2021, 3, 3, 1, 0), datetime(2021, 3, 4, 23, 59), datetime(2021, 3, 5, 0, 0)}
def test_keep_hour():
assert keep_hour(now, datetimes, 3) == {datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0),
assert keep_hour(now, dates, 3) == {datetime(2021, 3, 5, 20, 0, 0), datetime(2021, 3, 5, 21, 0, 0),
datetime(2021, 3, 5, 22, 0, 0)}
def test_keep_frequent():
assert keep_frequent(now, datetimes, 3) == {datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0),
assert keep_frequent(now, dates, 3) == {datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0),
datetime(2021, 3, 5, 22, 45, 0)}
def test_keep():
assert snapshots_to_keep(now, dates, yearly=2, monthly=2, daily=3, hourly=2, frequently=4) == \
[datetime(2020, 4, 5, 3, 4), datetime(2021, 1, 2, 3, 4),
datetime(2021, 2, 2, 12, 59), datetime(2021, 3, 3, 1, 0),
datetime(2021, 3, 4, 23, 59), datetime(2021, 3, 5, 0, 0),
datetime(2021, 3, 5, 21, 0, 0), datetime(2021, 3, 5, 22, 0, 0),
datetime(2021, 3, 5, 22, 15, 0), datetime(2021, 3, 5, 22, 30, 0), datetime(2021, 3, 5, 22, 45, 0)]

139
zfs_smart_snapshot/main.py
View File

@@ -1,31 +1,8 @@
from datetime import datetime
from datetime import datetime, timezone
from dateutil.relativedelta import relativedelta
def keep_date(now, snapshots, date_convert, delta, count):
keep = set()
if count < 1:
return keep
start_date = date_convert(now) - delta*(count-1)
for snapshot in snapshots:
if snapshot.date() >= start_date:
keep.add(snapshot)
start_date = date_convert(snapshot) + delta
if start_date > now.date():
break
return keep
def keep_year(now, snapshots, count):
return keep_date(now, snapshots, lambda d: d.date().replace(day=1, month=1), relativedelta(years=1), count)
def keep_month(now, snapshots, count):
return keep_date(now, snapshots, lambda d: d.date().replace(day=1), relativedelta(months=1), count)
def keep_day(now, snapshots, count):
return keep_date(now, snapshots, lambda d: d.date(), relativedelta(days=1), count)
from pprint import pprint
import subprocess
import sys
def keep_time(now, snapshots, convert, delta, count):
@@ -42,6 +19,18 @@ def keep_time(now, snapshots, convert, delta, count):
return keep
def keep_year(now, snapshots, count):
return keep_time(now, snapshots, lambda d: d.replace(month=1, day=1, hour=0, minute=0, second=0, microsecond=0), relativedelta(years=1), count)
def keep_month(now, snapshots, count):
return keep_time(now, snapshots, lambda d: d.replace(day=1, hour=0, minute=0, second=0, microsecond=0), relativedelta(months=1), count)
def keep_day(now, snapshots, count):
return keep_time(now, snapshots, lambda d: d.replace(hour=0, minute=0, second=0, microsecond=0), relativedelta(days=1), count)
def keep_hour(now, snapshots, count):
return keep_time(now, snapshots, lambda d: d.replace(minute=0, second=0, microsecond=0), relativedelta(hours=1),
count)
@@ -52,14 +41,98 @@ def keep_frequent(now, snapshots, count):
relativedelta(minutes=15), count)
def snapshots_to_keep(now, snapshots, yearly=0, monthly=0, weekly=0, daily=0, hourly=0, frequently=0):
def snapshots_to_keep(now, snapshots, yearly=0, monthly=0, daily=0, hourly=0, frequently=0):
snapshots.sort()
keep = set()
keep.union(keep_year(now, snapshots, yearly))
keep.union(keep_month(now, snapshots, monthly))
keep.union(keep_day(now, snapshots, daily))
return keep
keep.update(keep_year(now, snapshots, yearly))
keep.update(keep_month(now, snapshots, monthly))
keep.update(keep_day(now, snapshots, daily))
keep.update(keep_hour(now, snapshots, hourly))
keep.update(keep_frequent(now, snapshots, frequently))
return sorted(list(keep))
def parse_int(fields, name, fieldname):
if fields[fieldname] == '-':
return 0
try:
return int(fields[fieldname])
except ValueError:
print(f"Warning: Could not parse value {fields[fieldname]} for fs/zvol {name} prop {fieldname}",
output=sys.stderr)
return -1
def zfs_list(fieldnames, args):
proc = subprocess.run(['/usr/sbin/zfs', 'list', '-H', '-o', ','.join(['name'] + fieldnames)] + args,
capture_output=True, check=True, encoding='UTF-8')
pprint(proc.args)
if not fieldnames:
result = list()
else:
result = dict()
for line in proc.stdout.splitlines():
fields = line.split('\t')
if not fieldnames:
result.append(fields[0])
else:
result[fields[0]] = {n: v for n, v in zip(fieldnames, fields[1:])}
return result
def read_snapshot_props():
periods = ['yearly', 'monthly', 'daily', 'hourly', 'frequently']
fieldnames = ['org.blankertz:snapshot'] + ['org.blankertz:snapshot:' + period for period in periods]
raw_list = zfs_list(fieldnames, ['-t', 'filesystem,volume', '-r'])
result = dict()
for name, fields in raw_list.items():
result[name] = {'snapshot': True if fields['org.blankertz:snapshot'].lower() == 'true' else False}
result[name].update({period: parse_int(fields, name, 'org.blankertz:snapshot:' + period) for period in periods})
return result
def read_snapshots(fs_to_snap):
raw_list = zfs_list([], ['-t', 'snapshot'] + fs_to_snap) if fs_to_snap else []
result = dict()
for name in raw_list:
fs, snap = name.split('@')
print(f'{fs}, {snap}')
if snap.startswith('zfs-smart-snap'):
result[fs] = result.get(fs, []) + [snap]
return result
def find_latest_snapshots(existing_snaps):
result = dict()
for fs, snaps in existing_snaps.items():
snaps.sort()
result[fs] = snaps[-1]
return result
def written_since_last(fs, snap):
proc = subprocess.run(['/usr/sbin/zfs', 'list', '-Hp', '-o', f'written@{snap}', fs],
check=True, capture_output=True, encoding="UTF-8")
return int(proc.stdout) > 0
def main():
print("Hello, world!")
now = datetime.now(timezone.utc).replace(second=0, microsecond=0)
props = read_snapshot_props()
pprint(props)
fs_to_snap = [k for k, v in props.items() if v['snapshot']]
if fs_to_snap:
existing_snaps = read_snapshots(fs_to_snap)
pprint(existing_snaps)
latest_snaps = find_latest_snapshots(existing_snaps)
fs_to_really_snap = [fs for fs in fs_to_snap if fs not in latest_snaps or
written_since_last(fs, latest_snaps[fs])]
if fs_to_really_snap:
snapname = 'zfs-smart-snap-' + now.strftime('%Y-%m-%d-%H%M')
subprocess.run(['/usr/sbin/zfs', 'snapshot'] + [f'{fs}@{snapname}' for fs in fs_to_really_snap], check=True)
# TODO: Cleanup expired snapshots
if __name__ == "__main__":
main()