#!/usr/bin/env python3 """NAS-local copy job with read-only source, resumable rsync and status records. Config and file-name manifests stay on the NAS; never commit user data to Git. Existing files are retained during copying and checksum-checked afterwards. """ import fcntl import json import os from pathlib import Path import stat import subprocess import sys import time import traceback def main(): config_path = Path(sys.argv[1]).resolve(strict=True) job = config_path.parent config = json.loads(config_path.read_text()) mount = Path(config['mount']) source = mount / config['source_root'] destination = Path(config['destination']).resolve(strict=True) device = (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True) lock = (job.parent.parent / 'transfer.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight', 'planned_copy_bytes': config['planned_copy_bytes'], 'directories': config['directories']} def save(phase, **extra): state.update(phase=phase, updated=time.time(), **extra) temporary = job / 'status.tmp' temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n') temporary.replace(job / 'status.json') def run_to_log(args, log_name): with (job / log_name).open('wb') as log: return subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode save('preflight') try: properties = dict(line.split('=', 1) for line in subprocess.check_output( ['udevadm', 'info', '-q', 'property', '-n', str(device)], text=True).splitlines() if '=' in line) if properties.get('ID_SERIAL_SHORT') != config['serial'] or properties.get('ID_BUS') != 'usb': raise RuntimeError('USB device identity changed.') if not os.path.ismount(mount): raise RuntimeError('Source is not mounted.') mounted = json.loads(subprocess.check_output( ['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0] if Path(mounted['source']).resolve() != device or 'ro' not in mounted['options'].split(','): raise RuntimeError('Unexpected source mount or source is not read-only.') if subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1': raise RuntimeError('Block device is not read-only.') writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] state['source_write_counters_before'] = writes_before available = os.statvfs(destination) if available.f_bavail * available.f_frsize < config['planned_copy_bytes'] + 1024**3: raise RuntimeError('Insufficient destination free space.') existing = [] for name in config['directories']: folder = source / name if folder.is_symlink() or not folder.is_dir(): raise RuntimeError('Expected source directory missing or linked.') for base, dirs, files in os.walk(folder, followlinks=False, onerror=lambda e: (_ for _ in ()).throw(e)): for filename in files: entry = Path(base) / filename relative = entry.relative_to(source) target = destination / relative if os.path.lexists(target): existing.append(os.fsencode(relative)) (job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b'')) save('copying', existing_files_to_verify=len(existing), copy_started=time.time()) source_args = [str(source / name) for name in config['directories']] command = ['rsync', '-rltH', '--chown=1000:1001', '--ignore-existing', '--partial', '--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L', '--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/'] result = run_to_log(command, 'copy.log') if result: raise RuntimeError('rsync copy exited with status ' + str(result)) save('verifying_existing', copy_finished=time.time()) if existing: result = run_to_log(['rsync', '-rcnli', '--from0', '--files-from=' + str(job / 'existing-files.list'), '--out-format=%i %n%L', '--protect-args', str(source) + '/', str(destination) + '/'], 'existing-check.log') if result: raise RuntimeError('Existing-file checksum check failed: ' + str(result)) changes = (job / 'existing-check.log').read_text().strip() if changes: save('needs_attention', reason='Existing destination files differ; retained unchanged.', mismatch_log=str(job / 'existing-check.log')) return save('verifying_tree') result = run_to_log(['rsync', '-rltHni', '--out-format=%i %n%L', '--protect-args', *source_args, str(destination) + '/'], 'tree-check.log') if result or (job / 'tree-check.log').read_text().strip(): raise RuntimeError('Final directory comparison found differences; see tree-check.log.') save('complete', finished=time.time(), existing_files_checksum_match=True, new_files_verified_by_rsync_transfer_checksums=True) except BaseException as error: save('failed', error=str(error), finished=time.time()) traceback.print_exc() finally: try: if os.path.ismount(mount): subprocess.run(['umount', str(mount)], check=True, timeout=60) if config['original_ro'] == '0': subprocess.run(['blockdev', '--setrw', str(device)], check=True) writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] save(state['phase'], unmounted=True, source_write_counters_after=writes_after, source_writes_unchanged=writes_after == state.get('source_write_counters_before')) except BaseException as error: save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error)) fcntl.flock(lock, fcntl.LOCK_UN) if __name__ == '__main__': main()