132 行
7.4 KiB
Python
132 行
7.4 KiB
Python
#!/usr/bin/env python3
|
|
"""NAS-local copy job with read-only source, resumable rsync and status records.
|
|
|
|
Config and file-name manifests stay on the NAS; never commit user data to Git.
|
|
Existing files are retained during copying and checksum-checked afterwards.
|
|
"""
|
|
import fcntl
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
import traceback
|
|
|
|
|
|
def main():
|
|
config_path = Path(sys.argv[1]).resolve(strict=True)
|
|
job = config_path.parent
|
|
config = json.loads(config_path.read_text())
|
|
mount = Path(config['mount'])
|
|
source = mount / config['source_root']
|
|
destination = Path(config['destination']).resolve(strict=True)
|
|
device = Path(config['device_path']).resolve(strict=True) if config.get('device_path') else (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True)
|
|
managed = config.get('managed_mount', True)
|
|
os.environ['LC_ALL'] = 'C'
|
|
lock = (job.parent.parent / 'transfer.lock').open('a')
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight',
|
|
'planned_copy_bytes': config['planned_copy_bytes'], 'directories': config['directories']}
|
|
|
|
def save(phase, **extra):
|
|
state.update(phase=phase, updated=time.time(), **extra)
|
|
temporary = job / 'status.tmp'
|
|
temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n')
|
|
temporary.replace(job / 'status.json')
|
|
|
|
def run_to_log(args, log_name):
|
|
with (job / log_name).open('wb') as log:
|
|
return subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode
|
|
|
|
save('preflight')
|
|
try:
|
|
properties = dict(line.split('=', 1) for line in subprocess.check_output(
|
|
['udevadm', 'info', '-q', 'property', '-n', str(device)], text=True).splitlines() if '=' in line)
|
|
if properties.get('ID_SERIAL_SHORT') != config['serial'] or properties.get('ID_BUS') != 'usb':
|
|
raise RuntimeError('USB device identity changed.')
|
|
if not os.path.ismount(mount):
|
|
raise RuntimeError('Source is not mounted.')
|
|
mounted = json.loads(subprocess.check_output(
|
|
['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0]
|
|
if Path(mounted['source']).resolve() != device or (managed and 'ro' not in mounted['options'].split(',')):
|
|
raise RuntimeError('Unexpected source mount or source is not read-only.')
|
|
if managed and subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1':
|
|
raise RuntimeError('Block device is not read-only.')
|
|
writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
|
|
state['source_write_counters_before'] = writes_before
|
|
available = os.statvfs(destination)
|
|
if available.f_bavail * available.f_frsize < config['planned_copy_bytes'] + 1024**3:
|
|
raise RuntimeError('Insufficient destination free space.')
|
|
existing = []
|
|
for name in config['directories']:
|
|
folder = source / name
|
|
if folder.is_symlink() or not folder.is_dir():
|
|
raise RuntimeError('Expected source directory missing or linked.')
|
|
for base, dirs, files in os.walk(folder, followlinks=False, onerror=lambda e: (_ for _ in ()).throw(e)):
|
|
for filename in files:
|
|
entry = Path(base) / filename
|
|
relative = entry.relative_to(source)
|
|
target = destination / relative
|
|
if os.path.lexists(target):
|
|
existing.append(os.fsencode(relative))
|
|
(job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b''))
|
|
save('copying', existing_files_to_verify=len(existing), copy_started=time.time())
|
|
source_args = [str(source / name) for name in config['directories']]
|
|
command = ['rsync', '-rltH', '--chown=' + str(config.get('uid', 1000)) + ':' + str(config.get('gid', 1001)), '--ignore-existing', '--partial', '--no-whole-file',
|
|
'--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L',
|
|
'--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/']
|
|
result = run_to_log(command, 'copy.log')
|
|
if result:
|
|
raise RuntimeError('rsync copy exited with status ' + str(result))
|
|
save('verifying_existing', copy_finished=time.time())
|
|
if existing:
|
|
result = run_to_log(['rsync', '-rcnli', '--from0',
|
|
'--files-from=' + str(job / 'existing-files.list'),
|
|
'--out-format=%i %n%L', '--protect-args',
|
|
str(source) + '/', str(destination) + '/'], 'existing-check.log')
|
|
if result:
|
|
raise RuntimeError('Existing-file checksum check failed: ' + str(result))
|
|
changes = (job / 'existing-check.log').read_text().strip()
|
|
if changes:
|
|
save('needs_attention', reason='Existing destination files differ; retained unchanged.',
|
|
mismatch_log=str(job / 'existing-check.log'))
|
|
return
|
|
save('verifying_tree')
|
|
# Existing files keep their original metadata. Their contents were fully
|
|
# checksum-checked above, so a different mtime alone is not a copy failure.
|
|
result = run_to_log(['rsync', '-rlHni', '--size-only', '--out-format=%i %n%L', '--protect-args',
|
|
*source_args, str(destination) + '/'], 'tree-check.log')
|
|
if result or (job / 'tree-check.log').read_text().strip():
|
|
raise RuntimeError('Final directory comparison found differences; see tree-check.log.')
|
|
subprocess.run(['sync', '-f', str(destination)], check=True)
|
|
metadata_helper = Path(__file__).with_name('save_xattrs.py')
|
|
if config.get('filesystem') == 'apfs' and metadata_helper.exists():
|
|
save('saving_metadata')
|
|
metadata_target = destination / ('.dayu-diskcopy-metadata-' + job.name)
|
|
result = run_to_log(['/usr/bin/python3', str(metadata_helper), str(config_path), str(metadata_target)], 'metadata-summary.json')
|
|
if result:
|
|
raise RuntimeError('File copy passed but extended attribute archive failed.')
|
|
save('complete', finished=time.time(), existing_files_checksum_match=True,
|
|
new_files_verified_by_rsync_transfer_checksums=True)
|
|
except BaseException as error:
|
|
save('failed', error=str(error), finished=time.time())
|
|
traceback.print_exc()
|
|
finally:
|
|
try:
|
|
if managed and os.path.ismount(mount):
|
|
subprocess.run(['umount', str(mount)], check=True, timeout=60)
|
|
if managed and config['original_ro'] == '0':
|
|
subprocess.run(['blockdev', '--setrw', str(device)], check=True)
|
|
writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
|
|
save(state['phase'], unmounted=managed and not os.path.ismount(mount), source_write_counters_after=writes_after,
|
|
source_writes_unchanged=writes_after == state.get('source_write_counters_before'))
|
|
except BaseException as error:
|
|
save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error))
|
|
fcntl.flock(lock, fcntl.LOCK_UN)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|