diff --git a/HANDOFF.md b/HANDOFF.md index e05912a..fd4ed37 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,7 +4,19 @@ ## 当前目标 -基础 APFS 只读挂载已验证。下一步用普通测试文件验证素材读取及导入,再确定应用封装方案。 +基础 APFS 只读挂载已验证。当前用户要求把 4 TB APFS USB 盘的“火影秋决”和“LCQ”比赛目录复制到 NAS `/vol5/1000/剪辑盘备份`,正在 NAS 本机执行复制。 + +## 当前复制任务(2026-11-11 22:36 启动) + +- 新源盘:USB Tech,序列号 DD56419883A4F,容量 4096805658624 字节;APFS PARTUUID `d6bd791c-7bee-486b-8fef-6cf584381a55`,当前设备 sdg2。卷名 DayuMedia 4TB,未加密。 +- 只读挂载点:`/var/tmp/dayu-apfs-validation/mnt-DD56419883A4F`;源路径为该挂载点下 `root/火影秋决` 与 `root/LCQ荒野乱斗`。其他 AE/下载目录中名字含 LCQ 的辅助目录不在本次复制范围。 +- 目标目录:`/vol5/1000/剪辑盘备份`,实挂 /vol5 Btrfs,有约 3.95 TB 空闲。已有“火影秋决”约 46.8 GB。 +- 预检:7422 个条目,5881 个普通文件、1462 个目录、79 个符号链接;普通文件总逻辑大小 548102817612 字节。初步需传输 501332211263 字节,已有内容先保留不覆盖。 +- 程序:`scripts/copy-directories.py`,NAS 的独立后台进程执行 rsync,只复制新增内容,保留目录/文件名/时间/硬链接/符号链接,新增文件设为用户 1000、组 1001,不改 NAS 系统服务。 +- 任务记录:`/var/tmp/dayu-apfs-validation/jobs/copy-20261111-223613`;入口文件 `current-job` 指向当前任务。PID 1155511。`status.json` 为状态,`copy.log` 含实际传输进度。SSH 中断不终止任务。 +- 复制后校验先前已存在文件的完整校验和,并作目录差异核验;不同版本保留不覆盖,需按检查结果处理。新文件使用 rsync 传输校验,不声称进行了独立的全盘逐字节读回。 +- 结束时程序卸载 APFS 并恢复其内核只读标志,检查源分区写计数。错误/不一致记录在同目录日志。仅普通素材与工程的本地 NAS 备份,不把素材、文件名清单或用户数据提交到 Git。 +- 用户要求报告实际拷贝速度与剩余时间;持续在当前任务观察至完成。未实现飞牛 FPK 或系统文件管理集成。 ## 已确认 diff --git a/scripts/copy-directories.py b/scripts/copy-directories.py new file mode 100644 index 0000000..9c014ea --- /dev/null +++ b/scripts/copy-directories.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""NAS-local copy job with read-only source, resumable rsync and status records. + +Config and file-name manifests stay on the NAS; never commit user data to Git. +Existing files are retained during copying and checksum-checked afterwards. +""" +import fcntl +import json +import os +from pathlib import Path +import stat +import subprocess +import sys +import time +import traceback + + +def main(): + config_path = Path(sys.argv[1]).resolve(strict=True) + job = config_path.parent + config = json.loads(config_path.read_text()) + mount = Path(config['mount']) + source = mount / config['source_root'] + destination = Path(config['destination']).resolve(strict=True) + device = (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True) + lock = (job.parent.parent / 'transfer.lock').open('a') + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight', + 'planned_copy_bytes': config['planned_copy_bytes'], 'directories': config['directories']} + + def save(phase, **extra): + state.update(phase=phase, updated=time.time(), **extra) + temporary = job / 'status.tmp' + temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n') + temporary.replace(job / 'status.json') + + def run_to_log(args, log_name): + with (job / log_name).open('wb') as log: + return subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode + + save('preflight') + try: + properties = dict(line.split('=', 1) for line in subprocess.check_output( + ['udevadm', 'info', '-q', 'property', '-n', str(device)], text=True).splitlines() if '=' in line) + if properties.get('ID_SERIAL_SHORT') != config['serial'] or properties.get('ID_BUS') != 'usb': + raise RuntimeError('USB device identity changed.') + if not os.path.ismount(mount): + raise RuntimeError('Source is not mounted.') + mounted = json.loads(subprocess.check_output( + ['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0] + if Path(mounted['source']).resolve() != device or 'ro' not in mounted['options'].split(','): + raise RuntimeError('Unexpected source mount or source is not read-only.') + if subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1': + raise RuntimeError('Block device is not read-only.') + writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] + state['source_write_counters_before'] = writes_before + available = os.statvfs(destination) + if available.f_bavail * available.f_frsize < config['planned_copy_bytes'] + 1024**3: + raise RuntimeError('Insufficient destination free space.') + existing = [] + for name in config['directories']: + folder = source / name + if folder.is_symlink() or not folder.is_dir(): + raise RuntimeError('Expected source directory missing or linked.') + for base, dirs, files in os.walk(folder, followlinks=False, onerror=lambda e: (_ for _ in ()).throw(e)): + for filename in files: + entry = Path(base) / filename + relative = entry.relative_to(source) + target = destination / relative + if os.path.lexists(target): + existing.append(os.fsencode(relative)) + (job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b'')) + save('copying', existing_files_to_verify=len(existing), copy_started=time.time()) + source_args = [str(source / name) for name in config['directories']] + command = ['rsync', '-rltH', '--chown=1000:1001', '--ignore-existing', '--partial', + '--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L', + '--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/'] + result = run_to_log(command, 'copy.log') + if result: + raise RuntimeError('rsync copy exited with status ' + str(result)) + save('verifying_existing', copy_finished=time.time()) + if existing: + result = run_to_log(['rsync', '-rcnli', '--from0', + '--files-from=' + str(job / 'existing-files.list'), + '--out-format=%i %n%L', '--protect-args', + str(source) + '/', str(destination) + '/'], 'existing-check.log') + if result: + raise RuntimeError('Existing-file checksum check failed: ' + str(result)) + changes = (job / 'existing-check.log').read_text().strip() + if changes: + save('needs_attention', reason='Existing destination files differ; retained unchanged.', + mismatch_log=str(job / 'existing-check.log')) + return + save('verifying_tree') + result = run_to_log(['rsync', '-rltHni', '--out-format=%i %n%L', '--protect-args', + *source_args, str(destination) + '/'], 'tree-check.log') + if result or (job / 'tree-check.log').read_text().strip(): + raise RuntimeError('Final directory comparison found differences; see tree-check.log.') + save('complete', finished=time.time(), existing_files_checksum_match=True, + new_files_verified_by_rsync_transfer_checksums=True) + except BaseException as error: + save('failed', error=str(error), finished=time.time()) + traceback.print_exc() + finally: + try: + if os.path.ismount(mount): + subprocess.run(['umount', str(mount)], check=True, timeout=60) + if config['original_ro'] == '0': + subprocess.run(['blockdev', '--setrw', str(device)], check=True) + writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] + save(state['phase'], unmounted=True, source_write_counters_after=writes_after, + source_writes_unchanged=writes_after == state.get('source_write_counters_before')) + except BaseException as error: + save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error)) + fcntl.flock(lock, fcntl.LOCK_UN) + + +if __name__ == '__main__': + main()