Add resumable NAS-local read-only directory copy job

这个提交包含在:
mac 于 2026-11-11 22:37:42 +08:00 提交
1 父节点 616936fc1e
当前提交 604fce4fec
2 个文件已更改
+132 -1

没有与搜索条件相匹配的文件

+13 -1
查看文件
@@ -4,7 +4,19 @@
## 当前目标
基础 APFS 只读挂载已验证。下一步用普通测试文件验证素材读取及导入,再确定应用封装方案。
基础 APFS 只读挂载已验证。当前用户要求把 4 TB APFS USB 盘的“火影秋决”和“LCQ”比赛目录复制到 NAS `/vol5/1000/剪辑盘备份`,正在 NAS 本机执行复制。
## 当前复制任务(2026-11-11 22:36 启动)
- 新源盘:USB Tech,序列号 DD56419883A4F,容量 4096805658624 字节;APFS PARTUUID `d6bd791c-7bee-486b-8fef-6cf584381a55`,当前设备 sdg2。卷名 DayuMedia 4TB,未加密。
- 只读挂载点:`/var/tmp/dayu-apfs-validation/mnt-DD56419883A4F`;源路径为该挂载点下 `root/火影秋决` 与 `root/LCQ荒野乱斗`。其他 AE/下载目录中名字含 LCQ 的辅助目录不在本次复制范围。
- 目标目录:`/vol5/1000/剪辑盘备份`,实挂 /vol5 Btrfs,有约 3.95 TB 空闲。已有“火影秋决”约 46.8 GB。
- 预检:7422 个条目,5881 个普通文件、1462 个目录、79 个符号链接;普通文件总逻辑大小 548102817612 字节。初步需传输 501332211263 字节,已有内容先保留不覆盖。
- 程序:`scripts/copy-directories.py`,NAS 的独立后台进程执行 rsync,只复制新增内容,保留目录/文件名/时间/硬链接/符号链接,新增文件设为用户 1000、组 1001,不改 NAS 系统服务。
- 任务记录:`/var/tmp/dayu-apfs-validation/jobs/copy-20261111-223613`;入口文件 `current-job` 指向当前任务。PID 1155511。`status.json` 为状态,`copy.log` 含实际传输进度。SSH 中断不终止任务。
- 复制后校验先前已存在文件的完整校验和,并作目录差异核验;不同版本保留不覆盖,需按检查结果处理。新文件使用 rsync 传输校验,不声称进行了独立的全盘逐字节读回。
- 结束时程序卸载 APFS 并恢复其内核只读标志,检查源分区写计数。错误/不一致记录在同目录日志。仅普通素材与工程的本地 NAS 备份,不把素材、文件名清单或用户数据提交到 Git。
- 用户要求报告实际拷贝速度与剩余时间;持续在当前任务观察至完成。未实现飞牛 FPK 或系统文件管理集成。
## 已确认
+119
查看文件
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""NAS-local copy job with read-only source, resumable rsync and status records.
Config and file-name manifests stay on the NAS; never commit user data to Git.
Existing files are retained during copying and checksum-checked afterwards.
"""
import fcntl
import json
import os
from pathlib import Path
import stat
import subprocess
import sys
import time
import traceback
def main():
config_path = Path(sys.argv[1]).resolve(strict=True)
job = config_path.parent
config = json.loads(config_path.read_text())
mount = Path(config['mount'])
source = mount / config['source_root']
destination = Path(config['destination']).resolve(strict=True)
device = (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True)
lock = (job.parent.parent / 'transfer.lock').open('a')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight',
'planned_copy_bytes': config['planned_copy_bytes'], 'directories': config['directories']}
def save(phase, **extra):
state.update(phase=phase, updated=time.time(), **extra)
temporary = job / 'status.tmp'
temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n')
temporary.replace(job / 'status.json')
def run_to_log(args, log_name):
with (job / log_name).open('wb') as log:
return subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode
save('preflight')
try:
properties = dict(line.split('=', 1) for line in subprocess.check_output(
['udevadm', 'info', '-q', 'property', '-n', str(device)], text=True).splitlines() if '=' in line)
if properties.get('ID_SERIAL_SHORT') != config['serial'] or properties.get('ID_BUS') != 'usb':
raise RuntimeError('USB device identity changed.')
if not os.path.ismount(mount):
raise RuntimeError('Source is not mounted.')
mounted = json.loads(subprocess.check_output(
['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0]
if Path(mounted['source']).resolve() != device or 'ro' not in mounted['options'].split(','):
raise RuntimeError('Unexpected source mount or source is not read-only.')
if subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1':
raise RuntimeError('Block device is not read-only.')
writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
state['source_write_counters_before'] = writes_before
available = os.statvfs(destination)
if available.f_bavail * available.f_frsize < config['planned_copy_bytes'] + 1024**3:
raise RuntimeError('Insufficient destination free space.')
existing = []
for name in config['directories']:
folder = source / name
if folder.is_symlink() or not folder.is_dir():
raise RuntimeError('Expected source directory missing or linked.')
for base, dirs, files in os.walk(folder, followlinks=False, onerror=lambda e: (_ for _ in ()).throw(e)):
for filename in files:
entry = Path(base) / filename
relative = entry.relative_to(source)
target = destination / relative
if os.path.lexists(target):
existing.append(os.fsencode(relative))
(job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b''))
save('copying', existing_files_to_verify=len(existing), copy_started=time.time())
source_args = [str(source / name) for name in config['directories']]
command = ['rsync', '-rltH', '--chown=1000:1001', '--ignore-existing', '--partial',
'--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L',
'--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/']
result = run_to_log(command, 'copy.log')
if result:
raise RuntimeError('rsync copy exited with status ' + str(result))
save('verifying_existing', copy_finished=time.time())
if existing:
result = run_to_log(['rsync', '-rcnli', '--from0',
'--files-from=' + str(job / 'existing-files.list'),
'--out-format=%i %n%L', '--protect-args',
str(source) + '/', str(destination) + '/'], 'existing-check.log')
if result:
raise RuntimeError('Existing-file checksum check failed: ' + str(result))
changes = (job / 'existing-check.log').read_text().strip()
if changes:
save('needs_attention', reason='Existing destination files differ; retained unchanged.',
mismatch_log=str(job / 'existing-check.log'))
return
save('verifying_tree')
result = run_to_log(['rsync', '-rltHni', '--out-format=%i %n%L', '--protect-args',
*source_args, str(destination) + '/'], 'tree-check.log')
if result or (job / 'tree-check.log').read_text().strip():
raise RuntimeError('Final directory comparison found differences; see tree-check.log.')
save('complete', finished=time.time(), existing_files_checksum_match=True,
new_files_verified_by_rsync_transfer_checksums=True)
except BaseException as error:
save('failed', error=str(error), finished=time.time())
traceback.print_exc()
finally:
try:
if os.path.ismount(mount):
subprocess.run(['umount', str(mount)], check=True, timeout=60)
if config['original_ro'] == '0':
subprocess.run(['blockdev', '--setrw', str(device)], check=True)
writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
save(state['phase'], unmounted=True, source_write_counters_after=writes_after,
source_writes_unchanged=writes_after == state.get('source_write_counters_before'))
except BaseException as error:
save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error))
fcntl.flock(lock, fcntl.LOCK_UN)
if __name__ == '__main__':
main()