Add resumable NAS-local read-only directory copy job
这个提交包含在:
1 父节点
616936fc1e
当前提交
604fce4fec
2 个文件已更改
+132
-1
没有与搜索条件相匹配的文件
+13
-1
@@ -4,7 +4,19 @@
|
|||||||
|
|
||||||
## 当前目标
|
## 当前目标
|
||||||
|
|
||||||
基础 APFS 只读挂载已验证。下一步用普通测试文件验证素材读取及导入,再确定应用封装方案。
|
基础 APFS 只读挂载已验证。当前用户要求把 4 TB APFS USB 盘的“火影秋决”和“LCQ”比赛目录复制到 NAS `/vol5/1000/剪辑盘备份`,正在 NAS 本机执行复制。
|
||||||
|
|
||||||
|
## 当前复制任务(2026-11-11 22:36 启动)
|
||||||
|
|
||||||
|
- 新源盘:USB Tech,序列号 DD56419883A4F,容量 4096805658624 字节;APFS PARTUUID `d6bd791c-7bee-486b-8fef-6cf584381a55`,当前设备 sdg2。卷名 DayuMedia 4TB,未加密。
|
||||||
|
- 只读挂载点:`/var/tmp/dayu-apfs-validation/mnt-DD56419883A4F`;源路径为该挂载点下 `root/火影秋决` 与 `root/LCQ荒野乱斗`。其他 AE/下载目录中名字含 LCQ 的辅助目录不在本次复制范围。
|
||||||
|
- 目标目录:`/vol5/1000/剪辑盘备份`,实挂 /vol5 Btrfs,有约 3.95 TB 空闲。已有“火影秋决”约 46.8 GB。
|
||||||
|
- 预检:7422 个条目,5881 个普通文件、1462 个目录、79 个符号链接;普通文件总逻辑大小 548102817612 字节。初步需传输 501332211263 字节,已有内容先保留不覆盖。
|
||||||
|
- 程序:`scripts/copy-directories.py`,NAS 的独立后台进程执行 rsync,只复制新增内容,保留目录/文件名/时间/硬链接/符号链接,新增文件设为用户 1000、组 1001,不改 NAS 系统服务。
|
||||||
|
- 任务记录:`/var/tmp/dayu-apfs-validation/jobs/copy-20261111-223613`;入口文件 `current-job` 指向当前任务。PID 1155511。`status.json` 为状态,`copy.log` 含实际传输进度。SSH 中断不终止任务。
|
||||||
|
- 复制后校验先前已存在文件的完整校验和,并作目录差异核验;不同版本保留不覆盖,需按检查结果处理。新文件使用 rsync 传输校验,不声称进行了独立的全盘逐字节读回。
|
||||||
|
- 结束时程序卸载 APFS 并恢复其内核只读标志,检查源分区写计数。错误/不一致记录在同目录日志。仅普通素材与工程的本地 NAS 备份,不把素材、文件名清单或用户数据提交到 Git。
|
||||||
|
- 用户要求报告实际拷贝速度与剩余时间;持续在当前任务观察至完成。未实现飞牛 FPK 或系统文件管理集成。
|
||||||
|
|
||||||
## 已确认
|
## 已确认
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""NAS-local copy job with read-only source, resumable rsync and status records.
|
||||||
|
|
||||||
|
Config and file-name manifests stay on the NAS; never commit user data to Git.
|
||||||
|
Existing files are retained during copying and checksum-checked afterwards.
|
||||||
|
"""
|
||||||
|
import fcntl
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
config_path = Path(sys.argv[1]).resolve(strict=True)
|
||||||
|
job = config_path.parent
|
||||||
|
config = json.loads(config_path.read_text())
|
||||||
|
mount = Path(config['mount'])
|
||||||
|
source = mount / config['source_root']
|
||||||
|
destination = Path(config['destination']).resolve(strict=True)
|
||||||
|
device = (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True)
|
||||||
|
lock = (job.parent.parent / 'transfer.lock').open('a')
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight',
|
||||||
|
'planned_copy_bytes': config['planned_copy_bytes'], 'directories': config['directories']}
|
||||||
|
|
||||||
|
def save(phase, **extra):
|
||||||
|
state.update(phase=phase, updated=time.time(), **extra)
|
||||||
|
temporary = job / 'status.tmp'
|
||||||
|
temporary.write_text(json.dumps(state, ensure_ascii=False, indent=2) + '\n')
|
||||||
|
temporary.replace(job / 'status.json')
|
||||||
|
|
||||||
|
def run_to_log(args, log_name):
|
||||||
|
with (job / log_name).open('wb') as log:
|
||||||
|
return subprocess.run(args, stdout=log, stderr=subprocess.STDOUT).returncode
|
||||||
|
|
||||||
|
save('preflight')
|
||||||
|
try:
|
||||||
|
properties = dict(line.split('=', 1) for line in subprocess.check_output(
|
||||||
|
['udevadm', 'info', '-q', 'property', '-n', str(device)], text=True).splitlines() if '=' in line)
|
||||||
|
if properties.get('ID_SERIAL_SHORT') != config['serial'] or properties.get('ID_BUS') != 'usb':
|
||||||
|
raise RuntimeError('USB device identity changed.')
|
||||||
|
if not os.path.ismount(mount):
|
||||||
|
raise RuntimeError('Source is not mounted.')
|
||||||
|
mounted = json.loads(subprocess.check_output(
|
||||||
|
['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0]
|
||||||
|
if Path(mounted['source']).resolve() != device or 'ro' not in mounted['options'].split(','):
|
||||||
|
raise RuntimeError('Unexpected source mount or source is not read-only.')
|
||||||
|
if subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1':
|
||||||
|
raise RuntimeError('Block device is not read-only.')
|
||||||
|
writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
|
||||||
|
state['source_write_counters_before'] = writes_before
|
||||||
|
available = os.statvfs(destination)
|
||||||
|
if available.f_bavail * available.f_frsize < config['planned_copy_bytes'] + 1024**3:
|
||||||
|
raise RuntimeError('Insufficient destination free space.')
|
||||||
|
existing = []
|
||||||
|
for name in config['directories']:
|
||||||
|
folder = source / name
|
||||||
|
if folder.is_symlink() or not folder.is_dir():
|
||||||
|
raise RuntimeError('Expected source directory missing or linked.')
|
||||||
|
for base, dirs, files in os.walk(folder, followlinks=False, onerror=lambda e: (_ for _ in ()).throw(e)):
|
||||||
|
for filename in files:
|
||||||
|
entry = Path(base) / filename
|
||||||
|
relative = entry.relative_to(source)
|
||||||
|
target = destination / relative
|
||||||
|
if os.path.lexists(target):
|
||||||
|
existing.append(os.fsencode(relative))
|
||||||
|
(job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b''))
|
||||||
|
save('copying', existing_files_to_verify=len(existing), copy_started=time.time())
|
||||||
|
source_args = [str(source / name) for name in config['directories']]
|
||||||
|
command = ['rsync', '-rltH', '--chown=1000:1001', '--ignore-existing', '--partial',
|
||||||
|
'--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L',
|
||||||
|
'--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/']
|
||||||
|
result = run_to_log(command, 'copy.log')
|
||||||
|
if result:
|
||||||
|
raise RuntimeError('rsync copy exited with status ' + str(result))
|
||||||
|
save('verifying_existing', copy_finished=time.time())
|
||||||
|
if existing:
|
||||||
|
result = run_to_log(['rsync', '-rcnli', '--from0',
|
||||||
|
'--files-from=' + str(job / 'existing-files.list'),
|
||||||
|
'--out-format=%i %n%L', '--protect-args',
|
||||||
|
str(source) + '/', str(destination) + '/'], 'existing-check.log')
|
||||||
|
if result:
|
||||||
|
raise RuntimeError('Existing-file checksum check failed: ' + str(result))
|
||||||
|
changes = (job / 'existing-check.log').read_text().strip()
|
||||||
|
if changes:
|
||||||
|
save('needs_attention', reason='Existing destination files differ; retained unchanged.',
|
||||||
|
mismatch_log=str(job / 'existing-check.log'))
|
||||||
|
return
|
||||||
|
save('verifying_tree')
|
||||||
|
result = run_to_log(['rsync', '-rltHni', '--out-format=%i %n%L', '--protect-args',
|
||||||
|
*source_args, str(destination) + '/'], 'tree-check.log')
|
||||||
|
if result or (job / 'tree-check.log').read_text().strip():
|
||||||
|
raise RuntimeError('Final directory comparison found differences; see tree-check.log.')
|
||||||
|
save('complete', finished=time.time(), existing_files_checksum_match=True,
|
||||||
|
new_files_verified_by_rsync_transfer_checksums=True)
|
||||||
|
except BaseException as error:
|
||||||
|
save('failed', error=str(error), finished=time.time())
|
||||||
|
traceback.print_exc()
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
if os.path.ismount(mount):
|
||||||
|
subprocess.run(['umount', str(mount)], check=True, timeout=60)
|
||||||
|
if config['original_ro'] == '0':
|
||||||
|
subprocess.run(['blockdev', '--setrw', str(device)], check=True)
|
||||||
|
writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2]
|
||||||
|
save(state['phase'], unmounted=True, source_write_counters_after=writes_after,
|
||||||
|
source_writes_unchanged=writes_after == state.get('source_write_counters_before'))
|
||||||
|
except BaseException as error:
|
||||||
|
save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error))
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
在新工单中引用
屏蔽一个用户