From 58769c431fd1a0d1fbf7850ac256e6368f8e3488 Mon Sep 17 00:00:00 2001 From: mac Date: Wed, 11 Nov 2026 23:07:17 +0800 Subject: [PATCH] Build native fnOS disk-copy application with APFS and NTFS support --- HANDOFF.md | 8 +- application/broker.py | 313 +++++++++++++++++++++++++++ application/front.py | 86 ++++++++ application/index.html | 33 +++ application/service.py | 58 +++++ packaging/fpk/app/ui/config | 1 + packaging/fpk/cmd/config_callback | 3 + packaging/fpk/cmd/config_init | 3 + packaging/fpk/cmd/install_callback | 3 + packaging/fpk/cmd/install_init | 9 + packaging/fpk/cmd/main | 3 + packaging/fpk/cmd/uninstall_callback | 3 + packaging/fpk/cmd/uninstall_init | 6 + packaging/fpk/cmd/upgrade_callback | 3 + packaging/fpk/cmd/upgrade_init | 3 + packaging/fpk/config/privilege | 1 + packaging/fpk/config/resource | 1 + packaging/fpk/manifest | 16 ++ scripts/build-fpk.py | 80 +++++++ scripts/copy-directories.py | 24 +- tests/test_application.py | 85 ++++++++ 21 files changed, 734 insertions(+), 8 deletions(-) create mode 100644 application/broker.py create mode 100644 application/front.py create mode 100644 application/index.html create mode 100644 application/service.py create mode 100644 packaging/fpk/app/ui/config create mode 100644 packaging/fpk/cmd/config_callback create mode 100644 packaging/fpk/cmd/config_init create mode 100644 packaging/fpk/cmd/install_callback create mode 100644 packaging/fpk/cmd/install_init create mode 100644 packaging/fpk/cmd/main create mode 100644 packaging/fpk/cmd/uninstall_callback create mode 100644 packaging/fpk/cmd/uninstall_init create mode 100644 packaging/fpk/cmd/upgrade_callback create mode 100644 packaging/fpk/cmd/upgrade_init create mode 100644 packaging/fpk/config/privilege create mode 100644 packaging/fpk/config/resource create mode 100644 packaging/fpk/manifest create mode 100644 scripts/build-fpk.py create mode 100644 tests/test_application.py diff --git a/HANDOFF.md b/HANDOFF.md index f2464c9..e86ee89 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,7 +4,13 @@ ## 当前目标 -基础 APFS 只读挂载已验证。当前用户要求把 4 TB APFS USB 盘的“火影秋决”和“LCQ”比赛目录复制到 NAS `/vol5/1000/剪辑盘备份`,正在 NAS 本机执行复制。 +当前并行目标:完成已授权的两目录 NAS 本机复制;开发可安装的飞牛桌面应用“大鱼硬盘拷贝”,支持 APFS 与 NTFS 的识别、选目录和后台复制。 + +- 用户已要求停止逐条报告拷贝速度,保留内部监控,仅在完成或异常时告知。 +- 新应用沿用当前仓库与驱动,固定 appname 为 `dayu-diskcopy`、唯一安装入口 `/var/apps/dayu-diskcopy`。使用飞牛登录态网关,普通 Web 进程降权运行,设备操作由本机受限服务执行。 +- NAS 已核实为 fnOS `1.2.0701`,有 ntfs-3g、mkntfs、ntfscp。NTFS 已有系统挂载时优先复用,应用只读取源文件;不强行卸载或修改系统在用盘。未挂载的新盘使用只读挂载。 +- 应用第一版代码位于 `application/`,FPK 定义在 `packaging/fpk/`,构建入口 `scripts/build-fpk.py`。当前版本 0.1.0;使用官方网关 Unix Socket,不开放新的网络端口。Web 进程启动后降为专用包用户,原始设备操作由本机 Socket 服务处理并核验调用 UID。 +- 已通过五项边界测试:匿名及非管理员拒绝、管理员页面可读、写请求来源校验、路径穿越和符号链接越界拒绝、系统身份拒绝;Python/生命周期脚本语法通过。FPK 构建和实机界面/NTFS 镜像验证正在进行,尚未宣称正式交付。 ## 当前复制任务(2026-11-11 22:36 启动) diff --git a/application/broker.py b/application/broker.py new file mode 100644 index 0000000..891ce3b --- /dev/null +++ b/application/broker.py @@ -0,0 +1,313 @@ +"""Privileged local-only disk operations; no TCP listener and no shell execution.""" +import fcntl +import grp +import json +import os +from pathlib import Path +import pwd +import re +import signal +import socket +import socketserver +import stat +import struct +import subprocess +import time +import uuid + +ROOT = Path(__file__).resolve().parent +VAR = Path(os.environ['TRIM_PKGVAR']) +APP_UID = int(os.environ['TRIM_UID']) +APP_GID = int(os.environ['TRIM_GID']) +LEGACY = Path('/var/tmp/dayu-apfs-validation') +TERMINAL = {'complete', 'failed', 'needs_attention', 'cleanup_failed', 'cancelled'} + + +def command(args, timeout=30): + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, env={**os.environ, 'LC_ALL': 'C'}) + if result.returncode: + raise ValueError((result.stderr or result.stdout).strip()[-600:] or '操作失败') + return result.stdout.strip() + + +def beneath(path, root, must_exist=True): + path = Path(path).resolve(strict=must_exist) + root = Path(root).resolve(strict=True) + if not path.is_relative_to(root): + raise ValueError('所选路径超出可访问目录') + return path + + +def atomic(path, data): + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2)) + temporary.replace(path) + + +def disks(): + tree = json.loads(command(['lsblk', '-J', '-b', '-o', + 'NAME,PATH,TYPE,SIZE,FSTYPE,UUID,PARTUUID,TRAN,MODEL,SERIAL,MOUNTPOINTS'])) + result = [] + for disk in tree['blockdevices']: + if disk.get('tran') != 'usb' or not disk.get('size'): + continue + for part in [disk, *disk.get('children', [])]: + filesystem = (part.get('fstype') or '').lower() + if filesystem not in ('apfs', 'ntfs', 'ntfs3'): + continue + identity = part.get('partuuid') or part.get('uuid') + if not identity or not disk.get('serial'): + continue + result.append({'id': identity, 'partuuid': part.get('partuuid'), + 'uuid': part.get('uuid'), 'device': part['path'], 'filesystem': filesystem, + 'serial': disk['serial'], 'model': (disk.get('model') or 'USB 硬盘').strip(), + 'size': part['size'], 'mountpoints': [x for x in part.get('mountpoints', []) if x]}) + return result + + +def mounts(): + path = VAR / 'mounts.json' + return json.loads(path.read_text()) if path.exists() else {} + + +def jobs(uid=None): + result = [] + for parent, legacy in ((VAR / 'jobs', False), (LEGACY / 'jobs', True)): + if not parent.exists(): + continue + for folder in sorted(parent.iterdir(), reverse=True): + try: + config = json.loads((folder / 'config.json').read_text()) + state = json.loads((folder / 'status.json').read_text()) + owner = config.get('uid', 1000) + if uid is not None and uid != owner: + continue + data = (folder / 'copy.log').read_bytes()[-16000:].decode(errors='replace') + progress = list(re.finditer(r'([\d,]+)\s+(\d+)%\s+([\d.]+)([kMGT]?)B/s\s+', data)) + copied = int(progress[-1][1].replace(',', '')) if progress else 0 + phase = state['phase'] + paused = (folder / 'paused').exists() + if paused: + phase = 'paused' + alive = Path('/proc/' + str(state.get('pid', -1))).exists() + if not alive and phase not in TERMINAL and time.time() - state.get('started', 0) > 10: + phase = 'interrupted' + if phase == 'complete' and 'source_write_counters_after' not in state: + phase = 'finalizing' + speed = 0 + if progress and phase == 'copying': + multiplier = 1024 ** ('kMGT'.find(progress[-1][4]) + 1) if progress[-1][4] else 1 + speed = float(progress[-1][3]) * multiplier + total = config.get('planned_copy_bytes', 0) + if phase == 'complete': + copied = total + result.append({'id': ('legacy:' if legacy else '') + folder.name, + 'directories': config['directories'], 'destination': config['destination'], + 'phase': phase, 'bytes': copied, 'total': total, 'speed': speed, + 'eta': max(0, total - copied) / speed if speed else None, + 'started': state.get('started'), 'finished': state.get('finished'), + 'error': state.get('error') or state.get('reason') or state.get('cleanup_error'), + 'device_id': config.get('partuuid') or config.get('fs_uuid'), + 'legacy': legacy, 'uid': owner, '_folder': str(folder), '_config': config, + 'source_writes_unchanged': state.get('source_writes_unchanged')}) + except (OSError, ValueError, KeyError): + continue + return sorted(result, key=lambda item: item.get('started') or 0, reverse=True) + + +def active(): + return [job for job in jobs() if job['phase'] not in TERMINAL and job['phase'] != 'interrupted'] + + +def mount_disk(identity): + choices = [item for item in disks() if item['id'] == identity] + if len(choices) != 1: + raise ValueError('硬盘已移除或身份发生变化,请刷新') + disk = choices[0] + # Adopt the current authorized copy without remounting or moving its data. + for job in active(): + config = job['_config'] + if job['device_id'] == identity and os.path.ismount(config['mount']): + return {**disk, 'mount': config['mount'], 'base': str(Path(config['mount']) / config['source_root']), + 'managed': False, 'busy': True} + known = mounts() + if identity in known and os.path.ismount(known[identity]['mount']): + return {**disk, **known[identity]} + if disk['filesystem'].startswith('ntfs') and disk['mountpoints']: + mount = Path(disk['mountpoints'][0]) + return {**disk, 'mount': str(mount), 'base': str(mount), 'managed': False, 'busy': False} + if disk['mountpoints']: + raise ValueError('该硬盘已被其他程序挂载,请先停止使用') + device = Path(disk['device']) + target = VAR / 'mounts' / re.sub(r'[^a-zA-Z0-9_-]', '_', identity) + target.mkdir(parents=True, exist_ok=True) + if any(target.iterdir()): + raise ValueError('挂载目录不为空,已停止以保护数据') + original = command(['blockdev', '--getro', str(device)]) + command(['blockdev', '--setro', str(device)]) + try: + if disk['filesystem'] == 'apfs': + command([str(ROOT / 'bin/apfs-fuse'), '-o', 'ro,nodev,nosuid,noexec', str(device), str(target)]) + else: + command(['ntfs-3g', '-o', 'ro,nodev,nosuid,noexec', str(device), str(target)]) + if not os.path.ismount(target): + raise ValueError('无法只读挂载;加密卷或此文件系统特性可能暂不支持') + except BaseException: + if os.path.ismount(target): + command(['umount', str(target)]) + if original == '0': + command(['blockdev', '--setrw', str(device)]) + raise + base = target / 'root' if (target / 'root').is_dir() and (target / 'private-dir').is_dir() else target + known[identity] = {'mount': str(target), 'base': str(base), 'managed': True, + 'device': str(device), 'original_ro': original, 'serial': disk['serial']} + atomic(VAR / 'mounts.json', known) + return {**disk, **known[identity]} + + +def destination_roots(uid): + return [path for path in Path('/').glob('vol[0-9]*') + if (path / str(uid)).is_dir() and os.path.ismount(path)] + + +def target_path(path, uid, must_exist=True): + resolved = Path(path).resolve(strict=must_exist) + if not any(resolved.is_relative_to(root / str(uid)) for root in destination_roots(uid)): + raise ValueError('请选择当前用户存储空间中的目标目录') + return resolved + + +def listing(root, relative=''): + path = beneath(root / relative, root) + entries = [] + for entry in path.iterdir(): + if entry.name.startswith('.') or entry.is_symlink() or not entry.is_dir(): + continue + entries.append({'name': entry.name, 'path': str(entry.relative_to(root))}) + return {'path': str(path.relative_to(root)), 'entries': sorted(entries, key=lambda item: item['name'].casefold())} + + +def dispatch(request): + uid = int(request['uid']) + if uid < 1000: + raise ValueError('请使用飞牛管理员账号') + action = request['action'] + args = request.get('args', {}) + if action == 'status': + return {'disks': disks(), 'jobs': [{k: v for k, v in job.items() if not k.startswith('_')} + for job in jobs(uid)], 'version': os.environ.get('TRIM_APPVER', '0.1.0')} + if action == 'source': + disk = mount_disk(args['device']) + return {**listing(Path(disk['base']), args.get('path', '')), 'device': disk['id']} + if action == 'destinations': + if not args.get('path'): + return {'path': '', 'entries': [{'name': root.name + ' · 我的文件', 'path': str(root / str(uid))} + for root in destination_roots(uid)]} + path = target_path(args['path'], uid) + return {'path': str(path), 'entries': [{'name': item.name, 'path': str(item)} + for item in sorted(path.iterdir()) if item.is_dir() and not item.is_symlink() and not item.name.startswith('.')]} + if action == 'mkdir': + parent = target_path(args['path'], uid) + name = args['name'].strip() + if not name or name in ('.', '..') or '/' in name or '\0' in name or name.startswith('.'): + raise ValueError('文件夹名称无效') + new = parent / name + new.mkdir() + os.chown(new, uid, pwd.getpwuid(uid).pw_gid) + return {'path': str(new)} + if action == 'start': + if active(): + raise ValueError('已有拷贝任务运行中,请等待完成') + disk = mount_disk(args['device']) + base = Path(disk['base']) + paths = args.get('paths', []) + if not paths or len(paths) > 100: + raise ValueError('请选择 1–100 个目录') + folders = [beneath(base / name, base) for name in paths] + if any(not folder.is_dir() or folder == base for folder in folders): + raise ValueError('请选择硬盘中的具体目录') + if len({folder.name for folder in folders}) != len(folders): + raise ValueError('所选目录存在同名项,请分开拷贝') + parents = {folder.parent for folder in folders} + if len(parents) != 1: + raise ValueError('请选择同一层级下的目录') + destination = target_path(args['destination'], uid) + command(['runuser', '-u', pwd.getpwuid(uid).pw_name, '--', 'test', '-w', str(destination)]) + preview = command(['rsync', '-rltHn', '--ignore-existing', '--stats', '--protect-args', + *map(str, folders), str(destination) + '/'], timeout=120) + match = re.search(r'Total transferred file size: ([\d,]+)', preview) + if not match: + raise ValueError('无法计算拷贝大小,请稍后重试') + total = int(match[1].replace(',', '')) + available = os.statvfs(destination) + if total + 1024**3 > available.f_bavail * available.f_frsize: + raise ValueError('目标空间不足') + job = VAR / 'jobs' / (time.strftime('%Y%m%d-%H%M%S') + '-' + uuid.uuid4().hex[:6]) + job.mkdir(parents=True) + config = {'mount': disk['mount'], 'source_root': str(next(iter(parents)).relative_to(disk['mount'])), + 'destination': str(destination), 'partuuid': disk.get('partuuid'), 'fs_uuid': disk.get('uuid'), + 'device_path': disk['device'], 'serial': disk['serial'], 'uid': uid, 'gid': pwd.getpwuid(uid).pw_gid, + 'directories': [folder.name for folder in folders], 'planned_copy_bytes': total, + 'original_ro': disk.get('original_ro', '0'), 'managed_mount': disk['managed'], + 'filesystem': disk['filesystem'], 'selection_paths': paths} + atomic(job / 'config.json', config) + atomic(job / 'status.json', {'phase': 'preflight', 'started': time.time()}) + with (job / 'worker.log').open('ab') as log: + subprocess.Popen(['/usr/bin/python3', str(ROOT / 'copy_worker.py'), str(job / 'config.json')], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + return {'job': job.name} + if action == 'resume': + selected = next((item for item in jobs(uid) if item['id'] == args['job']), None) + if not selected or selected['legacy'] or selected['phase'] not in ('failed', 'interrupted'): + raise ValueError('此任务当前不能重试') + config = selected['_config'] + return dispatch({'uid': uid, 'action': 'start', 'args': { + 'device': selected['device_id'], 'paths': config['selection_paths'], 'destination': config['destination']}}) + if action == 'eject': + if any(job['device_id'] == args['device'] for job in active()): + raise ValueError('硬盘仍有拷贝任务,完成后才能释放') + known = mounts() + item = known.get(args['device']) + if not item or not item['managed']: + raise ValueError('此盘由飞牛系统挂载,请在系统“外接存储”中推出') + if os.path.ismount(item['mount']): + command(['umount', item['mount']]) + if item['original_ro'] == '0': + command(['blockdev', '--setrw', item['device']]) + del known[args['device']] + atomic(VAR / 'mounts.json', known) + return {'released': True} + raise ValueError('未知操作') + + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + try: + _, uid, _ = struct.unpack('3i', self.request.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) + if uid not in (0, APP_UID): + raise ValueError('拒绝未授权本机客户端') + raw = self.rfile.readline(262145) + if len(raw) > 262144: + raise ValueError('请求过大') + request = json.loads(raw) + result = {'ok': True, 'data': dispatch(request)} + except Exception as error: + result = {'ok': False, 'error': str(error)} + self.wfile.write((json.dumps(result, ensure_ascii=False) + '\n').encode()) + + +if __name__ == '__main__': + VAR.mkdir(exist_ok=True) + path = VAR / 'broker.sock' + path.unlink(missing_ok=True) + server = socketserver.UnixStreamServer(str(path), Handler) + os.chown(path, 0, APP_GID) + path.chmod(0o660) + signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt())) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + path.unlink(missing_ok=True) diff --git a/application/front.py b/application/front.py new file mode 100644 index 0000000..5b88c19 --- /dev/null +++ b/application/front.py @@ -0,0 +1,86 @@ +"""Unprivileged HTTP UI, reachable only through the authenticated fnOS gateway.""" +import http.server +import grp +import json +import os +from pathlib import Path +import socket +import socketserver +import urllib.parse + +ROOT = Path(__file__).resolve().parent +PREFIX = '/app/dayu-diskcopy' +VAR = Path(os.environ['TRIM_PKGVAR']) + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def reply(self, status, data, mime='application/json; charset=utf-8'): + payload = data if isinstance(data, bytes) else json.dumps(data, ensure_ascii=False).encode() + self.send_response(status) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(payload))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.end_headers() + self.wfile.write(payload) + + def handle_request(self, post=False): + try: + uid = self.headers.get('X-Trim-Userid', '') + if self.headers.get('X-Trim-Isadmin', '').lower() != 'true' or not uid.isdigit() or int(uid) < 1000: + return self.reply(403, {'error': '请从飞牛桌面使用管理员账号打开此应用'}) + path = urllib.parse.urlsplit(self.path).path + if path in (PREFIX, PREFIX + '/') and not post: + return self.reply(200, (ROOT / 'index.html').read_bytes(), 'text/html; charset=utf-8') + if not path.startswith(PREFIX + '/api/'): + return self.reply(404, {'error': '页面不存在'}) + action = path.removeprefix(PREFIX + '/api/') + writes = {'start', 'mkdir', 'eject', 'resume'} + if post != (action in writes): + return self.reply(405, {'error': '请求方法无效'}) + if post: + if self.headers.get('X-Dayu-Request') != '1': + return self.reply(403, {'error': '无效请求来源'}) + origin = self.headers.get('Origin') + if origin and urllib.parse.urlsplit(origin).netloc != self.headers.get('Host'): + return self.reply(403, {'error': '请求来源不匹配'}) + length = int(self.headers.get('Content-Length', '0')) + if length > 262144: + return self.reply(413, {'error': '请求过大'}) + args = json.loads(self.rfile.read(length) or '{}') + else: + args = {key: value[0] for key, value in urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).items()} + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(180) + client.connect(str(VAR / 'broker.sock')) + client.sendall((json.dumps({'uid': int(uid), 'action': action, 'args': args}) + '\n').encode()) + with client.makefile('rb') as response: + data = json.loads(response.readline(4 * 1024 * 1024)) + self.reply(200 if data['ok'] else 400, data.get('data') if data['ok'] else {'error': data['error']}) + except Exception: + self.reply(500, {'error': '服务暂时不可用,请稍后刷新或在应用中心重启应用'}) + + def do_GET(self): + self.handle_request() + + def do_POST(self): + self.handle_request(True) + + +class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True + + +if __name__ == '__main__': + path = ROOT / 'app.sock' + path.unlink(missing_ok=True) + server = Server(str(path), Handler) + os.chown(path, int(os.environ['TRIM_UID']), grp.getgrnam('Administrators').gr_gid) + path.chmod(0o660) + os.setgroups([]) + os.setgid(int(os.environ['TRIM_GID'])) + os.setuid(int(os.environ['TRIM_UID'])) + server.serve_forever() diff --git a/application/index.html b/application/index.html new file mode 100644 index 0000000..e2bc45e --- /dev/null +++ b/application/index.html @@ -0,0 +1,33 @@ + + + +大鱼硬盘拷贝 + +

大鱼硬盘拷贝

外接硬盘直连导入,拷贝在 NAS 本机完成

APFS · NTFS · 只读访问源盘
+
1选择源目录2选择存储位置3后台拷贝与校验
+

从外接硬盘拷贝

勾选同一层级中要拷贝的目录
USB
/
请选择 APFS 或 NTFS 硬盘
+

拷贝到 NAS

选择你的存储空间和目标目录
NAS
我的存储空间
文件直接写入此处
存储空间
当前目标:请选择一个目录
+
尚未选择源目录。同名文件会先保留,并在复制后校验。
+

拷贝任务

关闭窗口后仍继续运行
暂无拷贝任务
+
不格式化、不修复源盘;保留目录、文件名和链接。大鱼硬盘拷贝
+

新建目标文件夹

+ diff --git a/application/service.py b/application/service.py new file mode 100644 index 0000000..8abd75e --- /dev/null +++ b/application/service.py @@ -0,0 +1,58 @@ +"""fnOS lifecycle controller. Stops only the two service processes, not copy jobs.""" +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parent +VAR = Path(os.environ['TRIM_PKGVAR']) +STATE = VAR / 'services.json' + + +def alive(pid, script): + try: + return str(ROOT / script).encode() in (Path('/proc') / str(pid) / 'cmdline').read_bytes() + except OSError: + return False + + +def main(): + action = sys.argv[1] + VAR.mkdir(parents=True, exist_ok=True) + os.chown(VAR, 0, int(os.environ['TRIM_GID'])) + VAR.chmod(0o750) + records = json.loads(STATE.read_text()) if STATE.exists() else {} + if action == 'status': + raise SystemExit(0 if all(alive(records.get(name, -1), name) for name in ('broker.py', 'front.py')) else 3) + if action == 'stop': + for script, pid in records.items(): + if alive(pid, script): + os.kill(pid, signal.SIGTERM) + for _ in range(30): + if not any(alive(pid, script) for script, pid in records.items()): + break + time.sleep(.1) + STATE.unlink(missing_ok=True) + return + if action != 'start': + raise SystemExit(1) + for script in ('broker.py', 'front.py'): + if alive(records.get(script, -1), script): + continue + with (VAR / 'service.log').open('ab') as log: + process = subprocess.Popen(['/usr/bin/python3', str(ROOT / script)], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + records[script] = process.pid + STATE.write_text(json.dumps(records)) + for _ in range(50): + if (ROOT / 'app.sock').exists() and (VAR / 'broker.sock').exists(): + return + time.sleep(.1) + raise SystemExit('服务启动失败,请查看应用日志') + + +if __name__ == '__main__': + main() diff --git a/packaging/fpk/app/ui/config b/packaging/fpk/app/ui/config new file mode 100644 index 0000000..bcf6fa3 --- /dev/null +++ b/packaging/fpk/app/ui/config @@ -0,0 +1 @@ +{".url":{"dayu-diskcopy.main":{"title":"大鱼硬盘拷贝","icon":"images/icon_{0}.png","type":"iframe","protocol":"","gatewayPrefix":"/app/dayu-diskcopy","gatewaySocket":"app.sock","url":"/app/dayu-diskcopy","allUsers":false,"control":{"accessPerm":"readonly"}}}} diff --git a/packaging/fpk/cmd/config_callback b/packaging/fpk/cmd/config_callback new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/config_callback @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/cmd/config_init b/packaging/fpk/cmd/config_init new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/config_init @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/cmd/install_callback b/packaging/fpk/cmd/install_callback new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/install_callback @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/cmd/install_init b/packaging/fpk/cmd/install_init new file mode 100644 index 0000000..49f6e09 --- /dev/null +++ b/packaging/fpk/cmd/install_init @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail +for program in python3 rsync ntfs-3g; do + if ! command -v "$program" >/dev/null; then + printf '系统缺少运行组件:%s\n' "$program" > "$TRIM_TEMP_LOGFILE" + exit 1 + fi +done +exit 0 diff --git a/packaging/fpk/cmd/main b/packaging/fpk/cmd/main new file mode 100644 index 0000000..a059fc8 --- /dev/null +++ b/packaging/fpk/cmd/main @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exec /usr/bin/python3 "$TRIM_APPDEST/service.py" "$1" diff --git a/packaging/fpk/cmd/uninstall_callback b/packaging/fpk/cmd/uninstall_callback new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/uninstall_callback @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/cmd/uninstall_init b/packaging/fpk/cmd/uninstall_init new file mode 100644 index 0000000..009d5d0 --- /dev/null +++ b/packaging/fpk/cmd/uninstall_init @@ -0,0 +1,6 @@ +#!/bin/bash +set -euo pipefail +if test -f "$TRIM_APPDEST/service.py"; then + /usr/bin/python3 "$TRIM_APPDEST/service.py" stop +fi +exit 0 diff --git a/packaging/fpk/cmd/upgrade_callback b/packaging/fpk/cmd/upgrade_callback new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/upgrade_callback @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/cmd/upgrade_init b/packaging/fpk/cmd/upgrade_init new file mode 100644 index 0000000..d11b97f --- /dev/null +++ b/packaging/fpk/cmd/upgrade_init @@ -0,0 +1,3 @@ +#!/bin/bash +set -euo pipefail +exit 0 diff --git a/packaging/fpk/config/privilege b/packaging/fpk/config/privilege new file mode 100644 index 0000000..ea32f46 --- /dev/null +++ b/packaging/fpk/config/privilege @@ -0,0 +1 @@ +{"defaults":{"run-as":"root"},"username":"dayu_diskcopy","groupname":"dayu_diskcopy"} diff --git a/packaging/fpk/config/resource b/packaging/fpk/config/resource new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/packaging/fpk/config/resource @@ -0,0 +1 @@ +{} diff --git a/packaging/fpk/manifest b/packaging/fpk/manifest new file mode 100644 index 0000000..5ee08fc --- /dev/null +++ b/packaging/fpk/manifest @@ -0,0 +1,16 @@ +appname=dayu-diskcopy +version=0.1.0 +display_name=大鱼硬盘拷贝 +desc=直接读取 APFS、NTFS 外接硬盘,在飞牛 NAS 本机后台拷贝目录、查看进度和校验结果。 +source=thirdparty +platform=x86 +maintainer=大鱼 +maintainer_url=https://dayu.media +distributor=大鱼 +os_min_version=1.2.0701 +desktop_uidir=ui +desktop_applaunchname=dayu-diskcopy.main +ctl_stop=true +checkport=false +disable_authorization_path=false +changelog=首版:APFS 与 NTFS 只读访问、目录选择、后台复制、进度与校验。 diff --git a/scripts/build-fpk.py b/scripts/build-fpk.py new file mode 100644 index 0000000..e7bd734 --- /dev/null +++ b/scripts/build-fpk.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Build a self-contained native fnOS FPK with the validated Linux driver.""" +from pathlib import Path +import hashlib +import json +import os +import shutil +import struct +import subprocess +import sys +import zlib + +ROOT = Path(__file__).resolve().parent.parent +stage = ROOT / 'build/fpk' +if stage.exists(): + shutil.rmtree(stage) +shutil.copytree(ROOT / 'packaging/fpk', stage) +(stage / 'wizard').mkdir(exist_ok=True) +(stage / 'wizard/.gitkeep').touch() +app = stage / 'app' +for path in (ROOT / 'application').iterdir(): + if path.is_file(): + shutil.copy2(path, app / path.name) +shutil.copy2(ROOT / 'scripts/copy-directories.py', app / 'copy_worker.py') +shutil.copy2(ROOT / 'scripts/save-xattrs.py', app / 'save_xattrs.py') +(app / 'bin').mkdir() +for name, expected in [('apfs-fuse','89a7083c42fb498b021d9f7fabbc5b754eaae503649500e4926965fdcfbcb11e'), + ('apfsutil','692bbd961678c5db1a48e00cb2bc96c50e82dd77e39d87cfe11087608b4658ae')]: + binary = ROOT / 'build/linux-bin' / name + if hashlib.sha256(binary.read_bytes()).hexdigest() != expected: + raise SystemExit('Driver hash mismatch: ' + name) + shutil.copy2(binary, app / 'bin' / name) + (app / 'bin' / name).chmod(0o755) +shutil.copytree(ROOT / 'vendor/apfs-fuse', app / 'source/apfs-fuse') +shutil.copy2(ROOT / 'UPSTREAM.json', app / 'source/UPSTREAM.json') +shutil.copy2(ROOT / 'packaging/Dockerfile', app / 'source/Dockerfile') +for path in (stage / 'cmd').iterdir(): + path.chmod(0o755) + + +def icon(size): + def chunk(kind, payload): + return struct.pack('>I', len(payload)) + kind + payload + struct.pack('>I', zlib.crc32(kind + payload) & 0xffffffff) + rows = [] + for y in range(size): + row = bytearray() + for x in range(size): + px, py = x * 256 / size, y * 256 / size + radius = 50 + dx, dy = max(abs(px - 128) - 78, 0), max(abs(py - 128) - 78, 0) + inside = dx * dx + dy * dy <= radius * radius + color = (22, 118, 102, 255) if inside else (0, 0, 0, 0) + if 53 <= px <= 203 and 62 <= py <= 192 and (px < 63 or px > 193 or py < 72 or py > 182 or 144 < py < 153): + color = (255, 255, 255, 255) + if 78 <= px <= 90 and 164 <= py <= 172: + color = (255, 255, 255, 255) + if 100 <= px <= 126 and abs(py - (px - 3)) < 5 or 122 <= px <= 165 and abs(py - (249 - px)) < 5: + color = (158, 229, 199, 255) + row.extend(color) + rows.append(b'\0' + row) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', size, size, 8, 6, 0, 0, 0)) + chunk(b'IDAT', zlib.compress(b''.join(rows))) + chunk(b'IEND', b'') + + +images = app / 'ui/images' +images.mkdir(exist_ok=True) +for size in (64, 256): + payload = icon(size) + (images / ('icon_' + str(size) + '.png')).write_bytes(payload) + (stage / ('ICON.PNG' if size == 64 else 'ICON_256.PNG')).write_bytes(payload) +subprocess.run([sys.argv[1], 'build', '--directory', str(stage)], cwd=stage, check=True) +output = ROOT / 'dist' +output.mkdir(exist_ok=True) +packages = list(stage.glob('*.fpk')) +if len(packages) != 1: + raise SystemExit('Expected one FPK package') +version = next(line.split('=',1)[1].strip() for line in (stage / 'manifest').read_text().splitlines() if line.startswith('version=')) +destination = output / ('dayu-diskcopy-' + version + '-x86.fpk') +shutil.copy2(packages[0], destination) +print(destination) +print('sha256:', hashlib.sha256(destination.read_bytes()).hexdigest()) diff --git a/scripts/copy-directories.py b/scripts/copy-directories.py index 9c014ea..013ba35 100644 --- a/scripts/copy-directories.py +++ b/scripts/copy-directories.py @@ -22,7 +22,9 @@ def main(): mount = Path(config['mount']) source = mount / config['source_root'] destination = Path(config['destination']).resolve(strict=True) - device = (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True) + device = Path(config['device_path']).resolve(strict=True) if config.get('device_path') else (Path('/dev/disk/by-partuuid') / config['partuuid']).resolve(strict=True) + managed = config.get('managed_mount', True) + os.environ['LC_ALL'] = 'C' lock = (job.parent.parent / 'transfer.lock').open('a') fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) state = {'pid': os.getpid(), 'started': time.time(), 'phase': 'preflight', @@ -48,9 +50,9 @@ def main(): raise RuntimeError('Source is not mounted.') mounted = json.loads(subprocess.check_output( ['findmnt', '-J', '-M', str(mount), '-o', 'SOURCE,OPTIONS'], text=True))['filesystems'][0] - if Path(mounted['source']).resolve() != device or 'ro' not in mounted['options'].split(','): + if Path(mounted['source']).resolve() != device or (managed and 'ro' not in mounted['options'].split(',')): raise RuntimeError('Unexpected source mount or source is not read-only.') - if subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1': + if managed and subprocess.check_output(['blockdev', '--getro', str(device)], text=True).strip() != '1': raise RuntimeError('Block device is not read-only.') writes_before = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] state['source_write_counters_before'] = writes_before @@ -72,7 +74,7 @@ def main(): (job / 'existing-files.list').write_bytes(b'\0'.join(existing) + (b'\0' if existing else b'')) save('copying', existing_files_to_verify=len(existing), copy_started=time.time()) source_args = [str(source / name) for name in config['directories']] - command = ['rsync', '-rltH', '--chown=1000:1001', '--ignore-existing', '--partial', + command = ['rsync', '-rltH', '--chown=' + str(config.get('uid', 1000)) + ':' + str(config.get('gid', 1001)), '--ignore-existing', '--partial', '--no-whole-file', '--partial-dir=.dayu-apfs-partial', '--info=progress2,stats2', '--outbuf=L', '--stats', '--no-inc-recursive', '--protect-args', *source_args, str(destination) + '/'] result = run_to_log(command, 'copy.log') @@ -96,6 +98,14 @@ def main(): *source_args, str(destination) + '/'], 'tree-check.log') if result or (job / 'tree-check.log').read_text().strip(): raise RuntimeError('Final directory comparison found differences; see tree-check.log.') + subprocess.run(['sync', '-f', str(destination)], check=True) + metadata_helper = Path(__file__).with_name('save_xattrs.py') + if config.get('filesystem') == 'apfs' and metadata_helper.exists(): + save('saving_metadata') + metadata_target = destination / ('.dayu-diskcopy-metadata-' + job.name) + result = run_to_log(['/usr/bin/python3', str(metadata_helper), str(config_path), str(metadata_target)], 'metadata-summary.json') + if result: + raise RuntimeError('File copy passed but extended attribute archive failed.') save('complete', finished=time.time(), existing_files_checksum_match=True, new_files_verified_by_rsync_transfer_checksums=True) except BaseException as error: @@ -103,12 +113,12 @@ def main(): traceback.print_exc() finally: try: - if os.path.ismount(mount): + if managed and os.path.ismount(mount): subprocess.run(['umount', str(mount)], check=True, timeout=60) - if config['original_ro'] == '0': + if managed and config['original_ro'] == '0': subprocess.run(['blockdev', '--setrw', str(device)], check=True) writes_after = (Path('/sys/class/block') / device.name / 'stat').read_text().split()[4:8:2] - save(state['phase'], unmounted=True, source_write_counters_after=writes_after, + save(state['phase'], unmounted=managed and not os.path.ismount(mount), source_write_counters_after=writes_after, source_writes_unchanged=writes_after == state.get('source_write_counters_before')) except BaseException as error: save('cleanup_failed', prior_phase=state['phase'], cleanup_error=str(error)) diff --git a/tests/test_application.py b/tests/test_application.py new file mode 100644 index 0000000..7b5309f --- /dev/null +++ b/tests/test_application.py @@ -0,0 +1,85 @@ +import importlib.util +import json +import os +from pathlib import Path +import socket +import tempfile +import threading +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +def module(name): + os.environ.setdefault('TRIM_PKGVAR', '/tmp/dayu-diskcopy-test') + os.environ.setdefault('TRIM_UID', '99999') + os.environ.setdefault('TRIM_GID', '99999') + spec = importlib.util.spec_from_file_location(name, ROOT / 'application' / (name + '.py')) + result = importlib.util.module_from_spec(spec) + spec.loader.exec_module(result) + return result + + +class PathBoundaryTests(unittest.TestCase): + def test_path_traversal_and_symlink_escape_are_rejected(self): + broker = module('broker') + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) / 'root' + root.mkdir() + outside = Path(folder) / 'outside' + outside.mkdir() + (root / 'shortcut').symlink_to(outside) + self.assertEqual(broker.beneath(root, root), root.resolve()) + for path in (root / '../outside', root / 'shortcut'): + with self.assertRaises(ValueError): + broker.beneath(path, root) + + def test_system_identity_rejected_before_any_disk_operation(self): + broker = module('broker') + with self.assertRaises(ValueError): + broker.dispatch({'uid': 0, 'action': 'start', 'args': {}}) + + +class GatewayTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.front = module('front') + cls.temp = tempfile.TemporaryDirectory(dir='/tmp', prefix='dayu-ui-') + cls.path = str(Path(cls.temp.name) / 'web.sock') + cls.server = cls.front.Server(cls.path, cls.front.Handler) + cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True) + cls.thread.start() + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + cls.thread.join() + cls.temp.cleanup() + + def request(self, method, path, headers=''): + with socket.socket(socket.AF_UNIX) as client: + client.connect(self.path) + client.sendall((method + ' ' + path + ' HTTP/1.0\r\nHost: nas.local\r\n' + headers + '\r\n').encode()) + response = b'' + while data := client.recv(65536): + response += data + return int(response.split(b' ')[1]), response + + def test_anonymous_and_non_admin_are_rejected(self): + self.assertEqual(self.request('GET', '/app/dayu-diskcopy')[0], 403) + self.assertEqual(self.request('GET', '/app/dayu-diskcopy', 'X-Trim-Userid: 1000\r\nX-Trim-Isadmin: false\r\n')[0], 403) + + def test_admin_can_load_interface(self): + code, data = self.request('GET', '/app/dayu-diskcopy', 'X-Trim-Userid: 1000\r\nX-Trim-Isadmin: true\r\n') + self.assertEqual(code, 200) + self.assertIn('大鱼硬盘拷贝'.encode(), data) + + def test_writes_require_request_marker_and_same_origin(self): + headers = 'X-Trim-Userid: 1000\r\nX-Trim-Isadmin: true\r\nContent-Length: 0\r\n' + self.assertEqual(self.request('POST', '/app/dayu-diskcopy/api/start', headers)[0], 403) + self.assertEqual(self.request('POST', '/app/dayu-diskcopy/api/start', headers + 'X-Dayu-Request: 1\r\nOrigin: https://other.local\r\n')[0], 403) + + +if __name__ == '__main__': + unittest.main()