diff --git a/HANDOFF.md b/HANDOFF.md index f2464c9..e86ee89 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -4,7 +4,13 @@ ## 当前目标 -基础 APFS 只读挂载已验证。当前用户要求把 4 TB APFS USB 盘的“火影秋决”和“LCQ”比赛目录复制到 NAS `/vol5/1000/剪辑盘备份`,正在 NAS 本机执行复制。 +当前并行目标:完成已授权的两目录 NAS 本机复制;开发可安装的飞牛桌面应用“大鱼硬盘拷贝”,支持 APFS 与 NTFS 的识别、选目录和后台复制。 + +- 用户已要求停止逐条报告拷贝速度,保留内部监控,仅在完成或异常时告知。 +- 新应用沿用当前仓库与驱动,固定 appname 为 `dayu-diskcopy`、唯一安装入口 `/var/apps/dayu-diskcopy`。使用飞牛登录态网关,普通 Web 进程降权运行,设备操作由本机受限服务执行。 +- NAS 已核实为 fnOS `1.2.0701`,有 ntfs-3g、mkntfs、ntfscp。NTFS 已有系统挂载时优先复用,应用只读取源文件;不强行卸载或修改系统在用盘。未挂载的新盘使用只读挂载。 +- 应用第一版代码位于 `application/`,FPK 定义在 `packaging/fpk/`,构建入口 `scripts/build-fpk.py`。当前版本 0.1.0;使用官方网关 Unix Socket,不开放新的网络端口。Web 进程启动后降为专用包用户,原始设备操作由本机 Socket 服务处理并核验调用 UID。 +- 已通过五项边界测试:匿名及非管理员拒绝、管理员页面可读、写请求来源校验、路径穿越和符号链接越界拒绝、系统身份拒绝;Python/生命周期脚本语法通过。FPK 构建和实机界面/NTFS 镜像验证正在进行,尚未宣称正式交付。 ## 当前复制任务(2026-11-11 22:36 启动) diff --git a/application/broker.py b/application/broker.py new file mode 100644 index 0000000..891ce3b --- /dev/null +++ b/application/broker.py @@ -0,0 +1,313 @@ +"""Privileged local-only disk operations; no TCP listener and no shell execution.""" +import fcntl +import grp +import json +import os +from pathlib import Path +import pwd +import re +import signal +import socket +import socketserver +import stat +import struct +import subprocess +import time +import uuid + +ROOT = Path(__file__).resolve().parent +VAR = Path(os.environ['TRIM_PKGVAR']) +APP_UID = int(os.environ['TRIM_UID']) +APP_GID = int(os.environ['TRIM_GID']) +LEGACY = Path('/var/tmp/dayu-apfs-validation') +TERMINAL = {'complete', 'failed', 'needs_attention', 'cleanup_failed', 'cancelled'} + + +def command(args, timeout=30): + result = subprocess.run(args, capture_output=True, text=True, timeout=timeout, env={**os.environ, 'LC_ALL': 'C'}) + if result.returncode: + raise ValueError((result.stderr or result.stdout).strip()[-600:] or '操作失败') + return result.stdout.strip() + + +def beneath(path, root, must_exist=True): + path = Path(path).resolve(strict=must_exist) + root = Path(root).resolve(strict=True) + if not path.is_relative_to(root): + raise ValueError('所选路径超出可访问目录') + return path + + +def atomic(path, data): + temporary = path.with_suffix('.tmp') + temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2)) + temporary.replace(path) + + +def disks(): + tree = json.loads(command(['lsblk', '-J', '-b', '-o', + 'NAME,PATH,TYPE,SIZE,FSTYPE,UUID,PARTUUID,TRAN,MODEL,SERIAL,MOUNTPOINTS'])) + result = [] + for disk in tree['blockdevices']: + if disk.get('tran') != 'usb' or not disk.get('size'): + continue + for part in [disk, *disk.get('children', [])]: + filesystem = (part.get('fstype') or '').lower() + if filesystem not in ('apfs', 'ntfs', 'ntfs3'): + continue + identity = part.get('partuuid') or part.get('uuid') + if not identity or not disk.get('serial'): + continue + result.append({'id': identity, 'partuuid': part.get('partuuid'), + 'uuid': part.get('uuid'), 'device': part['path'], 'filesystem': filesystem, + 'serial': disk['serial'], 'model': (disk.get('model') or 'USB 硬盘').strip(), + 'size': part['size'], 'mountpoints': [x for x in part.get('mountpoints', []) if x]}) + return result + + +def mounts(): + path = VAR / 'mounts.json' + return json.loads(path.read_text()) if path.exists() else {} + + +def jobs(uid=None): + result = [] + for parent, legacy in ((VAR / 'jobs', False), (LEGACY / 'jobs', True)): + if not parent.exists(): + continue + for folder in sorted(parent.iterdir(), reverse=True): + try: + config = json.loads((folder / 'config.json').read_text()) + state = json.loads((folder / 'status.json').read_text()) + owner = config.get('uid', 1000) + if uid is not None and uid != owner: + continue + data = (folder / 'copy.log').read_bytes()[-16000:].decode(errors='replace') + progress = list(re.finditer(r'([\d,]+)\s+(\d+)%\s+([\d.]+)([kMGT]?)B/s\s+', data)) + copied = int(progress[-1][1].replace(',', '')) if progress else 0 + phase = state['phase'] + paused = (folder / 'paused').exists() + if paused: + phase = 'paused' + alive = Path('/proc/' + str(state.get('pid', -1))).exists() + if not alive and phase not in TERMINAL and time.time() - state.get('started', 0) > 10: + phase = 'interrupted' + if phase == 'complete' and 'source_write_counters_after' not in state: + phase = 'finalizing' + speed = 0 + if progress and phase == 'copying': + multiplier = 1024 ** ('kMGT'.find(progress[-1][4]) + 1) if progress[-1][4] else 1 + speed = float(progress[-1][3]) * multiplier + total = config.get('planned_copy_bytes', 0) + if phase == 'complete': + copied = total + result.append({'id': ('legacy:' if legacy else '') + folder.name, + 'directories': config['directories'], 'destination': config['destination'], + 'phase': phase, 'bytes': copied, 'total': total, 'speed': speed, + 'eta': max(0, total - copied) / speed if speed else None, + 'started': state.get('started'), 'finished': state.get('finished'), + 'error': state.get('error') or state.get('reason') or state.get('cleanup_error'), + 'device_id': config.get('partuuid') or config.get('fs_uuid'), + 'legacy': legacy, 'uid': owner, '_folder': str(folder), '_config': config, + 'source_writes_unchanged': state.get('source_writes_unchanged')}) + except (OSError, ValueError, KeyError): + continue + return sorted(result, key=lambda item: item.get('started') or 0, reverse=True) + + +def active(): + return [job for job in jobs() if job['phase'] not in TERMINAL and job['phase'] != 'interrupted'] + + +def mount_disk(identity): + choices = [item for item in disks() if item['id'] == identity] + if len(choices) != 1: + raise ValueError('硬盘已移除或身份发生变化,请刷新') + disk = choices[0] + # Adopt the current authorized copy without remounting or moving its data. + for job in active(): + config = job['_config'] + if job['device_id'] == identity and os.path.ismount(config['mount']): + return {**disk, 'mount': config['mount'], 'base': str(Path(config['mount']) / config['source_root']), + 'managed': False, 'busy': True} + known = mounts() + if identity in known and os.path.ismount(known[identity]['mount']): + return {**disk, **known[identity]} + if disk['filesystem'].startswith('ntfs') and disk['mountpoints']: + mount = Path(disk['mountpoints'][0]) + return {**disk, 'mount': str(mount), 'base': str(mount), 'managed': False, 'busy': False} + if disk['mountpoints']: + raise ValueError('该硬盘已被其他程序挂载,请先停止使用') + device = Path(disk['device']) + target = VAR / 'mounts' / re.sub(r'[^a-zA-Z0-9_-]', '_', identity) + target.mkdir(parents=True, exist_ok=True) + if any(target.iterdir()): + raise ValueError('挂载目录不为空,已停止以保护数据') + original = command(['blockdev', '--getro', str(device)]) + command(['blockdev', '--setro', str(device)]) + try: + if disk['filesystem'] == 'apfs': + command([str(ROOT / 'bin/apfs-fuse'), '-o', 'ro,nodev,nosuid,noexec', str(device), str(target)]) + else: + command(['ntfs-3g', '-o', 'ro,nodev,nosuid,noexec', str(device), str(target)]) + if not os.path.ismount(target): + raise ValueError('无法只读挂载;加密卷或此文件系统特性可能暂不支持') + except BaseException: + if os.path.ismount(target): + command(['umount', str(target)]) + if original == '0': + command(['blockdev', '--setrw', str(device)]) + raise + base = target / 'root' if (target / 'root').is_dir() and (target / 'private-dir').is_dir() else target + known[identity] = {'mount': str(target), 'base': str(base), 'managed': True, + 'device': str(device), 'original_ro': original, 'serial': disk['serial']} + atomic(VAR / 'mounts.json', known) + return {**disk, **known[identity]} + + +def destination_roots(uid): + return [path for path in Path('/').glob('vol[0-9]*') + if (path / str(uid)).is_dir() and os.path.ismount(path)] + + +def target_path(path, uid, must_exist=True): + resolved = Path(path).resolve(strict=must_exist) + if not any(resolved.is_relative_to(root / str(uid)) for root in destination_roots(uid)): + raise ValueError('请选择当前用户存储空间中的目标目录') + return resolved + + +def listing(root, relative=''): + path = beneath(root / relative, root) + entries = [] + for entry in path.iterdir(): + if entry.name.startswith('.') or entry.is_symlink() or not entry.is_dir(): + continue + entries.append({'name': entry.name, 'path': str(entry.relative_to(root))}) + return {'path': str(path.relative_to(root)), 'entries': sorted(entries, key=lambda item: item['name'].casefold())} + + +def dispatch(request): + uid = int(request['uid']) + if uid < 1000: + raise ValueError('请使用飞牛管理员账号') + action = request['action'] + args = request.get('args', {}) + if action == 'status': + return {'disks': disks(), 'jobs': [{k: v for k, v in job.items() if not k.startswith('_')} + for job in jobs(uid)], 'version': os.environ.get('TRIM_APPVER', '0.1.0')} + if action == 'source': + disk = mount_disk(args['device']) + return {**listing(Path(disk['base']), args.get('path', '')), 'device': disk['id']} + if action == 'destinations': + if not args.get('path'): + return {'path': '', 'entries': [{'name': root.name + ' · 我的文件', 'path': str(root / str(uid))} + for root in destination_roots(uid)]} + path = target_path(args['path'], uid) + return {'path': str(path), 'entries': [{'name': item.name, 'path': str(item)} + for item in sorted(path.iterdir()) if item.is_dir() and not item.is_symlink() and not item.name.startswith('.')]} + if action == 'mkdir': + parent = target_path(args['path'], uid) + name = args['name'].strip() + if not name or name in ('.', '..') or '/' in name or '\0' in name or name.startswith('.'): + raise ValueError('文件夹名称无效') + new = parent / name + new.mkdir() + os.chown(new, uid, pwd.getpwuid(uid).pw_gid) + return {'path': str(new)} + if action == 'start': + if active(): + raise ValueError('已有拷贝任务运行中,请等待完成') + disk = mount_disk(args['device']) + base = Path(disk['base']) + paths = args.get('paths', []) + if not paths or len(paths) > 100: + raise ValueError('请选择 1–100 个目录') + folders = [beneath(base / name, base) for name in paths] + if any(not folder.is_dir() or folder == base for folder in folders): + raise ValueError('请选择硬盘中的具体目录') + if len({folder.name for folder in folders}) != len(folders): + raise ValueError('所选目录存在同名项,请分开拷贝') + parents = {folder.parent for folder in folders} + if len(parents) != 1: + raise ValueError('请选择同一层级下的目录') + destination = target_path(args['destination'], uid) + command(['runuser', '-u', pwd.getpwuid(uid).pw_name, '--', 'test', '-w', str(destination)]) + preview = command(['rsync', '-rltHn', '--ignore-existing', '--stats', '--protect-args', + *map(str, folders), str(destination) + '/'], timeout=120) + match = re.search(r'Total transferred file size: ([\d,]+)', preview) + if not match: + raise ValueError('无法计算拷贝大小,请稍后重试') + total = int(match[1].replace(',', '')) + available = os.statvfs(destination) + if total + 1024**3 > available.f_bavail * available.f_frsize: + raise ValueError('目标空间不足') + job = VAR / 'jobs' / (time.strftime('%Y%m%d-%H%M%S') + '-' + uuid.uuid4().hex[:6]) + job.mkdir(parents=True) + config = {'mount': disk['mount'], 'source_root': str(next(iter(parents)).relative_to(disk['mount'])), + 'destination': str(destination), 'partuuid': disk.get('partuuid'), 'fs_uuid': disk.get('uuid'), + 'device_path': disk['device'], 'serial': disk['serial'], 'uid': uid, 'gid': pwd.getpwuid(uid).pw_gid, + 'directories': [folder.name for folder in folders], 'planned_copy_bytes': total, + 'original_ro': disk.get('original_ro', '0'), 'managed_mount': disk['managed'], + 'filesystem': disk['filesystem'], 'selection_paths': paths} + atomic(job / 'config.json', config) + atomic(job / 'status.json', {'phase': 'preflight', 'started': time.time()}) + with (job / 'worker.log').open('ab') as log: + subprocess.Popen(['/usr/bin/python3', str(ROOT / 'copy_worker.py'), str(job / 'config.json')], + stdin=subprocess.DEVNULL, stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + return {'job': job.name} + if action == 'resume': + selected = next((item for item in jobs(uid) if item['id'] == args['job']), None) + if not selected or selected['legacy'] or selected['phase'] not in ('failed', 'interrupted'): + raise ValueError('此任务当前不能重试') + config = selected['_config'] + return dispatch({'uid': uid, 'action': 'start', 'args': { + 'device': selected['device_id'], 'paths': config['selection_paths'], 'destination': config['destination']}}) + if action == 'eject': + if any(job['device_id'] == args['device'] for job in active()): + raise ValueError('硬盘仍有拷贝任务,完成后才能释放') + known = mounts() + item = known.get(args['device']) + if not item or not item['managed']: + raise ValueError('此盘由飞牛系统挂载,请在系统“外接存储”中推出') + if os.path.ismount(item['mount']): + command(['umount', item['mount']]) + if item['original_ro'] == '0': + command(['blockdev', '--setrw', item['device']]) + del known[args['device']] + atomic(VAR / 'mounts.json', known) + return {'released': True} + raise ValueError('未知操作') + + +class Handler(socketserver.StreamRequestHandler): + def handle(self): + try: + _, uid, _ = struct.unpack('3i', self.request.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, 12)) + if uid not in (0, APP_UID): + raise ValueError('拒绝未授权本机客户端') + raw = self.rfile.readline(262145) + if len(raw) > 262144: + raise ValueError('请求过大') + request = json.loads(raw) + result = {'ok': True, 'data': dispatch(request)} + except Exception as error: + result = {'ok': False, 'error': str(error)} + self.wfile.write((json.dumps(result, ensure_ascii=False) + '\n').encode()) + + +if __name__ == '__main__': + VAR.mkdir(exist_ok=True) + path = VAR / 'broker.sock' + path.unlink(missing_ok=True) + server = socketserver.UnixStreamServer(str(path), Handler) + os.chown(path, 0, APP_GID) + path.chmod(0o660) + signal.signal(signal.SIGTERM, lambda *_: (_ for _ in ()).throw(KeyboardInterrupt())) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + path.unlink(missing_ok=True) diff --git a/application/front.py b/application/front.py new file mode 100644 index 0000000..5b88c19 --- /dev/null +++ b/application/front.py @@ -0,0 +1,86 @@ +"""Unprivileged HTTP UI, reachable only through the authenticated fnOS gateway.""" +import http.server +import grp +import json +import os +from pathlib import Path +import socket +import socketserver +import urllib.parse + +ROOT = Path(__file__).resolve().parent +PREFIX = '/app/dayu-diskcopy' +VAR = Path(os.environ['TRIM_PKGVAR']) + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def reply(self, status, data, mime='application/json; charset=utf-8'): + payload = data if isinstance(data, bytes) else json.dumps(data, ensure_ascii=False).encode() + self.send_response(status) + self.send_header('Content-Type', mime) + self.send_header('Content-Length', str(len(payload))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.end_headers() + self.wfile.write(payload) + + def handle_request(self, post=False): + try: + uid = self.headers.get('X-Trim-Userid', '') + if self.headers.get('X-Trim-Isadmin', '').lower() != 'true' or not uid.isdigit() or int(uid) < 1000: + return self.reply(403, {'error': '请从飞牛桌面使用管理员账号打开此应用'}) + path = urllib.parse.urlsplit(self.path).path + if path in (PREFIX, PREFIX + '/') and not post: + return self.reply(200, (ROOT / 'index.html').read_bytes(), 'text/html; charset=utf-8') + if not path.startswith(PREFIX + '/api/'): + return self.reply(404, {'error': '页面不存在'}) + action = path.removeprefix(PREFIX + '/api/') + writes = {'start', 'mkdir', 'eject', 'resume'} + if post != (action in writes): + return self.reply(405, {'error': '请求方法无效'}) + if post: + if self.headers.get('X-Dayu-Request') != '1': + return self.reply(403, {'error': '无效请求来源'}) + origin = self.headers.get('Origin') + if origin and urllib.parse.urlsplit(origin).netloc != self.headers.get('Host'): + return self.reply(403, {'error': '请求来源不匹配'}) + length = int(self.headers.get('Content-Length', '0')) + if length > 262144: + return self.reply(413, {'error': '请求过大'}) + args = json.loads(self.rfile.read(length) or '{}') + else: + args = {key: value[0] for key, value in urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).items()} + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(180) + client.connect(str(VAR / 'broker.sock')) + client.sendall((json.dumps({'uid': int(uid), 'action': action, 'args': args}) + '\n').encode()) + with client.makefile('rb') as response: + data = json.loads(response.readline(4 * 1024 * 1024)) + self.reply(200 if data['ok'] else 400, data.get('data') if data['ok'] else {'error': data['error']}) + except Exception: + self.reply(500, {'error': '服务暂时不可用,请稍后刷新或在应用中心重启应用'}) + + def do_GET(self): + self.handle_request() + + def do_POST(self): + self.handle_request(True) + + +class Server(socketserver.ThreadingMixIn, socketserver.UnixStreamServer): + daemon_threads = True + + +if __name__ == '__main__': + path = ROOT / 'app.sock' + path.unlink(missing_ok=True) + server = Server(str(path), Handler) + os.chown(path, int(os.environ['TRIM_UID']), grp.getgrnam('Administrators').gr_gid) + path.chmod(0o660) + os.setgroups([]) + os.setgid(int(os.environ['TRIM_GID'])) + os.setuid(int(os.environ['TRIM_UID'])) + server.serve_forever() diff --git a/application/index.html b/application/index.html new file mode 100644 index 0000000..e2bc45e --- /dev/null +++ b/application/index.html @@ -0,0 +1,33 @@ + + + +
外接硬盘直连导入,拷贝在 NAS 本机完成