From 7aece444b014d66004d31085710f8161b2d55451 Mon Sep 17 00:00:00 2001 From: mac Date: Fri, 6 Nov 2026 19:18:22 +0800 Subject: [PATCH] WIP: benchmark mature trackers and bounded occlusion recovery before host integration --- research/TrackingRecoveryCandidates.md | 17 +++++ tools/tracking/benchmark_object.mm | 17 +++++ tools/tracking/benchmark_opensource.py | 22 ++++++ tools/tracking/benchmark_vision.mm | 23 +++++++ 大鱼调色/Plugin/MaskCore/DYObjectMotion.hpp | 51 ++++++++++++++ 大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp | 72 ++++++++++++++++++++ 开发交接.md | 2 + 7 files changed, 204 insertions(+) create mode 100644 research/TrackingRecoveryCandidates.md create mode 100644 tools/tracking/benchmark_object.mm create mode 100644 tools/tracking/benchmark_opensource.py create mode 100644 tools/tracking/benchmark_vision.mm create mode 100644 大鱼调色/Plugin/MaskCore/DYObjectMotion.hpp create mode 100644 大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp diff --git a/research/TrackingRecoveryCandidates.md b/research/TrackingRecoveryCandidates.md new file mode 100644 index 0000000..e0dd8d5 --- /dev/null +++ b/research/TrackingRecoveryCandidates.md @@ -0,0 +1,17 @@ +# 转身与短时遮挡候选(开发中) + +2026-11-06,接续正式 1.18.1(106)。用户要求稳定马赛克、可根据丢失前后运动补齐轨迹,并要求比较 GitHub 成熟算法。正式 APP 尚未改动。 + +现有 OpenCV LK 负责局部光流与仿射拟合,没有完整对象外观模型;转身时背景特征可能仍给出高置信度。不能仅沿错误光流继续外推。 + +已建立独立、无界面的本地基准:640×360,25fps,285 帧原实拍;另在第 130—139 帧人为遮住头部。素材和逐帧位置不上传仓库。 + +- Apple Vision accurate 对照:转身全段跟踪完成;原始连续更新版本在遮挡中会学习遮挡物。候选在失去可信观测时冻结最后可信图像,只从冻结图像试探重新定位。当前候选全段完成,短遮挡测试预测 11 帧后重新确认,热运行约 1ms/帧。此为局部基准,未接入正式宿主。 +- 候选运动保护:预测最多 0.8 秒,重新定位需连续两帧确认;限制跳跃和尺寸突变。计划在重新定位后用两端可信位置补齐中间键,尾部未确认预测不得作为成功轨迹提交。此部分尚未接入提交路径。 +- [OpenCV CSRT](https://github.com/opencv/opencv_contrib/tree/4.x/modules/tracking):完整相关滤波对象追踪,测试官方 OpenCV 4.11 wheel,后续比较同片段。 +- [OpenCV Zoo VitTrack](https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack):轻量 Transformer 单目标跟踪,有追踪分数,代码/模型目录 Apache-2.0;官方模型 714726 字节,SHA-256 `2990f0b7cd44d92afa48cd97db6de7be113fc1d9594fddb74e2725c10478e91d`,已核对 LFS 指针与完整下载。后续比较同片段。 +- [NanoTrack](https://github.com/HonglinChu/NanoTrack):提供原生 Mac/ncnn 示例;OpenCV Zoo 的对比说明指出其分数不适合判断丢失,暂列次选,不用 README 的速度当本机实测。 +- [OSTrack](https://github.com/botaoye/OSTrack):完整 Transformer 方案,原示例以 PyTorch/CUDA 环境为主;不把研究榜单直接当作 Mac 插件速度。 +- [ByteTrack](https://github.com/FoundationVision/ByteTrack):关联检测框与多目标身份,需要额外检测器,单独引入不能解决背头没有人脸框的问题。 + +在同素材准确度、短遮挡恢复、背景误锁、计算开销和宿主交互验证通过前,不替换正式版。 diff --git a/tools/tracking/benchmark_object.mm b/tools/tracking/benchmark_object.mm new file mode 100644 index 0000000..a14c332 --- /dev/null +++ b/tools/tracking/benchmark_object.mm @@ -0,0 +1,17 @@ +#import +#include "../../大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp" +#include +#include +int main(int argc,const char**argv){@autoreleasepool{ + if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);double fps=atof(argv[8]);dy::TrackerROI roi{float(atof(argv[4])),float(atof(argv[5])),float(atof(argv[6])),float(atof(argv[7])),0}; + std::ifstream input(argv[1],std::ios::binary);std::vector bytes(size_t(w)*h*4);dy::ObjectTracker tracker;NSMutableArray *frames=[NSMutableArray new];double total=0,maximum=0; + while(input.read((char*)bytes.data(),bytes.size())){@autoreleasepool{ + for(size_t i=0;i(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms); + [frames addObject:@{@"cx":@(r.cx),@"cy":@(r.cy),@"width":@(roi.width*r.scaleX),@"height":@(roi.height*r.scaleY),@"confidence":@(r.confidence),@"valid":@(r.valid),@"predicted":@(step.predicted),@"recovered":@(step.recovered),@"gap":@(step.gap),@"ms":@(ms)}];if(!step.valid)break; + }} + NSData *data=[NSJSONSerialization dataWithJSONObject:@{@"frames":frames,@"milliseconds":@(total),@"maximumMs":@(maximum)} options:NSJSONWritingPrettyPrinted error:nil];fwrite(data.bytes,1,data.length,stdout);return 0; +}} diff --git a/tools/tracking/benchmark_opensource.py b/tools/tracking/benchmark_opensource.py new file mode 100644 index 0000000..39b5646 --- /dev/null +++ b/tools/tracking/benchmark_opensource.py @@ -0,0 +1,22 @@ +"""Compare official OpenCV trackers on local caller-decoded frames; no GUI/media upload.""" +import argparse,json,time,pathlib +import cv2 as cv +import numpy as np +p=argparse.ArgumentParser();p.add_argument('frames');p.add_argument('algorithm',choices=['csrt','vit']);p.add_argument('--model');p.add_argument('--output',required=True);a=p.parse_args() +cv.setNumThreads(2) +if a.algorithm=='csrt':tracker=cv.TrackerCSRT_create() +else: + params=cv.TrackerVit_Params();params.net=a.model;tracker=cv.TrackerVit_create(params) +roi=(round((.486-.085/2)*640),round((.238-.155/2)*360),round(.085*640),round(.155*360)) +results=[] +with open(a.frames,'rb') as stream: + while data:=stream.read(640*360*4): + assert len(data)==640*360*4 + image=np.frombuffer(data,np.uint8).reshape(360,640,4)[:,:,:3].copy() + start=time.perf_counter() + if not results:tracker.init(image,roi);ok=True;box=roi;score=1. + else: + ok,box=tracker.update(image);score=tracker.getTrackingScore() if a.algorithm=='vit' else float(ok) + elapsed=(time.perf_counter()-start)*1000;x,y,w,h=box + results.append(dict(valid=bool(ok),confidence=float(score),cx=(x+w/2)/640,cy=(y+h/2)/360,width=w/640,height=h/360,ms=elapsed)) +pathlib.Path(a.output).write_text(json.dumps(dict(algorithm=a.algorithm,opencv=cv.__version__,threads=2,frames=results,milliseconds=sum(x['ms'] for x in results)),indent=2)+'\n') diff --git a/tools/tracking/benchmark_vision.mm b/tools/tracking/benchmark_vision.mm new file mode 100644 index 0000000..9215f6d --- /dev/null +++ b/tools/tracking/benchmark_vision.mm @@ -0,0 +1,23 @@ +// Read-only local media benchmark. No host app, uploads, or persisted images. +#import +#import +#import +#include +#include +int main(int argc,const char**argv){@autoreleasepool{ + if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]); + CGRect box=CGRectMake(atof(argv[4])-atof(argv[6])/2,1-atof(argv[5])-atof(argv[7])/2,atof(argv[6]),atof(argv[7])); + VNSequenceRequestHandler *sequence=[VNSequenceRequestHandler new]; + VNTrackObjectRequest *track=[[VNTrackObjectRequest alloc]initWithDetectedObjectObservation:[VNDetectedObjectObservation observationWithBoundingBox:box]]; + track.trackingLevel=atoi(argv[8])?VNRequestTrackingLevelAccurate:VNRequestTrackingLevelFast; + CVPixelBufferRef buffer=nullptr;CVPixelBufferCreate(kCFAllocatorDefault,w,h,kCVPixelFormatType_32BGRA,(__bridge CFDictionaryRef)@{(id)kCVPixelBufferIOSurfacePropertiesKey:@{}},&buffer); + if(!buffer)return 3;std::ifstream input(argv[1],std::ios::binary);std::vector bytes(size_t(w)*h*4);NSMutableArray *frames=[NSMutableArray new];double total=0,maximum=0; + while(input.read((char*)bytes.data(),bytes.size())){@autoreleasepool{ + CVPixelBufferLockBaseAddress(buffer,0);auto dst=(uint8_t*)CVPixelBufferGetBaseAddress(buffer);for(int y=0;y(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms); + VNDetectedObjectObservation *r=track.results.firstObject;CGRect b=r.boundingBox; + [frames addObject:@{@"cx":@(CGRectGetMidX(b)),@"cy":@(1-CGRectGetMidY(b)),@"width":@(b.size.width),@"height":@(b.size.height),@"confidence":@(r.confidence),@"valid":@(ok&&r!=nil),@"ms":@(ms)}];if(!ok||!r)break;track.inputObservation=r; + }} + CVPixelBufferRelease(buffer);NSData *data=[NSJSONSerialization dataWithJSONObject:@{@"frames":frames,@"milliseconds":@(total),@"maximumMs":@(maximum)} options:NSJSONWritingPrettyPrinted error:nil];fwrite(data.bytes,1,data.length,stdout);return 0; +}} diff --git a/大鱼调色/Plugin/MaskCore/DYObjectMotion.hpp b/大鱼调色/Plugin/MaskCore/DYObjectMotion.hpp new file mode 100644 index 0000000..0c8da2f --- /dev/null +++ b/大鱼调色/Plugin/MaskCore/DYObjectMotion.hpp @@ -0,0 +1,51 @@ +#pragma once +#include +#include + +namespace dy { +struct ObjectBox {float cx=.5f,cy=.5f,width=.1f,height=.1f,confidence=0;}; +struct ObjectStep {ObjectBox box;bool valid=false,predicted=false,recovered=false;int gap=0;}; + +// Bounded prediction in analysis-frame order (also valid for reverse passes). +// An unconfirmed tail is never an observed trajectory. Host commit trims it. +class ObjectMotion { + ObjectBox trusted_{},candidate_{}; + float vx_=0,vy_=0; + int missing_=0,limit_=20,candidateCount_=0; + bool active_=false; + static bool finite(ObjectBox b){return std::isfinite(b.cx)&&std::isfinite(b.cy)&&std::isfinite(b.width)&&std::isfinite(b.height)&&std::isfinite(b.confidence)&&b.width>.003f&&b.height>.003f&&b.width<=1&&b.height<=1&&b.cx>=0&&b.cx<=1&&b.cy>=0&&b.cy<=1;} +public: + void reset(ObjectBox b,double fps){trusted_=b;vx_=vy_=0;missing_=candidateCount_=0;limit_=std::clamp(int(std::ceil(std::clamp(fps,1.,240.)*.8)),2,192);active_=finite(b);} + int missing()const{return missing_;} + bool active()const{return active_;} + ObjectBox prediction(int ahead=1)const{auto b=trusted_;b.cx+=vx_*(missing_+ahead);b.cy+=vy_*(missing_+ahead);return b;} + ObjectStep advance(ObjectBox measurement,bool observed){ + if(!active_)return {}; + auto expected=prediction(); + float size=std::max(trusted_.width,trusted_.height); + float delta=std::hypot(measurement.cx-expected.cx,measurement.cy-expected.cy); + float scaleX=measurement.width/trusted_.width,scaleY=measurement.height/trusted_.height; + bool good=observed&&finite(measurement)&&measurement.confidence>=(missing_?.70f:.55f)&& + delta.65f&&scaleX<1.55f&&scaleY>.65f&&scaleY<1.55f; + if(good&&missing_){ + if(candidateCount_&&std::hypot(measurement.cx-candidate_.cx,measurement.cy-candidate_.cy)=2; + }else if(!good)candidateCount_=0; + if(good){ + int gap=missing_;float dt=float(gap+1); + float nx=(measurement.cx-trusted_.cx)/dt,ny=(measurement.cy-trusted_.cy)/dt; + float maximum=size*.12f;float length=std::hypot(nx,ny);if(length>maximum){nx*=maximum/length;ny*=maximum/length;} + vx_=gap?nx:.65f*vx_+.35f*nx;vy_=gap?ny:.65f*vy_+.35f*ny; + trusted_=measurement;missing_=candidateCount_=0;return {measurement,true,false,gap>0,gap}; + } + ++missing_; + if(missing_>limit_||!finite(expected)){active_=false;return {trusted_,false,false,false,missing_};} + // Preview grows conservatively during a gap; committed gaps are later + // reconstructed from both confirmed endpoints, not these guesses. + expected.confidence=.2f*(1.f-float(missing_)/float(limit_+1)); + float margin=1.f+.2f*float(missing_)/float(limit_);expected.width*=margin;expected.height*=margin; + return {expected,true,true,false,missing_}; + } +}; +} diff --git a/大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp b/大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp new file mode 100644 index 0000000..e1cc2aa --- /dev/null +++ b/大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp @@ -0,0 +1,72 @@ +#pragma once +#import +#import +#include "DYObjectMotion.hpp" +#include "DYTracker.hpp" + +namespace dy { +// Mosaic/object tracking uses small RGB frames and never trains on a lost +// observation. A fresh sequence probes from the last trusted image during gaps. +// All methods are called on the serialized host analysis worker, never UI refresh. +class ObjectTracker { + VNSequenceRequestHandler *sequence_=nil; + VNTrackObjectRequest *request_=nil; + CVPixelBufferRef current_=nullptr,trusted_=nullptr; + ObjectBox trustedBox_{}; + ObjectMotion motion_; + TrackerROI seed_{}; + int width_=0,height_=0; + bool started_=false; + static CGRect rect(ObjectBox b){return CGRectMake(b.cx-b.width*.5f,1-b.cy-b.height*.5f,b.width,b.height);} + void newSequence(ObjectBox box){ + [request_ release];[sequence_ release];sequence_=[VNSequenceRequestHandler new]; + request_=[[VNTrackObjectRequest alloc]initWithDetectedObjectObservation:[VNDetectedObjectObservation observationWithBoundingBox:rect(box)]]; + request_.trackingLevel=VNRequestTrackingLevelAccurate; + } + bool perform(CVPixelBufferRef image,ObjectBox& box){ + NSError *error=nil;BOOL ok=[sequence_ performRequests:@[request_] onCVPixelBuffer:image orientation:kCGImagePropertyOrientationUp error:&error]; + VNDetectedObjectObservation *observation=request_.results.firstObject; + if(!ok||!observation)return false; + CGRect b=observation.boundingBox;box={float(CGRectGetMidX(b)),float(1-CGRectGetMidY(b)),float(b.size.width),float(b.size.height),observation.confidence}; + request_.inputObservation=observation;return true; + } + bool copyInput(const uint8_t *rgba,int w,int h,bool flip){ + if(!rgba||w<16||h<16||w>640||h>640)return false; + if(w!=width_||h!=height_)return false; + CVPixelBufferLockBaseAddress(current_,0);auto *out=(uint8_t*)CVPixelBufferGetBaseAddress(current_);size_t stride=CVPixelBufferGetBytesPerRow(current_); + for(int y=0;y640||h>640)return false; + for(auto p:{¤t_,&trusted_})if(CVPixelBufferCreate(kCFAllocatorDefault,w,h,kCVPixelFormatType_32BGRA,(CFDictionaryRef)@{(id)kCVPixelBufferIOSurfacePropertiesKey:@{}},p)!=kCVReturnSuccess){stop();return false;} + trustedBox_={roi.cx,roi.cy,roi.width,roi.height,1};motion_.reset(trustedBox_,fps);if(!motion_.active()||!copyInput(rgba,w,h,flip)){stop();return false;} + newSequence(trustedBox_);ObjectBox first;if(!perform(current_,first)){stop();return false;} + std::swap(current_,trusted_);started_=true;return true; + } + ObjectStep track(const uint8_t *rgba,int w,int h,bool flip){ + if(!started_||!copyInput(rgba,w,h,flip))return {}; + if(motion_.missing()){ + newSequence(trustedBox_);ObjectBox ignored;if(!perform(trusted_,ignored))return {}; + } + ObjectBox observed{};bool found=perform(current_,observed); + auto step=motion_.advance(observed,found); + if(step.valid&&!step.predicted){trustedBox_=observed;std::swap(current_,trusted_);} + return step; + } + int pendingFrames()const{return motion_.missing();} + TrackResult result(ObjectStep step)const{ + TrackResult r;r.cx=step.box.cx;r.cy=step.box.cy;r.confidence=step.box.confidence;r.valid=step.valid; + // Keep a mosaic's authored aspect and avoid narrowing it across a turn. + r.scale=r.scaleX=r.scaleY=std::max(step.box.width/seed_.width,step.box.height/seed_.height); + r.rotation=0;r.failure=step.valid?TrackFailure::None:TrackFailure::LowConfidence;return r; + } +}; +} diff --git a/开发交接.md b/开发交接.md index 7ffa4f6..3b3f308 100644 --- a/开发交接.md +++ b/开发交接.md @@ -4,6 +4,8 @@ ## 当前基线 +2026-11-06 后续开发中:用户要求继续解决转身偏移,短时预测/重新定位后补齐,并比较 GitHub 成熟算法。当前独立候选已验证 Vision 冻结可信模板能跨过人工短遮挡;尚未接入 APP,正在与 OpenCV CSRT/VitTrack 做同片段比较。[候选与接手入口](research/TrackingRecoveryCandidates.md)。正式安装与线上仍为下述 1.18.1;不要把独立基准当作正式交付。 + **1.18.1(106)已正式安装、发布 Git Release,并更新文章 15792 与主 APP 公测通道。** 实际编译提交 `1cc7f8edb2acb3cfc62bd72447c2956ad7a01601`;发行标签 `v1.18.1-tracking-ui-20261106` 指向 `5fe4b1933e015a906fab36b65dcbc45a0d6bee49`。APP 公证 `2c4e6cba-8178-4a50-9198-83e66be37add` Accepted。安装后 964 项原生检查通过,FCP 独立副本完成两轮 285 帧跟踪及模块/参数交互。 - 跟踪保留 OpenCV LK,Metal 灰度回读减少 75%,目标区域特征检测与金字塔复用减少 CPU 工作。同一 4K 片段算法计算 8.78 秒降至 1.31 秒(约 6.72 倍),轨迹逐帧一致;不含宿主供图、解码、预览与渲染,不等于全流程速度。