WIP: benchmark mature trackers and bounded occlusion recovery before host integration

这个提交包含在:
mac 于 2026-11-06 19:18:22 +08:00 提交
1 父节点 1f430a164a
当前提交 7aece444b0
7 个文件已更改
+204

没有与搜索条件相匹配的文件

+17
查看文件
@@ -0,0 +1,17 @@
# 转身与短时遮挡候选(开发中)
2026-11-06,接续正式 1.18.1(106)。用户要求稳定马赛克、可根据丢失前后运动补齐轨迹,并要求比较 GitHub 成熟算法。正式 APP 尚未改动。
现有 OpenCV LK 负责局部光流与仿射拟合,没有完整对象外观模型;转身时背景特征可能仍给出高置信度。不能仅沿错误光流继续外推。
已建立独立、无界面的本地基准:640×360,25fps,285 帧原实拍;另在第 130—139 帧人为遮住头部。素材和逐帧位置不上传仓库。
- Apple Vision accurate 对照:转身全段跟踪完成;原始连续更新版本在遮挡中会学习遮挡物。候选在失去可信观测时冻结最后可信图像,只从冻结图像试探重新定位。当前候选全段完成,短遮挡测试预测 11 帧后重新确认,热运行约 1ms/帧。此为局部基准,未接入正式宿主。
- 候选运动保护:预测最多 0.8 秒,重新定位需连续两帧确认;限制跳跃和尺寸突变。计划在重新定位后用两端可信位置补齐中间键,尾部未确认预测不得作为成功轨迹提交。此部分尚未接入提交路径。
- [OpenCV CSRT](https://github.com/opencv/opencv_contrib/tree/4.x/modules/tracking):完整相关滤波对象追踪,测试官方 OpenCV 4.11 wheel,后续比较同片段。
- [OpenCV Zoo VitTrack](https://github.com/opencv/opencv_zoo/tree/main/models/object_tracking_vittrack):轻量 Transformer 单目标跟踪,有追踪分数,代码/模型目录 Apache-2.0;官方模型 714726 字节,SHA-256 `2990f0b7cd44d92afa48cd97db6de7be113fc1d9594fddb74e2725c10478e91d`,已核对 LFS 指针与完整下载。后续比较同片段。
- [NanoTrack](https://github.com/HonglinChu/NanoTrack):提供原生 Mac/ncnn 示例;OpenCV Zoo 的对比说明指出其分数不适合判断丢失,暂列次选,不用 README 的速度当本机实测。
- [OSTrack](https://github.com/botaoye/OSTrack):完整 Transformer 方案,原示例以 PyTorch/CUDA 环境为主;不把研究榜单直接当作 Mac 插件速度。
- [ByteTrack](https://github.com/FoundationVision/ByteTrack):关联检测框与多目标身份,需要额外检测器,单独引入不能解决背头没有人脸框的问题。
在同素材准确度、短遮挡恢复、背景误锁、计算开销和宿主交互验证通过前,不替换正式版。
+17
查看文件
@@ -0,0 +1,17 @@
#import <Foundation/Foundation.h>
#include "../../大鱼调色/Plugin/MaskCore/DYObjectTracker.hpp"
#include <fstream>
#include <chrono>
int main(int argc,const char**argv){@autoreleasepool{
if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);double fps=atof(argv[8]);dy::TrackerROI roi{float(atof(argv[4])),float(atof(argv[5])),float(atof(argv[6])),float(atof(argv[7])),0};
std::ifstream input(argv[1],std::ios::binary);std::vector<uint8_t> bytes(size_t(w)*h*4);dy::ObjectTracker tracker;NSMutableArray *frames=[NSMutableArray new];double total=0,maximum=0;
while(input.read((char*)bytes.data(),bytes.size())){@autoreleasepool{
for(size_t i=0;i<bytes.size();i+=4)std::swap(bytes[i],bytes[i+2]);
auto start=std::chrono::steady_clock::now();dy::ObjectStep step;bool initial=frames.count==0;
if(initial){step.valid=tracker.reset(bytes.data(),w,h,false,roi,fps);step.box={roi.cx,roi.cy,roi.width,roi.height,1};}
else step=tracker.track(bytes.data(),w,h,false);
auto r=tracker.result(step);double ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms);
[frames addObject:@{@"cx":@(r.cx),@"cy":@(r.cy),@"width":@(roi.width*r.scaleX),@"height":@(roi.height*r.scaleY),@"confidence":@(r.confidence),@"valid":@(r.valid),@"predicted":@(step.predicted),@"recovered":@(step.recovered),@"gap":@(step.gap),@"ms":@(ms)}];if(!step.valid)break;
}}
NSData *data=[NSJSONSerialization dataWithJSONObject:@{@"frames":frames,@"milliseconds":@(total),@"maximumMs":@(maximum)} options:NSJSONWritingPrettyPrinted error:nil];fwrite(data.bytes,1,data.length,stdout);return 0;
}}
+22
查看文件
@@ -0,0 +1,22 @@
"""Compare official OpenCV trackers on local caller-decoded frames; no GUI/media upload."""
import argparse,json,time,pathlib
import cv2 as cv
import numpy as np
p=argparse.ArgumentParser();p.add_argument('frames');p.add_argument('algorithm',choices=['csrt','vit']);p.add_argument('--model');p.add_argument('--output',required=True);a=p.parse_args()
cv.setNumThreads(2)
if a.algorithm=='csrt':tracker=cv.TrackerCSRT_create()
else:
params=cv.TrackerVit_Params();params.net=a.model;tracker=cv.TrackerVit_create(params)
roi=(round((.486-.085/2)*640),round((.238-.155/2)*360),round(.085*640),round(.155*360))
results=[]
with open(a.frames,'rb') as stream:
while data:=stream.read(640*360*4):
assert len(data)==640*360*4
image=np.frombuffer(data,np.uint8).reshape(360,640,4)[:,:,:3].copy()
start=time.perf_counter()
if not results:tracker.init(image,roi);ok=True;box=roi;score=1.
else:
ok,box=tracker.update(image);score=tracker.getTrackingScore() if a.algorithm=='vit' else float(ok)
elapsed=(time.perf_counter()-start)*1000;x,y,w,h=box
results.append(dict(valid=bool(ok),confidence=float(score),cx=(x+w/2)/640,cy=(y+h/2)/360,width=w/640,height=h/360,ms=elapsed))
pathlib.Path(a.output).write_text(json.dumps(dict(algorithm=a.algorithm,opencv=cv.__version__,threads=2,frames=results,milliseconds=sum(x['ms'] for x in results)),indent=2)+'\n')
+23
查看文件
@@ -0,0 +1,23 @@
// Read-only local media benchmark. No host app, uploads, or persisted images.
#import <Foundation/Foundation.h>
#import <Vision/Vision.h>
#import <CoreVideo/CoreVideo.h>
#include <fstream>
#include <chrono>
int main(int argc,const char**argv){@autoreleasepool{
if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);
CGRect box=CGRectMake(atof(argv[4])-atof(argv[6])/2,1-atof(argv[5])-atof(argv[7])/2,atof(argv[6]),atof(argv[7]));
VNSequenceRequestHandler *sequence=[VNSequenceRequestHandler new];
VNTrackObjectRequest *track=[[VNTrackObjectRequest alloc]initWithDetectedObjectObservation:[VNDetectedObjectObservation observationWithBoundingBox:box]];
track.trackingLevel=atoi(argv[8])?VNRequestTrackingLevelAccurate:VNRequestTrackingLevelFast;
CVPixelBufferRef buffer=nullptr;CVPixelBufferCreate(kCFAllocatorDefault,w,h,kCVPixelFormatType_32BGRA,(__bridge CFDictionaryRef)@{(id)kCVPixelBufferIOSurfacePropertiesKey:@{}},&buffer);
if(!buffer)return 3;std::ifstream input(argv[1],std::ios::binary);std::vector<uint8_t> bytes(size_t(w)*h*4);NSMutableArray *frames=[NSMutableArray new];double total=0,maximum=0;
while(input.read((char*)bytes.data(),bytes.size())){@autoreleasepool{
CVPixelBufferLockBaseAddress(buffer,0);auto dst=(uint8_t*)CVPixelBufferGetBaseAddress(buffer);for(int y=0;y<h;y++)memcpy(dst+y*CVPixelBufferGetBytesPerRow(buffer),bytes.data()+size_t(y)*w*4,w*4);CVPixelBufferUnlockBaseAddress(buffer,0);
auto start=std::chrono::steady_clock::now();NSError *error=nil;BOOL ok=[sequence performRequests:@[track] onCVPixelBuffer:buffer orientation:kCGImagePropertyOrientationUp error:&error];
double ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms);
VNDetectedObjectObservation *r=track.results.firstObject;CGRect b=r.boundingBox;
[frames addObject:@{@"cx":@(CGRectGetMidX(b)),@"cy":@(1-CGRectGetMidY(b)),@"width":@(b.size.width),@"height":@(b.size.height),@"confidence":@(r.confidence),@"valid":@(ok&&r!=nil),@"ms":@(ms)}];if(!ok||!r)break;track.inputObservation=r;
}}
CVPixelBufferRelease(buffer);NSData *data=[NSJSONSerialization dataWithJSONObject:@{@"frames":frames,@"milliseconds":@(total),@"maximumMs":@(maximum)} options:NSJSONWritingPrettyPrinted error:nil];fwrite(data.bytes,1,data.length,stdout);return 0;
}}
@@ -0,0 +1,51 @@
#pragma once
#include <algorithm>
#include <cmath>
namespace dy {
struct ObjectBox {float cx=.5f,cy=.5f,width=.1f,height=.1f,confidence=0;};
struct ObjectStep {ObjectBox box;bool valid=false,predicted=false,recovered=false;int gap=0;};
// Bounded prediction in analysis-frame order (also valid for reverse passes).
// An unconfirmed tail is never an observed trajectory. Host commit trims it.
class ObjectMotion {
ObjectBox trusted_{},candidate_{};
float vx_=0,vy_=0;
int missing_=0,limit_=20,candidateCount_=0;
bool active_=false;
static bool finite(ObjectBox b){return std::isfinite(b.cx)&&std::isfinite(b.cy)&&std::isfinite(b.width)&&std::isfinite(b.height)&&std::isfinite(b.confidence)&&b.width>.003f&&b.height>.003f&&b.width<=1&&b.height<=1&&b.cx>=0&&b.cx<=1&&b.cy>=0&&b.cy<=1;}
public:
void reset(ObjectBox b,double fps){trusted_=b;vx_=vy_=0;missing_=candidateCount_=0;limit_=std::clamp(int(std::ceil(std::clamp(fps,1.,240.)*.8)),2,192);active_=finite(b);}
int missing()const{return missing_;}
bool active()const{return active_;}
ObjectBox prediction(int ahead=1)const{auto b=trusted_;b.cx+=vx_*(missing_+ahead);b.cy+=vy_*(missing_+ahead);return b;}
ObjectStep advance(ObjectBox measurement,bool observed){
if(!active_)return {};
auto expected=prediction();
float size=std::max(trusted_.width,trusted_.height);
float delta=std::hypot(measurement.cx-expected.cx,measurement.cy-expected.cy);
float scaleX=measurement.width/trusted_.width,scaleY=measurement.height/trusted_.height;
bool good=observed&&finite(measurement)&&measurement.confidence>=(missing_?.70f:.55f)&&
delta<size*(missing_?.65f:.45f)&&scaleX>.65f&&scaleX<1.55f&&scaleY>.65f&&scaleY<1.55f;
if(good&&missing_){
if(candidateCount_&&std::hypot(measurement.cx-candidate_.cx,measurement.cy-candidate_.cy)<size*.25f)++candidateCount_;
else candidateCount_=1;
candidate_=measurement;good=candidateCount_>=2;
}else if(!good)candidateCount_=0;
if(good){
int gap=missing_;float dt=float(gap+1);
float nx=(measurement.cx-trusted_.cx)/dt,ny=(measurement.cy-trusted_.cy)/dt;
float maximum=size*.12f;float length=std::hypot(nx,ny);if(length>maximum){nx*=maximum/length;ny*=maximum/length;}
vx_=gap?nx:.65f*vx_+.35f*nx;vy_=gap?ny:.65f*vy_+.35f*ny;
trusted_=measurement;missing_=candidateCount_=0;return {measurement,true,false,gap>0,gap};
}
++missing_;
if(missing_>limit_||!finite(expected)){active_=false;return {trusted_,false,false,false,missing_};}
// Preview grows conservatively during a gap; committed gaps are later
// reconstructed from both confirmed endpoints, not these guesses.
expected.confidence=.2f*(1.f-float(missing_)/float(limit_+1));
float margin=1.f+.2f*float(missing_)/float(limit_);expected.width*=margin;expected.height*=margin;
return {expected,true,true,false,missing_};
}
};
}
@@ -0,0 +1,72 @@
#pragma once
#import <Vision/Vision.h>
#import <CoreVideo/CoreVideo.h>
#include "DYObjectMotion.hpp"
#include "DYTracker.hpp"
namespace dy {
// Mosaic/object tracking uses small RGB frames and never trains on a lost
// observation. A fresh sequence probes from the last trusted image during gaps.
// All methods are called on the serialized host analysis worker, never UI refresh.
class ObjectTracker {
VNSequenceRequestHandler *sequence_=nil;
VNTrackObjectRequest *request_=nil;
CVPixelBufferRef current_=nullptr,trusted_=nullptr;
ObjectBox trustedBox_{};
ObjectMotion motion_;
TrackerROI seed_{};
int width_=0,height_=0;
bool started_=false;
static CGRect rect(ObjectBox b){return CGRectMake(b.cx-b.width*.5f,1-b.cy-b.height*.5f,b.width,b.height);}
void newSequence(ObjectBox box){
[request_ release];[sequence_ release];sequence_=[VNSequenceRequestHandler new];
request_=[[VNTrackObjectRequest alloc]initWithDetectedObjectObservation:[VNDetectedObjectObservation observationWithBoundingBox:rect(box)]];
request_.trackingLevel=VNRequestTrackingLevelAccurate;
}
bool perform(CVPixelBufferRef image,ObjectBox& box){
NSError *error=nil;BOOL ok=[sequence_ performRequests:@[request_] onCVPixelBuffer:image orientation:kCGImagePropertyOrientationUp error:&error];
VNDetectedObjectObservation *observation=request_.results.firstObject;
if(!ok||!observation)return false;
CGRect b=observation.boundingBox;box={float(CGRectGetMidX(b)),float(1-CGRectGetMidY(b)),float(b.size.width),float(b.size.height),observation.confidence};
request_.inputObservation=observation;return true;
}
bool copyInput(const uint8_t *rgba,int w,int h,bool flip){
if(!rgba||w<16||h<16||w>640||h>640)return false;
if(w!=width_||h!=height_)return false;
CVPixelBufferLockBaseAddress(current_,0);auto *out=(uint8_t*)CVPixelBufferGetBaseAddress(current_);size_t stride=CVPixelBufferGetBytesPerRow(current_);
for(int y=0;y<h;++y){const auto *row=rgba+size_t(flip?h-1-y:y)*w*4;auto *dst=out+y*stride;for(int x=0;x<w;++x){dst[x*4]=row[x*4+2];dst[x*4+1]=row[x*4+1];dst[x*4+2]=row[x*4];dst[x*4+3]=255;}}
CVPixelBufferUnlockBaseAddress(current_,0);return true;
}
public:
ObjectTracker()=default;
ObjectTracker(const ObjectTracker&)=delete;
ObjectTracker& operator=(const ObjectTracker&)=delete;
~ObjectTracker(){stop();}
void stop(){[request_ release];request_=nil;[sequence_ release];sequence_=nil;if(current_)CVPixelBufferRelease(current_);if(trusted_)CVPixelBufferRelease(trusted_);current_=trusted_=nullptr;width_=height_=0;started_=false;}
bool reset(const uint8_t *rgba,int w,int h,bool flip,TrackerROI roi,double fps){
stop();seed_=roi;width_=w;height_=h;
if(w<16||h<16||w>640||h>640)return false;
for(auto p:{&current_,&trusted_})if(CVPixelBufferCreate(kCFAllocatorDefault,w,h,kCVPixelFormatType_32BGRA,(CFDictionaryRef)@{(id)kCVPixelBufferIOSurfacePropertiesKey:@{}},p)!=kCVReturnSuccess){stop();return false;}
trustedBox_={roi.cx,roi.cy,roi.width,roi.height,1};motion_.reset(trustedBox_,fps);if(!motion_.active()||!copyInput(rgba,w,h,flip)){stop();return false;}
newSequence(trustedBox_);ObjectBox first;if(!perform(current_,first)){stop();return false;}
std::swap(current_,trusted_);started_=true;return true;
}
ObjectStep track(const uint8_t *rgba,int w,int h,bool flip){
if(!started_||!copyInput(rgba,w,h,flip))return {};
if(motion_.missing()){
newSequence(trustedBox_);ObjectBox ignored;if(!perform(trusted_,ignored))return {};
}
ObjectBox observed{};bool found=perform(current_,observed);
auto step=motion_.advance(observed,found);
if(step.valid&&!step.predicted){trustedBox_=observed;std::swap(current_,trusted_);}
return step;
}
int pendingFrames()const{return motion_.missing();}
TrackResult result(ObjectStep step)const{
TrackResult r;r.cx=step.box.cx;r.cy=step.box.cy;r.confidence=step.box.confidence;r.valid=step.valid;
// Keep a mosaic's authored aspect and avoid narrowing it across a turn.
r.scale=r.scaleX=r.scaleY=std::max(step.box.width/seed_.width,step.box.height/seed_.height);
r.rotation=0;r.failure=step.valid?TrackFailure::None:TrackFailure::LowConfidence;return r;
}
};
}
+2
查看文件
@@ -4,6 +4,8 @@
## 当前基线
2026-11-06 后续开发中:用户要求继续解决转身偏移,短时预测/重新定位后补齐,并比较 GitHub 成熟算法。当前独立候选已验证 Vision 冻结可信模板能跨过人工短遮挡;尚未接入 APP,正在与 OpenCV CSRT/VitTrack 做同片段比较。[候选与接手入口](research/TrackingRecoveryCandidates.md)。正式安装与线上仍为下述 1.18.1;不要把独立基准当作正式交付。
**1.18.1(106)已正式安装、发布 Git Release,并更新文章 15792 与主 APP 公测通道。** 实际编译提交 `1cc7f8edb2acb3cfc62bd72447c2956ad7a01601`;发行标签 `v1.18.1-tracking-ui-20261106` 指向 `5fe4b1933e015a906fab36b65dcbc45a0d6bee49`。APP 公证 `2c4e6cba-8178-4a50-9198-83e66be37add` Accepted。安装后 964 项原生检查通过,FCP 独立副本完成两轮 285 帧跟踪及模块/参数交互。
- 跟踪保留 OpenCV LK,Metal 灰度回读减少 75%,目标区域特征检测与金字塔复用减少 CPU 工作。同一 4K 片段算法计算 8.78 秒降至 1.31 秒(约 6.72 倍),轨迹逐帧一致;不含宿主供图、解码、预览与渲染,不等于全流程速度。