Keep stable OpenCV flow with compact Metal capture and pyramid reuse

这个提交包含在:
mac 于 2026-11-06 18:01:57 +08:00 提交
1 父节点 48ac192d4b
当前提交 1cc7f8edb2
15 个文件已更改
+184 -29

没有与搜索条件相匹配的文件

+6
查看文件
@@ -7,3 +7,9 @@
- 同帧六组合生成图及 Metal 版 12 项算法回归通过。4K 小目标 12 帧从 523.46ms 到 161.89ms,不代表宿主整体速度。实拍、正式宿主、遮挡和界面交互待继续验证。
- UI:DYToolManager 每次单个参数读取都重建完整层状态;Inspector 每组重复读取宿主参数;工作区每次标量变化都重建列表。候选改为单次 Action 快照、复用本次读回、仅在列表模型变化时重建。加入同 Action 写后读与外部编辑失效检查。
- 资料:[Apple GPU 同步](https://developer.apple.com/documentation/metal/resource-synchronization)、[Apple MPS 调优](https://developer.apple.com/documentation/metalperformanceshaders/tuning-hints)、[OpenCV LK](https://docs.opencv.org/3.4.4/dc/d6b/group__video__track.html)。Metal LK 是本产品原有实现,不能声称与 OpenCV 逐帧严格等价。
## 整段验证后的决策
旧 Metal LK 在 4K 实拍转身到第 133 帧时丢轨,CPU 完成 285 帧。该实验不接入正式路径,仅保留可重现基准和拒收证据。最终候选使用 Metal 直接生成 Gray8,读回量从每个 4K 帧约 33.2MB 降为 8.3MB;目标区域特征检测和 OpenCV 双向金字塔复用共同减少 CPU 工作。
同一整段、相同起点,原 CPU 算法处理 8776.57ms,最终候选 1306.09ms,285 帧中心/缩放/旋转差异均为 0。该数字不含供图、解码、磁盘、宿主 UI 与渲染,不等于整体追踪快 6.7 倍。正式宿主及新版整体自检仍待完成。
+1 -1
查看文件
@@ -82,7 +82,7 @@ def main():
if dest.is_symlink():dest.unlink()
if not dest.exists():dest.symlink_to(src)
bridge=ROOT/'tools/tracking/DYOpenCVTracker.cpp';obj=libs/'DYOpenCVTracker.o';runtime=libs/'libDayuOpenCVTracker.a'
identity=hashlib.sha256(bridge.read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYOpenCVTracker.hpp').read_bytes()+marker.read_bytes()).hexdigest()
identity=hashlib.sha256(bridge.read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYOpenCVTracker.hpp').read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYTracker.hpp').read_bytes()+marker.read_bytes()).hexdigest()
bridgeMarker=CACHE/'bridge-build.json'
if not runtime.exists() or not bridgeMarker.exists() or json.loads(bridgeMarker.read_text()).get('identity')!=identity:
run(['xcrun','clang++','-std=c++17','-arch','arm64','-mmacosx-version-min=14.6','-O3','-fvisibility=hidden',
+11 -4
查看文件
@@ -8,6 +8,7 @@
namespace dy {
struct OpenCVTracker::Impl {
cv::Mat previous;
std::vector<cv::Mat> previousPyramid;
std::shared_ptr<FlowBackend> flow;
std::vector<cv::Point2f> points, polygon;
std::vector<FlowPoint> mask;
@@ -19,6 +20,7 @@ struct OpenCVTracker::Impl {
static cv::Mat gray8(const GrayFrame& f) {
if(!f.valid())return {};
if(f.pixels.empty())return cv::Mat(f.height,f.width,CV_8UC1,const_cast<uint8_t*>(f.bytes.data())).clone();
cv::Mat out(f.height,f.width,CV_8UC1);
for(size_t i=0;i<f.pixels.size();++i) {
float x=f.pixels[i];
@@ -59,7 +61,7 @@ void OpenCVTracker::setFlowBackend(std::shared_ptr<FlowBackend> flow){impl_->flo
const char* OpenCVTracker::version(){return CV_VERSION;}
TrackResult OpenCVTracker::result()const{return impl_->value;}
bool OpenCVTracker::active()const{return impl_->running;}
void OpenCVTracker::stop(){impl_->running=false;impl_->value.valid=false;}
void OpenCVTracker::stop(){impl_->running=false;impl_->value.valid=false;impl_->previous.release();impl_->previousPyramid.clear();impl_->points.clear();}
bool OpenCVTracker::reset(const GrayFrame& f,TrackerROI roi) {
auto& t=*impl_;t.running=false;t.frameIndex=1;t.points.clear();t.polygon.clear();
@@ -81,6 +83,7 @@ bool OpenCVTracker::reset(const GrayFrame& f,TrackerROI roi) {
t.detect();t.value.features=int(t.points.size());t.value.inliers=t.value.features;
if(t.points.size()<4){t.fail(TrackFailure::InsufficientTexture);return false;}
if(t.flow&&!t.flow->reset(f)){t.fail(TrackFailure::Backend);return false;}
if(!t.flow)cv::buildOpticalFlowPyramid(t.previous,t.previousPyramid,{21,21},3);
t.running=true;t.value.valid=true;t.value.confidence=1;return true;
} catch(const cv::Exception&){t.fail(TrackFailure::Backend);return false;}
}
@@ -96,6 +99,7 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) {
t.value.features=int(t.points.size());t.value.matches=0;t.value.inliers=0;t.value.medianError=0;
if(t.points.size()<4)return t.fail(TrackFailure::InsufficientTexture);
std::vector<cv::Point2f> src,dst,movingSrc,movingDst;
std::vector<cv::Mat> nextPyramid;
if(t.flow){
std::vector<FlowPoint> input;input.reserve(t.points.size());
for(auto p:t.points)input.push_back({p.x,p.y});
@@ -106,8 +110,11 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) {
std::vector<cv::Point2f> forward,backward;
std::vector<uchar> fs,bs;std::vector<float> fe,be;
cv::TermCriteria termination(cv::TermCriteria::COUNT|cv::TermCriteria::EPS,30,.01);
cv::calcOpticalFlowPyrLK(t.previous,next,t.points,forward,fs,fe,{21,21},3,termination,0,1e-4);
cv::calcOpticalFlowPyrLK(next,t.previous,forward,backward,bs,be,{21,21},3,termination,0,1e-4);
// Build each frame and its gradients once; reuse them for both flow
// directions and as the previous frame on the next analysis callback.
cv::buildOpticalFlowPyramid(next,nextPyramid,{21,21},3);
cv::calcOpticalFlowPyrLK(t.previousPyramid,nextPyramid,t.points,forward,fs,fe,{21,21},3,termination,0,1e-4);
cv::calcOpticalFlowPyrLK(nextPyramid,t.previousPyramid,forward,backward,bs,be,{21,21},3,termination,0,1e-4);
for(size_t i=0;i<t.points.size();++i) {
// Checking both statuses also rejects a failed backward solve.
if(!fs[i]||!bs[i]||cv::norm(backward[i]-t.points[i])>=1.)continue;
@@ -150,7 +157,7 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) {
t.value.scaleX=float(totalX);t.value.scaleY=float(totalY);t.value.scale=std::sqrt(t.value.scaleX*t.value.scaleY);
t.value.rotation+=float(angle);t.value.confidence=confidence;t.value.valid=true;t.value.failure=TrackFailure::None;
t.points.clear();for(size_t i=0;i<dst.size();++i)t.points.push_back(dst[i]+(center-cv::Point2f(float(tx),float(ty))));
t.previous=next;++t.frameIndex;return t.value;
t.previous=next;t.previousPyramid=std::move(nextPyramid);++t.frameIndex;return t.value;
}catch(const cv::Exception&){return t.fail(TrackFailure::Backend);}
}
}
+4 -3
查看文件
@@ -6,16 +6,17 @@
#include <fstream>
#include <chrono>
int main(int argc,const char **argv){@autoreleasepool{
if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);if(w<16||h<16||w>8192||h>8192)return 2;
if(argc!=9&&argc!=10)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);if(w<16||h<16||w>8192||h>8192)return 2;
bool compact=argc==10&&strcmp(argv[9],"compact")==0;
dy::TrackerROI roi{float(atof(argv[5])),float(atof(argv[6])),float(atof(argv[7])),float(atof(argv[8])),0};
NSMutableArray *runs=[NSMutableArray array];
for(int gpu=0;gpu<2;++gpu){
for(int gpu=0;gpu<(argc==10?1:2);++gpu){
dy::OpenCVTracker tracker;if(gpu){auto b=DYToolCreateMetalFlow([NSURL fileURLWithPath:[NSString stringWithUTF8String:argv[4]]],nullptr);if(!b)return 3;tracker.setFlowBackend(b);}
std::ifstream input(argv[1],std::ios::binary);if(!input)return 2;
std::vector<uint8_t> bytes(size_t(w)*h);dy::GrayFrame f{w,h,std::vector<float>(bytes.size())};
NSMutableArray *frames=[NSMutableArray array];double total=0,maximum=0;
while(input.read((char*)bytes.data(),bytes.size())){
for(size_t i=0;i<bytes.size();++i)f.pixels[i]=bytes[i]/255.f;
if(compact){f.pixels.clear();f.bytes=bytes;}else for(size_t i=0;i<bytes.size();++i)f.pixels[i]=bytes[i]/255.f;
auto start=std::chrono::steady_clock::now();bool ok=frames.count?tracker.track(f).valid:tracker.reset(f,roi);
double ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms);auto r=tracker.result();
[frames addObject:@{@"valid":@(ok),@"cx":@(r.cx),@"cy":@(r.cy),@"scaleX":@(r.scaleX),@"scaleY":@(r.scaleY),@"rotation":@(r.rotation),@"confidence":@(r.confidence),@"failure":@((int)r.failure)}];if(!ok)break;
+10
查看文件
@@ -75,5 +75,15 @@ int main() {
auto ms=std::chrono::duration<double,std::milli>(std::chrono::steady_clock::now()-begin).count();
std::cerr<<"4K generated images plus tracking: "<<ms<<"ms\n";
}
{
dy::OpenCVTracker floats,bytes;bool ok=true;double difference=0;
for(int i=0;i<12;++i){auto f=frame(640,480,1.25*i,.5*i);dy::GrayFrame b;b.width=f.width;b.height=f.height;
b.bytes.resize(f.pixels.size());for(size_t p=0;p<f.pixels.size();++p){b.bytes[p]=uint8_t(std::lround(f.pixels[p]*255));f.pixels[p]=b.bytes[p]/255.f;}
b.flipVertically();b.flipVertically();
ok&=i?floats.track(f).valid&&bytes.track(b).valid:floats.reset(f,{.5,.5,.4,.4,0})&&bytes.reset(b,{.5,.5,.4,.4,0});
auto a=floats.result(),c=bytes.result();difference=std::max(difference,std::hypot(double(a.cx-c.cx)*640,double(a.cy-c.cy)*480));
}
check("compact-gpu-byte-input-matches-quantized-float-and-flip",ok&&difference<1e-5,difference);
}
std::cout<<"],\"pass\":"<<(failed?"false":"true")<<",\"count\":"<<count<<",\"runtimeTargetAppUsed\":false}\n";return failed?1:0;
}
+111
查看文件
@@ -0,0 +1,111 @@
{
"candidateVersion": "1.18.1",
"build": 106,
"sourceBaseline": "ca4abaf2c3dc67c482aaf7182668c3d6d7098268",
"scope": "Local caller-decoded real 4K clip, 25fps, identical 285 frames and initial head ROI. Timed reset/track only; excludes decode, disk I/O, FxAnalyzer scheduling, preview and rendering. This is not end-to-end host speed.",
"resolution": [
3840,
2160
],
"baseline": {
"frames": 285,
"milliseconds": 8776.568616
},
"candidate": {
"frames": 285,
"milliseconds": 1306.087993999999
},
"speedup": 6.7197376105732785,
"maximumTrajectoryDifferences": {
"cx": 0.0,
"cy": 0.0,
"scaleX": 0,
"scaleY": 0,
"rotation": 0
},
"opencvRegression": {
"version": "4.5.1",
"checks": [
{
"name": "pinned-public-opencv-version",
"pass": true,
"error": 0
},
{
"name": "300-feature-capacity",
"pass": true,
"error": 300
},
{
"name": "translation-rotation-independent-xy-scale",
"pass": true,
"error": 0.00329193
},
{
"name": "free-path-outside-creation-box-12-frame-subpixel",
"pass": true,
"error": 0.014753
},
{
"name": "periodic-redetection-keeps-static-position",
"pass": true,
"error": 0
},
{
"name": "unrelated-cut-stops-with-last-trusted-position",
"pass": true,
"error": 0
},
{
"name": "flat-mask-rejected",
"pass": true,
"error": 0
},
{
"name": "moving-majority-prefers-subject-over-static-background",
"pass": true,
"error": 0.00237274
},
{
"name": "independent-tracker-objects-do-not-share-motion",
"pass": true,
"error": 0
},
{
"name": "invalid-frame-preserves-authored-geometry",
"pass": true,
"error": 0
},
{
"name": "nonfinite-source-rejected",
"pass": true,
"error": 0
},
{
"name": "4k-complete-source-frame",
"pass": true,
"error": 0.000107325
},
{
"name": "compact-gpu-byte-input-matches-quantized-float-and-flip",
"pass": true,
"error": 0
}
],
"pass": true,
"count": 13,
"runtimeTargetAppUsed": false
},
"rejectedMetalLK": {
"completedFrames": 132,
"failureFrameIndex": 132,
"reason": "Old Metal LK loses target during turn while OpenCV completes whole clip. Not enabled in production."
},
"productionChanges": [
"Metal emits compact gray8 instead of float; readback 75% smaller",
"Feature detection restricted to target bounds plus halo at full resolution",
"OpenCV pyramids and derivatives reused across both directions and adjacent frames",
"CPU OpenCV LK retained; no experimental optical-flow backend enabled"
],
"hostQA": "Pending signed/notarized installed candidate"
}
+2
查看文件
@@ -31,6 +31,7 @@ public:
~MetalFlow(){for(int i=0;i<2;i++)[frames_[i] release];[points_ release];[results_ release];[pyramid_ release];[flow_ release];[queue_ release];[device_ release];}
bool ready()const{return device_&&queue_&&pyramid_&&flow_&&flow_.threadExecutionWidth==32;}
bool reset(const dy::GrayFrame& f) override {
if(f.valid()&&f.pixels.empty())return reset(f.expanded());
if(!ready()||!f.valid())return false;
info_={};info_.levels=1;info_.width[0]=f.width;info_.height[0]=f.height;
int total=f.width*f.height;
@@ -47,6 +48,7 @@ public:
id<MTLCommandBuffer>b=[queue_ commandBuffer];pyramid(b,frames_[0]);return finish(b);
}
bool advance(const dy::GrayFrame& f,const std::vector<dy::FlowPoint>& points,std::vector<dy::FlowMatch>& matches) override {
if(f.valid()&&f.pixels.empty())return advance(f.expanded(),points,matches);
if(!f.valid()||f.width!=info_.width[0]||f.height!=info_.height[0]||points.empty()||points.size()>dy::TrackerFeatureLimit)return false;
int next=1-previous_;memcpy(frames_[next].contents,f.pixels.data(),f.pixels.size()*sizeof(float));
memcpy(points_.contents,points.data(),points.size()*sizeof(dy::FlowPoint));
+7 -3
查看文件
@@ -2,7 +2,11 @@
// CF asks the host for video gamma 2.2, then Lanczos-scales to RGBA8 before
// OpenCV gray conversion. Adapt our linear host contract in a separate texture.
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error {
return [self toolGrayFrame:tile preview:nil error:error];
auto result=[self toolGrayFrame:tile preview:nil error:error];
// The diagnostic/legacy caller expects normalized floats. Actual analysis
// keeps the GPU-produced bytes through OpenCV, without float expansion.
if(result.valid()&&result.pixels.empty()){result.pixels.resize(result.bytes.size());for(size_t i=0;i<result.bytes.size();++i)result.pixels[i]=result.bytes[i]/255.f;result.bytes.clear();}
return result;
}
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile preview:(NSDictionary**)preview error:(NSError**)error {
@synchronized(_toolCaptureDevices){
@@ -27,7 +31,7 @@
id<MTLTexture>encoded=entry[@"encoded"],bytes=entry[@"bytes"];
if(!encoded||encoded.width!=source.width||encoded.height!=source.height||encoded.pixelFormat!=format){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format width:source.width height:source.height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;encoded=[[device newTextureWithDescriptor:d]autorelease];if(!encoded)return {};entry[@"encoded"]=encoded;}
if(!bytes||bytes.width!=width||bytes.height!=height){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:width height:height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;bytes=[[device newTextureWithDescriptor:d]autorelease];if(!bytes)return {};entry[@"bytes"]=bytes;}
id<MTLBuffer>buffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height*sizeof(float)){buffer=[[device newBufferWithLength:width*height*sizeof(float) options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;}
id<MTLBuffer>buffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height){buffer=[[device newBufferWithLength:width*height options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;}
id<MTLCommandBuffer>commands=[entry[@"queue"]commandBuffer];id<MTLComputeCommandEncoder>encoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gamma"]];[encoder setTexture:source atIndex:0];[encoder setTexture:encoded atIndex:1];[encoder dispatchThreads:MTLSizeMake(source.width,source.height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding];
[entry[@"scale"]encodeToCommandBuffer:commands sourceTexture:encoded destinationTexture:bytes];
vector_uint3 info={(unsigned)width,(unsigned)height,(unsigned)(tile.imageOrigin==kFxImageOrigin_BOTTOM_LEFT)};encoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gray"]];[encoder setTexture:bytes atIndex:0];[encoder setBuffer:buffer offset:0 atIndex:0];[encoder setBytes:&info length:sizeof(info)atIndex:1];[encoder dispatchThreads:MTLSizeMake(width,height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding];
@@ -58,6 +62,6 @@
*preview=@{@"rgba":[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length],
@"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)};
}
dy::GrayFrame result{int(width),int(height),std::vector<float>(width*height)};memcpy(result.pixels.data(),buffer.contents,width*height*sizeof(float));return result;
dy::GrayFrame result;result.width=int(width);result.height=int(height);const auto*begin=static_cast<const uint8_t*>(buffer.contents);result.bytes.assign(begin,begin+width*height);return result;
}
}
+3 -10
查看文件
@@ -238,18 +238,11 @@
NSDictionary *preview=nil;
dy::GrayFrame frame=[self toolGrayFrame:tile preview:_trackingSingle?nullptr:&preview error:error];
if(!frame.valid())return [self stopToolAnalysis:@"宿主未返回完整源画面" error:error];
if(!_trackingSamples.count){if(_trackingReferenceFrame.valid()&&frame.width==_trackingReferenceFrame.width&&frame.height==_trackingReferenceFrame.height){double same=0,flip=0;size_t count=0;for(int y=0;y<frame.height;y+=8)for(int x=0;x<frame.width;x+=8){float p=frame.pixels[y*frame.width+x];same+=fabs(p-_trackingReferenceFrame.pixels[y*frame.width+x]);flip+=fabs(p-_trackingReferenceFrame.pixels[(frame.height-1-y)*frame.width+x]);count++;}same/=MAX(size_t(1),count);flip/=MAX(size_t(1),count);_trackingFlip=flip<.035&&flip+.008<same&&flip<same*.8;}os_log(OS_LOG_DEFAULT,"dayu_tool_track_seed width=%{public}d height=%{public}d flip=%{public}d time=%{public}.6f",frame.width,frame.height,(int)_trackingFlip,CMTimeGetSeconds(time));}
if(_trackingFlip)for(int y=0;y<frame.height/2;y++)for(int x=0;x<frame.width;x++)std::swap(frame.pixels[y*frame.width+x],frame.pixels[(frame.height-1-y)*frame.width+x]);
if(!_trackingSamples.count){if(_trackingReferenceFrame.valid()&&frame.width==_trackingReferenceFrame.width&&frame.height==_trackingReferenceFrame.height){double same=0,flip=0;size_t count=0;for(int y=0;y<frame.height;y+=8)for(int x=0;x<frame.width;x+=8){float p=frame.at(y*frame.width+x);same+=fabs(p-_trackingReferenceFrame.at(y*frame.width+x));flip+=fabs(p-_trackingReferenceFrame.at((frame.height-1-y)*frame.width+x));count++;}same/=MAX(size_t(1),count);flip/=MAX(size_t(1),count);_trackingFlip=flip<.035&&flip+.008<same&&flip<same*.8;}os_log(OS_LOG_DEFAULT,"dayu_tool_track_seed width=%{public}d height=%{public}d flip=%{public}d time=%{public}.6f",frame.width,frame.height,(int)_trackingFlip,CMTimeGetSeconds(time));}
if(_trackingFlip)frame.flipVertically();
BOOL stamp=_trackingStampMotion.active;
dy::TrackResult target,source;
if(!_trackingSamples.count){
// Full-resolution mask analysis benefits from GPU flow. Tiny inputs and
// the separately tuned clone-stamp pair retain the CPU implementation.
std::shared_ptr<dy::FlowBackend> flow;
if(!stamp&&size_t(frame.width)*frame.height>=1920u*1080u)
flow=DYToolCreateMetalFlow([[NSBundle bundleForClass:self.class]URLForResource:@"default" withExtension:@"metallib"],nullptr);
_toolTracker.setFlowBackend(flow);
os_log(OS_LOG_DEFAULT,"dayu_tool_track_backend metal=%{public}d",bool(flow));
dy::TrackerROI targetROI={_trackingSeed.centerX,_trackingSeed.centerY,_trackingSeed.width,_trackingSeed.height,_trackingSeed.angle};
if(!_toolTracker.reset(frame,targetROI))
return [self finishToolAnalysis:stamp?@"覆盖区域内纹理不足,原图章已保留":@"蒙版内纹理不足,原蒙版已保留" error:error];
@@ -313,7 +306,7 @@
}
- (BOOL)cleanupAnalysis:(NSError**)error {
if(!_trackingActive.load()||_trackingFinishing.exchange(true))return YES;BOOL changed=_trackingSamples.count>1;if(changed&&!DYMergeTracking(_trackingCandidate,_trackingSamples,_trackingFrameDuration)){[_trackingMessage release];_trackingMessage=[@"轨迹合并失败,原蒙版已保留" copy];_trackingDiscard=YES;}
_toolTracker.stop();_toolTracker.setFlowBackend({});
_toolTracker.stop();
if(_trackingStampMotion.active)_stampSourceTracker.stop();
uint64_t generation=_trackingGeneration;BOOL second=_trackingBoth&&!_trackingBackward&&!_trackingCancelled.load()&&!_trackingDiscard;
dispatch_async(dispatch_get_main_queue(),^{if(self->_trackingGeneration!=generation)return;
@@ -63,6 +63,9 @@
for(NSUInteger y=0;y<h;y++)for(NSUInteger x=0;x<w;x++){float encoded=x%3==1?.8f:.2f,linear=powf((encoded+.055f)/1.055f,2.4f);auto i=(y*w+x)*4;pixels[i]=pixels[i+1]=pixels[i+2]=linear;pixels[i+3]=1;}
[[fine metalTextureForDevice:gpu]replaceRegion:MTLRegionMake2D(0,0,w,h) mipmapLevel:0 withBytes:pixels.data() bytesPerRow:w*8];auto gray=[plugin toolGrayFrame:fine error:nil];double maximum=0;for(NSUInteger y=0;y<(NSUInteger)gray.height;y++)for(NSUInteger x=0;x<(NSUInteger)gray.width;x++){double code=x%3==1?.8:.2;double expected=std::round(std::pow(std::pow((code+.055)/1.055,2.4),1/2.2)*255)/255.;maximum=std::max(maximum,fabs(double(gray.pixels[y*gray.width+x])-expected));}
run.record(@"tracking-4k-frame-preserves-full-detail-for-opencv",gray.valid()&&gray.width==3840&&gray.height==2160&&maximum<.002,@{@"width":@(gray.width),@"height":@(gray.height),@"maximumError":@(maximum)});
auto compact=[plugin toolGrayFrame:fine preview:nullptr error:nil];BOOL identical=compact.valid()&&compact.pixels.empty()&&compact.bytes.size()==w*h;
for(size_t i=0;i<compact.bytes.size()&&identical;++i)identical=compact.at(i)==gray.pixels[i];
run.record(@"tracking-metal-gray8-compact-readback-matches-float-reference",identical,@{@"readbackBytes":@(compact.bytes.size()),@"previousReadbackBytes":@(gray.pixels.size()*sizeof(float))});
}
// UI aliases of a shared object are one tracking target. Switching from the
// global editor to either linked module must not cancel a valid analysis.
@@ -27,6 +27,7 @@
- (BOOL)analyzeFrame:(FxImageTile*)frame atTime:(CMTime)time error:(NSError**)error;
- (BOOL)cleanupAnalysis:(NSError**)error;
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error;
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile preview:(NSDictionary**)preview error:(NSError**)error;
@end
@interface DYToolCanvasProbe : DYToolTrackingProbe
+2 -2
查看文件
@@ -54,9 +54,9 @@ kernel void dyToolEncodeGamma22(texture2d<float,access::read> source [[texture(0
if(pos.x>=output.get_width()||pos.y>=output.get_height())return;float4 pixel=source.read(pos);float alpha=max(pixel.a,0.f);float3 straight=alpha>0?pixel.rgb/alpha:float3(0);output.write(float4(pow(max(straight,float3(0)),float3(1.f/2.2f))*alpha,alpha),pos);
}
// Lanczos RGBA8 -> top-first OpenCV Gray8. Analysis never changes color output.
kernel void dyToolCapture(texture2d<float> source [[texture(0)]],device float* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){
kernel void dyToolCapture(texture2d<float> source [[texture(0)]],device uchar* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){
if(pos.x>=info.x||pos.y>=info.y)return;uint2 point=uint2(pos.x,info.z?info.y-1-pos.y:pos.y);uint3 rgb=uint3(rint(source.read(point).rgb*255.f));
pixels[pos.y*info.x+pos.x]=float((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15)/255.f;
pixels[pos.y*info.x+pos.x]=uchar((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15);
}
// Reuse the analysis source texture for the lightweight color preview.
+3 -3
查看文件
@@ -17,9 +17,9 @@
<key>CFBundlePackageType</key>
<string>XPC!</string>
<key>CFBundleShortVersionString</key>
<string>1.18.0</string>
<string>1.18.1</string>
<key>CFBundleVersion</key>
<string>105</string>
<string>106</string>
<key>NSHumanReadableCopyright</key>
<string></string>
<key>PlugInKit</key>
@@ -71,7 +71,7 @@
<key>uuid</key>
<string>1C17B73B-F229-4C09-A434-EAF6DFF5F22A</string>
<key>version</key>
<string>1.18.0</string>
<string>1.18.1</string>
</dict>
<dict>
<key>className</key>
+18 -1
查看文件
@@ -23,9 +23,24 @@ constexpr float TrackerFitTolerance = 3.f;
struct GrayFrame {
int width = 0, height = 0;
std::vector<float> pixels;
// Compact GPU analysis input. Legacy/synthetic callers keep using floats.
std::vector<uint8_t> bytes;
bool valid() const {
return width >= 16 && height >= 16 &&
pixels.size() == static_cast<size_t>(width) * height;
(pixels.size() == static_cast<size_t>(width) * height ||
(pixels.empty() && bytes.size() == static_cast<size_t>(width) * height));
}
float at(size_t i) const {return pixels.empty()?bytes[i]/255.f:pixels[i];}
GrayFrame expanded() const {
GrayFrame f{width,height,std::vector<float>(bytes.size())};
for(size_t i=0;i<bytes.size();++i)f.pixels[i]=bytes[i]/255.f;
return f;
}
void flipVertically() {
for(int y=0;y<height/2;++y)for(int x=0;x<width;++x){
size_t a=size_t(y)*width+x,b=size_t(height-1-y)*width+x;
if(pixels.empty())std::swap(bytes[a],bytes[b]);else std::swap(pixels[a],pixels[b]);
}
}
static GrayFrame fromRGBA8(const uint8_t* data, int w, int h, size_t stride) {
if (!data || w <= 0 || h <= 0 || stride < static_cast<size_t>(w) * 4) return {};
@@ -347,6 +362,7 @@ public:
void setBackend(std::shared_ptr<FlowBackend> b) {backend_=std::move(b);active_=false;}
void setMask(std::vector<P> p) {mask_=std::move(p);}
bool reset(const GrayFrame& frame,TrackerROI roi) {
if(frame.valid()&&frame.pixels.empty())return reset(frame.expanded(),roi);
active_=false;points_.clear();previous_.clear();cumulative_={};frameIndex_=0;
result_={roi.cx,roi.cy,1,0,0,0,0,false};
if (!frame.valid() || !std::isfinite(roi.cx) || !std::isfinite(roi.cy) ||
@@ -361,6 +377,7 @@ public:
return true;
}
TrackResult track(const GrayFrame& frame) {
if(frame.valid()&&frame.pixels.empty())return track(frame.expanded());
if (!active_ || !frame.valid() || frame.width!=width_ || frame.height!=height_) {
result_.valid=false;result_.confidence=0;result_.failure=TrackFailure::InvalidFrame;active_=false;return result_;
}
+2 -2
查看文件
@@ -17,9 +17,9 @@
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.18.0</string>
<string>1.18.1</string>
<key>CFBundleVersion</key>
<string>105</string>
<string>106</string>
<key>LSMinimumSystemVersion</key>
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
<key>NSHumanReadableCopyright</key>