diff --git a/research/TrackingMetal1.18.1.md b/research/TrackingMetal1.18.1.md index 7e551d0..8dc2d93 100644 --- a/research/TrackingMetal1.18.1.md +++ b/research/TrackingMetal1.18.1.md @@ -7,3 +7,9 @@ - 同帧六组合生成图及 Metal 版 12 项算法回归通过。4K 小目标 12 帧从 523.46ms 到 161.89ms,不代表宿主整体速度。实拍、正式宿主、遮挡和界面交互待继续验证。 - UI:DYToolManager 每次单个参数读取都重建完整层状态;Inspector 每组重复读取宿主参数;工作区每次标量变化都重建列表。候选改为单次 Action 快照、复用本次读回、仅在列表模型变化时重建。加入同 Action 写后读与外部编辑失效检查。 - 资料:[Apple GPU 同步](https://developer.apple.com/documentation/metal/resource-synchronization)、[Apple MPS 调优](https://developer.apple.com/documentation/metalperformanceshaders/tuning-hints)、[OpenCV LK](https://docs.opencv.org/3.4.4/dc/d6b/group__video__track.html)。Metal LK 是本产品原有实现,不能声称与 OpenCV 逐帧严格等价。 + +## 整段验证后的决策 + +旧 Metal LK 在 4K 实拍转身到第 133 帧时丢轨,CPU 完成 285 帧。该实验不接入正式路径,仅保留可重现基准和拒收证据。最终候选使用 Metal 直接生成 Gray8,读回量从每个 4K 帧约 33.2MB 降为 8.3MB;目标区域特征检测和 OpenCV 双向金字塔复用共同减少 CPU 工作。 + +同一整段、相同起点,原 CPU 算法处理 8776.57ms,最终候选 1306.09ms,285 帧中心/缩放/旋转差异均为 0。该数字不含供图、解码、磁盘、宿主 UI 与渲染,不等于整体追踪快 6.7 倍。正式宿主及新版整体自检仍待完成。 diff --git a/tools/build_tracking_opencv.py b/tools/build_tracking_opencv.py index ae80f4b..39ced66 100644 --- a/tools/build_tracking_opencv.py +++ b/tools/build_tracking_opencv.py @@ -82,7 +82,7 @@ def main(): if dest.is_symlink():dest.unlink() if not dest.exists():dest.symlink_to(src) bridge=ROOT/'tools/tracking/DYOpenCVTracker.cpp';obj=libs/'DYOpenCVTracker.o';runtime=libs/'libDayuOpenCVTracker.a' - identity=hashlib.sha256(bridge.read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYOpenCVTracker.hpp').read_bytes()+marker.read_bytes()).hexdigest() + identity=hashlib.sha256(bridge.read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYOpenCVTracker.hpp').read_bytes()+(ROOT/'大鱼调色/Plugin/MaskCore/DYTracker.hpp').read_bytes()+marker.read_bytes()).hexdigest() bridgeMarker=CACHE/'bridge-build.json' if not runtime.exists() or not bridgeMarker.exists() or json.loads(bridgeMarker.read_text()).get('identity')!=identity: run(['xcrun','clang++','-std=c++17','-arch','arm64','-mmacosx-version-min=14.6','-O3','-fvisibility=hidden', diff --git a/tools/tracking/DYOpenCVTracker.cpp b/tools/tracking/DYOpenCVTracker.cpp index 52f4d62..5f63ca5 100644 --- a/tools/tracking/DYOpenCVTracker.cpp +++ b/tools/tracking/DYOpenCVTracker.cpp @@ -8,6 +8,7 @@ namespace dy { struct OpenCVTracker::Impl { cv::Mat previous; + std::vector previousPyramid; std::shared_ptr flow; std::vector points, polygon; std::vector mask; @@ -19,6 +20,7 @@ struct OpenCVTracker::Impl { static cv::Mat gray8(const GrayFrame& f) { if(!f.valid())return {}; + if(f.pixels.empty())return cv::Mat(f.height,f.width,CV_8UC1,const_cast(f.bytes.data())).clone(); cv::Mat out(f.height,f.width,CV_8UC1); for(size_t i=0;i flow){impl_->flo const char* OpenCVTracker::version(){return CV_VERSION;} TrackResult OpenCVTracker::result()const{return impl_->value;} bool OpenCVTracker::active()const{return impl_->running;} -void OpenCVTracker::stop(){impl_->running=false;impl_->value.valid=false;} +void OpenCVTracker::stop(){impl_->running=false;impl_->value.valid=false;impl_->previous.release();impl_->previousPyramid.clear();impl_->points.clear();} bool OpenCVTracker::reset(const GrayFrame& f,TrackerROI roi) { auto& t=*impl_;t.running=false;t.frameIndex=1;t.points.clear();t.polygon.clear(); @@ -81,6 +83,7 @@ bool OpenCVTracker::reset(const GrayFrame& f,TrackerROI roi) { t.detect();t.value.features=int(t.points.size());t.value.inliers=t.value.features; if(t.points.size()<4){t.fail(TrackFailure::InsufficientTexture);return false;} if(t.flow&&!t.flow->reset(f)){t.fail(TrackFailure::Backend);return false;} + if(!t.flow)cv::buildOpticalFlowPyramid(t.previous,t.previousPyramid,{21,21},3); t.running=true;t.value.valid=true;t.value.confidence=1;return true; } catch(const cv::Exception&){t.fail(TrackFailure::Backend);return false;} } @@ -96,6 +99,7 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) { t.value.features=int(t.points.size());t.value.matches=0;t.value.inliers=0;t.value.medianError=0; if(t.points.size()<4)return t.fail(TrackFailure::InsufficientTexture); std::vector src,dst,movingSrc,movingDst; + std::vector nextPyramid; if(t.flow){ std::vector input;input.reserve(t.points.size()); for(auto p:t.points)input.push_back({p.x,p.y}); @@ -106,8 +110,11 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) { std::vector forward,backward; std::vector fs,bs;std::vector fe,be; cv::TermCriteria termination(cv::TermCriteria::COUNT|cv::TermCriteria::EPS,30,.01); - cv::calcOpticalFlowPyrLK(t.previous,next,t.points,forward,fs,fe,{21,21},3,termination,0,1e-4); - cv::calcOpticalFlowPyrLK(next,t.previous,forward,backward,bs,be,{21,21},3,termination,0,1e-4); + // Build each frame and its gradients once; reuse them for both flow + // directions and as the previous frame on the next analysis callback. + cv::buildOpticalFlowPyramid(next,nextPyramid,{21,21},3); + cv::calcOpticalFlowPyrLK(t.previousPyramid,nextPyramid,t.points,forward,fs,fe,{21,21},3,termination,0,1e-4); + cv::calcOpticalFlowPyrLK(nextPyramid,t.previousPyramid,forward,backward,bs,be,{21,21},3,termination,0,1e-4); for(size_t i=0;i=1.)continue; @@ -150,7 +157,7 @@ TrackResult OpenCVTracker::track(const GrayFrame& f) { t.value.scaleX=float(totalX);t.value.scaleY=float(totalY);t.value.scale=std::sqrt(t.value.scaleX*t.value.scaleY); t.value.rotation+=float(angle);t.value.confidence=confidence;t.value.valid=true;t.value.failure=TrackFailure::None; t.points.clear();for(size_t i=0;i #include int main(int argc,const char **argv){@autoreleasepool{ - if(argc!=9)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);if(w<16||h<16||w>8192||h>8192)return 2; + if(argc!=9&&argc!=10)return 2;int w=atoi(argv[2]),h=atoi(argv[3]);if(w<16||h<16||w>8192||h>8192)return 2; + bool compact=argc==10&&strcmp(argv[9],"compact")==0; dy::TrackerROI roi{float(atof(argv[5])),float(atof(argv[6])),float(atof(argv[7])),float(atof(argv[8])),0}; NSMutableArray *runs=[NSMutableArray array]; - for(int gpu=0;gpu<2;++gpu){ + for(int gpu=0;gpu<(argc==10?1:2);++gpu){ dy::OpenCVTracker tracker;if(gpu){auto b=DYToolCreateMetalFlow([NSURL fileURLWithPath:[NSString stringWithUTF8String:argv[4]]],nullptr);if(!b)return 3;tracker.setFlowBackend(b);} std::ifstream input(argv[1],std::ios::binary);if(!input)return 2; std::vector bytes(size_t(w)*h);dy::GrayFrame f{w,h,std::vector(bytes.size())}; NSMutableArray *frames=[NSMutableArray array];double total=0,maximum=0; while(input.read((char*)bytes.data(),bytes.size())){ - for(size_t i=0;i(std::chrono::steady_clock::now()-start).count();total+=ms;maximum=std::max(maximum,ms);auto r=tracker.result(); [frames addObject:@{@"valid":@(ok),@"cx":@(r.cx),@"cy":@(r.cy),@"scaleX":@(r.scaleX),@"scaleY":@(r.scaleY),@"rotation":@(r.rotation),@"confidence":@(r.confidence),@"failure":@((int)r.failure)}];if(!ok)break; diff --git a/tools/tracking/verify_opencv_tracker.cpp b/tools/tracking/verify_opencv_tracker.cpp index e50a789..2be9fcc 100644 --- a/tools/tracking/verify_opencv_tracker.cpp +++ b/tools/tracking/verify_opencv_tracker.cpp @@ -75,5 +75,15 @@ int main() { auto ms=std::chrono::duration(std::chrono::steady_clock::now()-begin).count(); std::cerr<<"4K generated images plus tracking: "<b=[queue_ commandBuffer];pyramid(b,frames_[0]);return finish(b); } bool advance(const dy::GrayFrame& f,const std::vector& points,std::vector& matches) override { + if(f.valid()&&f.pixels.empty())return advance(f.expanded(),points,matches); if(!f.valid()||f.width!=info_.width[0]||f.height!=info_.height[0]||points.empty()||points.size()>dy::TrackerFeatureLimit)return false; int next=1-previous_;memcpy(frames_[next].contents,f.pixels.data(),f.pixels.size()*sizeof(float)); memcpy(points_.contents,points.data(),points.size()*sizeof(dy::FlowPoint)); diff --git a/大鱼调色/Plugin/DYToolGrayCapture.inc.h b/大鱼调色/Plugin/DYToolGrayCapture.inc.h index 5d24c55..1dd02fc 100644 --- a/大鱼调色/Plugin/DYToolGrayCapture.inc.h +++ b/大鱼调色/Plugin/DYToolGrayCapture.inc.h @@ -2,7 +2,11 @@ // CF asks the host for video gamma 2.2, then Lanczos-scales to RGBA8 before // OpenCV gray conversion. Adapt our linear host contract in a separate texture. - (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error { - return [self toolGrayFrame:tile preview:nil error:error]; + auto result=[self toolGrayFrame:tile preview:nil error:error]; + // The diagnostic/legacy caller expects normalized floats. Actual analysis + // keeps the GPU-produced bytes through OpenCV, without float expansion. + if(result.valid()&&result.pixels.empty()){result.pixels.resize(result.bytes.size());for(size_t i=0;iencoded=entry[@"encoded"],bytes=entry[@"bytes"]; if(!encoded||encoded.width!=source.width||encoded.height!=source.height||encoded.pixelFormat!=format){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format width:source.width height:source.height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;encoded=[[device newTextureWithDescriptor:d]autorelease];if(!encoded)return {};entry[@"encoded"]=encoded;} if(!bytes||bytes.width!=width||bytes.height!=height){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:width height:height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;bytes=[[device newTextureWithDescriptor:d]autorelease];if(!bytes)return {};entry[@"bytes"]=bytes;} - idbuffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height*sizeof(float)){buffer=[[device newBufferWithLength:width*height*sizeof(float) options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;} + idbuffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height){buffer=[[device newBufferWithLength:width*height options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;} idcommands=[entry[@"queue"]commandBuffer];idencoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gamma"]];[encoder setTexture:source atIndex:0];[encoder setTexture:encoded atIndex:1];[encoder dispatchThreads:MTLSizeMake(source.width,source.height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding]; [entry[@"scale"]encodeToCommandBuffer:commands sourceTexture:encoded destinationTexture:bytes]; vector_uint3 info={(unsigned)width,(unsigned)height,(unsigned)(tile.imageOrigin==kFxImageOrigin_BOTTOM_LEFT)};encoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gray"]];[encoder setTexture:bytes atIndex:0];[encoder setBuffer:buffer offset:0 atIndex:0];[encoder setBytes:&info length:sizeof(info)atIndex:1];[encoder dispatchThreads:MTLSizeMake(width,height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding]; @@ -58,6 +62,6 @@ *preview=@{@"rgba":[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length], @"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)}; } - dy::GrayFrame result{int(width),int(height),std::vector(width*height)};memcpy(result.pixels.data(),buffer.contents,width*height*sizeof(float));return result; + dy::GrayFrame result;result.width=int(width);result.height=int(height);const auto*begin=static_cast(buffer.contents);result.bytes.assign(begin,begin+width*height);return result; } } diff --git a/大鱼调色/Plugin/DYToolHostMethods.inc.h b/大鱼调色/Plugin/DYToolHostMethods.inc.h index e155217..e647504 100644 --- a/大鱼调色/Plugin/DYToolHostMethods.inc.h +++ b/大鱼调色/Plugin/DYToolHostMethods.inc.h @@ -238,18 +238,11 @@ NSDictionary *preview=nil; dy::GrayFrame frame=[self toolGrayFrame:tile preview:_trackingSingle?nullptr:&preview error:error]; if(!frame.valid())return [self stopToolAnalysis:@"宿主未返回完整源画面" error:error]; - if(!_trackingSamples.count){if(_trackingReferenceFrame.valid()&&frame.width==_trackingReferenceFrame.width&&frame.height==_trackingReferenceFrame.height){double same=0,flip=0;size_t count=0;for(int y=0;y flow; - if(!stamp&&size_t(frame.width)*frame.height>=1920u*1080u) - flow=DYToolCreateMetalFlow([[NSBundle bundleForClass:self.class]URLForResource:@"default" withExtension:@"metallib"],nullptr); - _toolTracker.setFlowBackend(flow); - os_log(OS_LOG_DEFAULT,"dayu_tool_track_backend metal=%{public}d",bool(flow)); dy::TrackerROI targetROI={_trackingSeed.centerX,_trackingSeed.centerY,_trackingSeed.width,_trackingSeed.height,_trackingSeed.angle}; if(!_toolTracker.reset(frame,targetROI)) return [self finishToolAnalysis:stamp?@"覆盖区域内纹理不足,原图章已保留":@"蒙版内纹理不足,原蒙版已保留" error:error]; @@ -313,7 +306,7 @@ } - (BOOL)cleanupAnalysis:(NSError**)error { if(!_trackingActive.load()||_trackingFinishing.exchange(true))return YES;BOOL changed=_trackingSamples.count>1;if(changed&&!DYMergeTracking(_trackingCandidate,_trackingSamples,_trackingFrameDuration)){[_trackingMessage release];_trackingMessage=[@"轨迹合并失败,原蒙版已保留" copy];_trackingDiscard=YES;} - _toolTracker.stop();_toolTracker.setFlowBackend({}); + _toolTracker.stop(); if(_trackingStampMotion.active)_stampSourceTracker.stop(); uint64_t generation=_trackingGeneration;BOOL second=_trackingBoth&&!_trackingBackward&&!_trackingCancelled.load()&&!_trackingDiscard; dispatch_async(dispatch_get_main_queue(),^{if(self->_trackingGeneration!=generation)return; diff --git a/大鱼调色/Plugin/DYToolTrackingDiagnostics.inc.h b/大鱼调色/Plugin/DYToolTrackingDiagnostics.inc.h index 48b88c9..10088dc 100644 --- a/大鱼调色/Plugin/DYToolTrackingDiagnostics.inc.h +++ b/大鱼调色/Plugin/DYToolTrackingDiagnostics.inc.h @@ -63,6 +63,9 @@ for(NSUInteger y=0;y source [[texture(0 if(pos.x>=output.get_width()||pos.y>=output.get_height())return;float4 pixel=source.read(pos);float alpha=max(pixel.a,0.f);float3 straight=alpha>0?pixel.rgb/alpha:float3(0);output.write(float4(pow(max(straight,float3(0)),float3(1.f/2.2f))*alpha,alpha),pos); } // Lanczos RGBA8 -> top-first OpenCV Gray8. Analysis never changes color output. -kernel void dyToolCapture(texture2d source [[texture(0)]],device float* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){ +kernel void dyToolCapture(texture2d source [[texture(0)]],device uchar* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){ if(pos.x>=info.x||pos.y>=info.y)return;uint2 point=uint2(pos.x,info.z?info.y-1-pos.y:pos.y);uint3 rgb=uint3(rint(source.read(point).rgb*255.f)); - pixels[pos.y*info.x+pos.x]=float((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15)/255.f; + pixels[pos.y*info.x+pos.x]=uchar((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15); } // Reuse the analysis source texture for the lightweight color preview. diff --git a/大鱼调色/Plugin/Info.plist b/大鱼调色/Plugin/Info.plist index 2e1297b..1bafe12 100644 --- a/大鱼调色/Plugin/Info.plist +++ b/大鱼调色/Plugin/Info.plist @@ -17,9 +17,9 @@ CFBundlePackageType XPC! CFBundleShortVersionString - 1.18.0 + 1.18.1 CFBundleVersion - 105 + 106 NSHumanReadableCopyright PlugInKit @@ -71,7 +71,7 @@ uuid 1C17B73B-F229-4C09-A434-EAF6DFF5F22A version - 1.18.0 + 1.18.1 className diff --git a/大鱼调色/Plugin/MaskCore/DYTracker.hpp b/大鱼调色/Plugin/MaskCore/DYTracker.hpp index 275f073..fc6c988 100644 --- a/大鱼调色/Plugin/MaskCore/DYTracker.hpp +++ b/大鱼调色/Plugin/MaskCore/DYTracker.hpp @@ -23,9 +23,24 @@ constexpr float TrackerFitTolerance = 3.f; struct GrayFrame { int width = 0, height = 0; std::vector pixels; + // Compact GPU analysis input. Legacy/synthetic callers keep using floats. + std::vector bytes; bool valid() const { return width >= 16 && height >= 16 && - pixels.size() == static_cast(width) * height; + (pixels.size() == static_cast(width) * height || + (pixels.empty() && bytes.size() == static_cast(width) * height)); + } + float at(size_t i) const {return pixels.empty()?bytes[i]/255.f:pixels[i];} + GrayFrame expanded() const { + GrayFrame f{width,height,std::vector(bytes.size())}; + for(size_t i=0;i(w) * 4) return {}; @@ -347,6 +362,7 @@ public: void setBackend(std::shared_ptr b) {backend_=std::move(b);active_=false;} void setMask(std::vector

p) {mask_=std::move(p);} bool reset(const GrayFrame& frame,TrackerROI roi) { + if(frame.valid()&&frame.pixels.empty())return reset(frame.expanded(),roi); active_=false;points_.clear();previous_.clear();cumulative_={};frameIndex_=0; result_={roi.cx,roi.cy,1,0,0,0,0,false}; if (!frame.valid() || !std::isfinite(roi.cx) || !std::isfinite(roi.cy) || @@ -361,6 +377,7 @@ public: return true; } TrackResult track(const GrayFrame& frame) { + if(frame.valid()&&frame.pixels.empty())return track(frame.expanded()); if (!active_ || !frame.valid() || frame.width!=width_ || frame.height!=height_) { result_.valid=false;result_.confidence=0;result_.failure=TrackFailure::InvalidFrame;active_=false;return result_; } diff --git a/大鱼调色/Wrapper Application/Info.plist b/大鱼调色/Wrapper Application/Info.plist index 8d8841a..900b3da 100644 --- a/大鱼调色/Wrapper Application/Info.plist +++ b/大鱼调色/Wrapper Application/Info.plist @@ -17,9 +17,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.18.0 + 1.18.1 CFBundleVersion - 105 + 106 LSMinimumSystemVersion $(MACOSX_DEPLOYMENT_TARGET) NSHumanReadableCopyright