Keep stable OpenCV flow with compact Metal capture and pyramid reuse
这个提交包含在:
1 父节点
48ac192d4b
当前提交
1cc7f8edb2
15 个文件已更改
+184
-29
没有与搜索条件相匹配的文件
@@ -31,6 +31,7 @@ public:
|
||||
~MetalFlow(){for(int i=0;i<2;i++)[frames_[i] release];[points_ release];[results_ release];[pyramid_ release];[flow_ release];[queue_ release];[device_ release];}
|
||||
bool ready()const{return device_&&queue_&&pyramid_&&flow_&&flow_.threadExecutionWidth==32;}
|
||||
bool reset(const dy::GrayFrame& f) override {
|
||||
if(f.valid()&&f.pixels.empty())return reset(f.expanded());
|
||||
if(!ready()||!f.valid())return false;
|
||||
info_={};info_.levels=1;info_.width[0]=f.width;info_.height[0]=f.height;
|
||||
int total=f.width*f.height;
|
||||
@@ -47,6 +48,7 @@ public:
|
||||
id<MTLCommandBuffer>b=[queue_ commandBuffer];pyramid(b,frames_[0]);return finish(b);
|
||||
}
|
||||
bool advance(const dy::GrayFrame& f,const std::vector<dy::FlowPoint>& points,std::vector<dy::FlowMatch>& matches) override {
|
||||
if(f.valid()&&f.pixels.empty())return advance(f.expanded(),points,matches);
|
||||
if(!f.valid()||f.width!=info_.width[0]||f.height!=info_.height[0]||points.empty()||points.size()>dy::TrackerFeatureLimit)return false;
|
||||
int next=1-previous_;memcpy(frames_[next].contents,f.pixels.data(),f.pixels.size()*sizeof(float));
|
||||
memcpy(points_.contents,points.data(),points.size()*sizeof(dy::FlowPoint));
|
||||
|
||||
@@ -2,7 +2,11 @@
|
||||
// CF asks the host for video gamma 2.2, then Lanczos-scales to RGBA8 before
|
||||
// OpenCV gray conversion. Adapt our linear host contract in a separate texture.
|
||||
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error {
|
||||
return [self toolGrayFrame:tile preview:nil error:error];
|
||||
auto result=[self toolGrayFrame:tile preview:nil error:error];
|
||||
// The diagnostic/legacy caller expects normalized floats. Actual analysis
|
||||
// keeps the GPU-produced bytes through OpenCV, without float expansion.
|
||||
if(result.valid()&&result.pixels.empty()){result.pixels.resize(result.bytes.size());for(size_t i=0;i<result.bytes.size();++i)result.pixels[i]=result.bytes[i]/255.f;result.bytes.clear();}
|
||||
return result;
|
||||
}
|
||||
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile preview:(NSDictionary**)preview error:(NSError**)error {
|
||||
@synchronized(_toolCaptureDevices){
|
||||
@@ -27,7 +31,7 @@
|
||||
id<MTLTexture>encoded=entry[@"encoded"],bytes=entry[@"bytes"];
|
||||
if(!encoded||encoded.width!=source.width||encoded.height!=source.height||encoded.pixelFormat!=format){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format width:source.width height:source.height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;encoded=[[device newTextureWithDescriptor:d]autorelease];if(!encoded)return {};entry[@"encoded"]=encoded;}
|
||||
if(!bytes||bytes.width!=width||bytes.height!=height){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:width height:height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;bytes=[[device newTextureWithDescriptor:d]autorelease];if(!bytes)return {};entry[@"bytes"]=bytes;}
|
||||
id<MTLBuffer>buffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height*sizeof(float)){buffer=[[device newBufferWithLength:width*height*sizeof(float) options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;}
|
||||
id<MTLBuffer>buffer=entry[@"buffer"];if(!buffer||buffer.length!=width*height){buffer=[[device newBufferWithLength:width*height options:MTLResourceStorageModeShared]autorelease];if(!buffer)return {};entry[@"buffer"]=buffer;}
|
||||
id<MTLCommandBuffer>commands=[entry[@"queue"]commandBuffer];id<MTLComputeCommandEncoder>encoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gamma"]];[encoder setTexture:source atIndex:0];[encoder setTexture:encoded atIndex:1];[encoder dispatchThreads:MTLSizeMake(source.width,source.height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding];
|
||||
[entry[@"scale"]encodeToCommandBuffer:commands sourceTexture:encoded destinationTexture:bytes];
|
||||
vector_uint3 info={(unsigned)width,(unsigned)height,(unsigned)(tile.imageOrigin==kFxImageOrigin_BOTTOM_LEFT)};encoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gray"]];[encoder setTexture:bytes atIndex:0];[encoder setBuffer:buffer offset:0 atIndex:0];[encoder setBytes:&info length:sizeof(info)atIndex:1];[encoder dispatchThreads:MTLSizeMake(width,height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding];
|
||||
@@ -58,6 +62,6 @@
|
||||
*preview=@{@"rgba":[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length],
|
||||
@"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)};
|
||||
}
|
||||
dy::GrayFrame result{int(width),int(height),std::vector<float>(width*height)};memcpy(result.pixels.data(),buffer.contents,width*height*sizeof(float));return result;
|
||||
dy::GrayFrame result;result.width=int(width);result.height=int(height);const auto*begin=static_cast<const uint8_t*>(buffer.contents);result.bytes.assign(begin,begin+width*height);return result;
|
||||
}
|
||||
}
|
||||
@@ -238,18 +238,11 @@
|
||||
NSDictionary *preview=nil;
|
||||
dy::GrayFrame frame=[self toolGrayFrame:tile preview:_trackingSingle?nullptr:&preview error:error];
|
||||
if(!frame.valid())return [self stopToolAnalysis:@"宿主未返回完整源画面" error:error];
|
||||
if(!_trackingSamples.count){if(_trackingReferenceFrame.valid()&&frame.width==_trackingReferenceFrame.width&&frame.height==_trackingReferenceFrame.height){double same=0,flip=0;size_t count=0;for(int y=0;y<frame.height;y+=8)for(int x=0;x<frame.width;x+=8){float p=frame.pixels[y*frame.width+x];same+=fabs(p-_trackingReferenceFrame.pixels[y*frame.width+x]);flip+=fabs(p-_trackingReferenceFrame.pixels[(frame.height-1-y)*frame.width+x]);count++;}same/=MAX(size_t(1),count);flip/=MAX(size_t(1),count);_trackingFlip=flip<.035&&flip+.008<same&&flip<same*.8;}os_log(OS_LOG_DEFAULT,"dayu_tool_track_seed width=%{public}d height=%{public}d flip=%{public}d time=%{public}.6f",frame.width,frame.height,(int)_trackingFlip,CMTimeGetSeconds(time));}
|
||||
if(_trackingFlip)for(int y=0;y<frame.height/2;y++)for(int x=0;x<frame.width;x++)std::swap(frame.pixels[y*frame.width+x],frame.pixels[(frame.height-1-y)*frame.width+x]);
|
||||
if(!_trackingSamples.count){if(_trackingReferenceFrame.valid()&&frame.width==_trackingReferenceFrame.width&&frame.height==_trackingReferenceFrame.height){double same=0,flip=0;size_t count=0;for(int y=0;y<frame.height;y+=8)for(int x=0;x<frame.width;x+=8){float p=frame.at(y*frame.width+x);same+=fabs(p-_trackingReferenceFrame.at(y*frame.width+x));flip+=fabs(p-_trackingReferenceFrame.at((frame.height-1-y)*frame.width+x));count++;}same/=MAX(size_t(1),count);flip/=MAX(size_t(1),count);_trackingFlip=flip<.035&&flip+.008<same&&flip<same*.8;}os_log(OS_LOG_DEFAULT,"dayu_tool_track_seed width=%{public}d height=%{public}d flip=%{public}d time=%{public}.6f",frame.width,frame.height,(int)_trackingFlip,CMTimeGetSeconds(time));}
|
||||
if(_trackingFlip)frame.flipVertically();
|
||||
BOOL stamp=_trackingStampMotion.active;
|
||||
dy::TrackResult target,source;
|
||||
if(!_trackingSamples.count){
|
||||
// Full-resolution mask analysis benefits from GPU flow. Tiny inputs and
|
||||
// the separately tuned clone-stamp pair retain the CPU implementation.
|
||||
std::shared_ptr<dy::FlowBackend> flow;
|
||||
if(!stamp&&size_t(frame.width)*frame.height>=1920u*1080u)
|
||||
flow=DYToolCreateMetalFlow([[NSBundle bundleForClass:self.class]URLForResource:@"default" withExtension:@"metallib"],nullptr);
|
||||
_toolTracker.setFlowBackend(flow);
|
||||
os_log(OS_LOG_DEFAULT,"dayu_tool_track_backend metal=%{public}d",bool(flow));
|
||||
dy::TrackerROI targetROI={_trackingSeed.centerX,_trackingSeed.centerY,_trackingSeed.width,_trackingSeed.height,_trackingSeed.angle};
|
||||
if(!_toolTracker.reset(frame,targetROI))
|
||||
return [self finishToolAnalysis:stamp?@"覆盖区域内纹理不足,原图章已保留":@"蒙版内纹理不足,原蒙版已保留" error:error];
|
||||
@@ -313,7 +306,7 @@
|
||||
}
|
||||
- (BOOL)cleanupAnalysis:(NSError**)error {
|
||||
if(!_trackingActive.load()||_trackingFinishing.exchange(true))return YES;BOOL changed=_trackingSamples.count>1;if(changed&&!DYMergeTracking(_trackingCandidate,_trackingSamples,_trackingFrameDuration)){[_trackingMessage release];_trackingMessage=[@"轨迹合并失败,原蒙版已保留" copy];_trackingDiscard=YES;}
|
||||
_toolTracker.stop();_toolTracker.setFlowBackend({});
|
||||
_toolTracker.stop();
|
||||
if(_trackingStampMotion.active)_stampSourceTracker.stop();
|
||||
uint64_t generation=_trackingGeneration;BOOL second=_trackingBoth&&!_trackingBackward&&!_trackingCancelled.load()&&!_trackingDiscard;
|
||||
dispatch_async(dispatch_get_main_queue(),^{if(self->_trackingGeneration!=generation)return;
|
||||
|
||||
@@ -63,6 +63,9 @@
|
||||
for(NSUInteger y=0;y<h;y++)for(NSUInteger x=0;x<w;x++){float encoded=x%3==1?.8f:.2f,linear=powf((encoded+.055f)/1.055f,2.4f);auto i=(y*w+x)*4;pixels[i]=pixels[i+1]=pixels[i+2]=linear;pixels[i+3]=1;}
|
||||
[[fine metalTextureForDevice:gpu]replaceRegion:MTLRegionMake2D(0,0,w,h) mipmapLevel:0 withBytes:pixels.data() bytesPerRow:w*8];auto gray=[plugin toolGrayFrame:fine error:nil];double maximum=0;for(NSUInteger y=0;y<(NSUInteger)gray.height;y++)for(NSUInteger x=0;x<(NSUInteger)gray.width;x++){double code=x%3==1?.8:.2;double expected=std::round(std::pow(std::pow((code+.055)/1.055,2.4),1/2.2)*255)/255.;maximum=std::max(maximum,fabs(double(gray.pixels[y*gray.width+x])-expected));}
|
||||
run.record(@"tracking-4k-frame-preserves-full-detail-for-opencv",gray.valid()&&gray.width==3840&&gray.height==2160&&maximum<.002,@{@"width":@(gray.width),@"height":@(gray.height),@"maximumError":@(maximum)});
|
||||
auto compact=[plugin toolGrayFrame:fine preview:nullptr error:nil];BOOL identical=compact.valid()&&compact.pixels.empty()&&compact.bytes.size()==w*h;
|
||||
for(size_t i=0;i<compact.bytes.size()&&identical;++i)identical=compact.at(i)==gray.pixels[i];
|
||||
run.record(@"tracking-metal-gray8-compact-readback-matches-float-reference",identical,@{@"readbackBytes":@(compact.bytes.size()),@"previousReadbackBytes":@(gray.pixels.size()*sizeof(float))});
|
||||
}
|
||||
// UI aliases of a shared object are one tracking target. Switching from the
|
||||
// global editor to either linked module must not cancel a valid analysis.
|
||||
|
||||
@@ -27,6 +27,7 @@
|
||||
- (BOOL)analyzeFrame:(FxImageTile*)frame atTime:(CMTime)time error:(NSError**)error;
|
||||
- (BOOL)cleanupAnalysis:(NSError**)error;
|
||||
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error;
|
||||
- (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile preview:(NSDictionary**)preview error:(NSError**)error;
|
||||
@end
|
||||
|
||||
@interface DYToolCanvasProbe : DYToolTrackingProbe
|
||||
|
||||
@@ -54,9 +54,9 @@ kernel void dyToolEncodeGamma22(texture2d<float,access::read> source [[texture(0
|
||||
if(pos.x>=output.get_width()||pos.y>=output.get_height())return;float4 pixel=source.read(pos);float alpha=max(pixel.a,0.f);float3 straight=alpha>0?pixel.rgb/alpha:float3(0);output.write(float4(pow(max(straight,float3(0)),float3(1.f/2.2f))*alpha,alpha),pos);
|
||||
}
|
||||
// Lanczos RGBA8 -> top-first OpenCV Gray8. Analysis never changes color output.
|
||||
kernel void dyToolCapture(texture2d<float> source [[texture(0)]],device float* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){
|
||||
kernel void dyToolCapture(texture2d<float> source [[texture(0)]],device uchar* pixels [[buffer(0)]],constant uint3& info [[buffer(1)]],uint2 pos [[thread_position_in_grid]]){
|
||||
if(pos.x>=info.x||pos.y>=info.y)return;uint2 point=uint2(pos.x,info.z?info.y-1-pos.y:pos.y);uint3 rgb=uint3(rint(source.read(point).rgb*255.f));
|
||||
pixels[pos.y*info.x+pos.x]=float((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15)/255.f;
|
||||
pixels[pos.y*info.x+pos.x]=uchar((rgb.r*9798u+rgb.g*19235u+rgb.b*3735u+16384u)>>15);
|
||||
}
|
||||
|
||||
// Reuse the analysis source texture for the lightweight color preview.
|
||||
|
||||
+3
-3
@@ -17,9 +17,9 @@
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>XPC!</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.18.0</string>
|
||||
<string>1.18.1</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>105</string>
|
||||
<string>106</string>
|
||||
<key>NSHumanReadableCopyright</key>
|
||||
<string></string>
|
||||
<key>PlugInKit</key>
|
||||
@@ -71,7 +71,7 @@
|
||||
<key>uuid</key>
|
||||
<string>1C17B73B-F229-4C09-A434-EAF6DFF5F22A</string>
|
||||
<key>version</key>
|
||||
<string>1.18.0</string>
|
||||
<string>1.18.1</string>
|
||||
</dict>
|
||||
<dict>
|
||||
<key>className</key>
|
||||
|
||||
@@ -23,9 +23,24 @@ constexpr float TrackerFitTolerance = 3.f;
|
||||
struct GrayFrame {
|
||||
int width = 0, height = 0;
|
||||
std::vector<float> pixels;
|
||||
// Compact GPU analysis input. Legacy/synthetic callers keep using floats.
|
||||
std::vector<uint8_t> bytes;
|
||||
bool valid() const {
|
||||
return width >= 16 && height >= 16 &&
|
||||
pixels.size() == static_cast<size_t>(width) * height;
|
||||
(pixels.size() == static_cast<size_t>(width) * height ||
|
||||
(pixels.empty() && bytes.size() == static_cast<size_t>(width) * height));
|
||||
}
|
||||
float at(size_t i) const {return pixels.empty()?bytes[i]/255.f:pixels[i];}
|
||||
GrayFrame expanded() const {
|
||||
GrayFrame f{width,height,std::vector<float>(bytes.size())};
|
||||
for(size_t i=0;i<bytes.size();++i)f.pixels[i]=bytes[i]/255.f;
|
||||
return f;
|
||||
}
|
||||
void flipVertically() {
|
||||
for(int y=0;y<height/2;++y)for(int x=0;x<width;++x){
|
||||
size_t a=size_t(y)*width+x,b=size_t(height-1-y)*width+x;
|
||||
if(pixels.empty())std::swap(bytes[a],bytes[b]);else std::swap(pixels[a],pixels[b]);
|
||||
}
|
||||
}
|
||||
static GrayFrame fromRGBA8(const uint8_t* data, int w, int h, size_t stride) {
|
||||
if (!data || w <= 0 || h <= 0 || stride < static_cast<size_t>(w) * 4) return {};
|
||||
@@ -347,6 +362,7 @@ public:
|
||||
void setBackend(std::shared_ptr<FlowBackend> b) {backend_=std::move(b);active_=false;}
|
||||
void setMask(std::vector<P> p) {mask_=std::move(p);}
|
||||
bool reset(const GrayFrame& frame,TrackerROI roi) {
|
||||
if(frame.valid()&&frame.pixels.empty())return reset(frame.expanded(),roi);
|
||||
active_=false;points_.clear();previous_.clear();cumulative_={};frameIndex_=0;
|
||||
result_={roi.cx,roi.cy,1,0,0,0,0,false};
|
||||
if (!frame.valid() || !std::isfinite(roi.cx) || !std::isfinite(roi.cy) ||
|
||||
@@ -361,6 +377,7 @@ public:
|
||||
return true;
|
||||
}
|
||||
TrackResult track(const GrayFrame& frame) {
|
||||
if(frame.valid()&&frame.pixels.empty())return track(frame.expanded());
|
||||
if (!active_ || !frame.valid() || frame.width!=width_ || frame.height!=height_) {
|
||||
result_.valid=false;result_.confidence=0;result_.failure=TrackFailure::InvalidFrame;active_=false;return result_;
|
||||
}
|
||||
|
||||
@@ -17,9 +17,9 @@
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.18.0</string>
|
||||
<string>1.18.1</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>105</string>
|
||||
<string>106</string>
|
||||
<key>LSMinimumSystemVersion</key>
|
||||
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
|
||||
<key>NSHumanReadableCopyright</key>
|
||||
|
||||
在新工单中引用
屏蔽一个用户