让 Metal 直写 Vision 追踪输入并加入 LUT 高清分屏预览

这个提交包含在:
mac 于 2026-11-08 19:11:01 +08:00 提交
1 父节点 ef0e62419a
当前提交 63c49b0135
22 个文件已更改
+496 -86

没有与搜索条件相匹配的文件

+69 -6
查看文件
@@ -13,7 +13,41 @@
[self captureTrackingTile:tile gray:&result preview:preview error:error];
return result;
}
- (CVMetalTextureRef)newTrackingVisionTexture:(NSMutableDictionary*)entry size:(vector_uint3)size
image:(CVPixelBufferRef*)image {
CVMetalTextureCacheRef cache=(CVMetalTextureCacheRef)entry[@"visionTextureCache"];
if(!cache){
if(CVMetalTextureCacheCreate(kCFAllocatorDefault,nullptr,entry[@"device"],nullptr,&cache)!=kCVReturnSuccess)return nullptr;
entry[@"visionTextureCache"]=(id)cache;
CFRelease(cache);
}
CVPixelBufferPoolRef pool=(CVPixelBufferPoolRef)entry[@"visionPool"];
if(!pool||[entry[@"visionWidth"]unsignedIntValue]!=size.x||[entry[@"visionHeight"]unsignedIntValue]!=size.y){
NSDictionary *attributes=@{(id)kCVPixelBufferPixelFormatTypeKey:@(kCVPixelFormatType_32BGRA),
(id)kCVPixelBufferWidthKey:@(size.x),(id)kCVPixelBufferHeightKey:@(size.y),
(id)kCVPixelBufferIOSurfacePropertiesKey:@{},(id)kCVPixelBufferMetalCompatibilityKey:@YES};
if(CVPixelBufferPoolCreate(kCFAllocatorDefault,nullptr,(CFDictionaryRef)attributes,&pool)!=kCVReturnSuccess)return nullptr;
entry[@"visionPool"]=(id)pool;CFRelease(pool);
entry[@"visionWidth"]=@(size.x);entry[@"visionHeight"]=@(size.y);
}
NSDictionary *limit=@{(id)kCVPixelBufferPoolAllocationThresholdKey:@6};
if(CVPixelBufferPoolCreatePixelBufferWithAuxAttributes(kCFAllocatorDefault,pool,(CFDictionaryRef)limit,image)!=kCVReturnSuccess)return nullptr;
CVMetalTextureRef view=nullptr;
NSDictionary *usage=@{(id)kCVMetalTextureUsage:@(MTLTextureUsageShaderWrite)};
CVReturn status=CVMetalTextureCacheCreateTextureFromImage(kCFAllocatorDefault,cache,*image,(CFDictionaryRef)usage,
MTLPixelFormatBGRA8Unorm,size.x,size.y,0,&view);
if(status!=kCVReturnSuccess||!view||!CVMetalTextureGetTexture(view)){
if(view)CFRelease(view);
CVPixelBufferRelease(*image);*image=nullptr;
return nullptr;
}
return view;
}
- (BOOL)captureTrackingTile:(FxImageTile*)tile gray:(dy::GrayFrame*)gray preview:(NSDictionary**)preview error:(NSError**)error {
return [self captureTrackingTile:tile gray:gray preview:preview visionFlip:nil error:error];
}
- (BOOL)captureTrackingTile:(FxImageTile*)tile gray:(dy::GrayFrame*)gray preview:(NSDictionary**)preview
visionFlip:(NSNumber*)visionFlip error:(NSError**)error {
@synchronized(_toolCaptureDevices){
FxRect bounds=tile.tilePixelBounds,image=tile.imagePixelBounds;
if(bounds.left!=image.left||bounds.right!=image.right||bounds.bottom!=image.bottom||bounds.top!=image.top)return NO;
@@ -26,6 +60,10 @@
id<MTLComputePipelineState>pipeline=function?[device newComputePipelineStateWithFunction:function error:nil]:nil;
entry[@"previewPipeline"]=pipeline;
[pipeline release];[function release];
function=[library newFunctionWithName:@"dyToolVisionPreview"];
pipeline=function?[device newComputePipelineStateWithFunction:function error:nil]:nil;
entry[@"visionPipeline"]=pipeline;
[pipeline release];[function release];
}
[scale release];[queue release];[grayPipeline release];[gammaPipeline release];[gray release];[gamma release];[library release];
}[devices release];if(!entry)return NO;
@@ -57,10 +95,28 @@
[encoder endEncoding];
}
id<MTLBuffer>previewBuffer=nil;
CVPixelBufferRef visionImage=nullptr;
CVMetalTextureRef visionTexture=nullptr;
vector_uint3 previewInfo={0,0,flip};
if(preview&&entry[@"previewPipeline"]){
if(preview){
double scale=std::min({1.,640./width,360./height});
previewInfo.x=std::max(1u,(unsigned)(width*scale));previewInfo.y=std::max(1u,(unsigned)(height*scale));
}
if(preview&&visionFlip&&entry[@"visionPipeline"]){
visionTexture=[self newTrackingVisionTexture:entry size:previewInfo image:&visionImage];
if(visionTexture){
uint32_t reverseRows=visionFlip.boolValue;
encoder=[commands computeCommandEncoder];
[encoder setComputePipelineState:entry[@"visionPipeline"]];
[encoder setTexture:bytes atIndex:0];
[encoder setTexture:CVMetalTextureGetTexture(visionTexture) atIndex:1];
[encoder setBytes:&previewInfo length:sizeof(previewInfo) atIndex:0];
[encoder setBytes:&reverseRows length:sizeof(reverseRows) atIndex:1];
[encoder dispatchThreads:MTLSizeMake(previewInfo.x,previewInfo.y,1) threadsPerThreadgroup:MTLSizeMake(16,16,1)];
[encoder endEncoding];
}
}
if(preview&&!visionTexture&&entry[@"previewPipeline"]){
NSUInteger size=previewInfo.x*previewInfo.y*4;
previewBuffer=entry[@"previewBuffer"];
if(previewBuffer.length!=size){
@@ -75,14 +131,21 @@
[encoder endEncoding];
}
}
if(!gray&&!previewBuffer)return NO;
if(!gray&&!previewBuffer&&!visionTexture)return NO;
[commands commit];[commands waitUntilCompleted];
if(commands.status!=MTLCommandBufferStatusCompleted){if(error)*error=commands.error;return NO;}
if(previewBuffer){
if(visionTexture)CFRelease(visionTexture);
if(commands.status!=MTLCommandBufferStatusCompleted){
if(visionImage)CVPixelBufferRelease(visionImage);
if(error)*error=commands.error;
return NO;
}
if(previewBuffer||visionImage){
double pixelAspect=tile.pixelAspect;
double aspect=double(width)/height*(std::isfinite(pixelAspect)&&pixelAspect>0?pixelAspect:1);
*preview=@{@"rgba":[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length],
@"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)};
NSString *key=visionImage?@"visionImage":@"rgba";
id pixels=visionImage?(id)visionImage:[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length];
*preview=@{key:pixels,@"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)};
if(visionImage)CVPixelBufferRelease(visionImage);
}
if(gray){
gray->width=int(width);gray->height=int(height);