// Analysis only. The color renderer continues to use the original linear input. // CF asks the host for video gamma 2.2, then Lanczos-scales to RGBA8 before // OpenCV gray conversion. Adapt our linear host contract in a separate texture. - (dy::GrayFrame)toolGrayFrame:(FxImageTile*)tile error:(NSError**)error { auto result=[self toolGrayFrame:tile preview:nil error:error]; // The diagnostic/legacy caller expects normalized floats. Actual analysis // keeps the GPU-produced bytes without expanding the full frame to floats. if(result.valid()&&result.pixels.empty()){result.pixels.resize(result.bytes.size());for(size_t i=0;idevice=nil;for(idcandidate in devices)if(candidate.registryID==tile.deviceRegistryID)device=candidate; if(device){idlibrary=[device newLibraryWithURL:[[NSBundle bundleForClass:self.class]URLForResource:@"default" withExtension:@"metallib"]error:error];idgray=[library newFunctionWithName:@"dyToolCapture"],gamma=[library newFunctionWithName:@"dyToolEncodeGamma22"];idgrayPipeline=gray?[device newComputePipelineStateWithFunction:gray error:error]:nil,gammaPipeline=gamma?[device newComputePipelineStateWithFunction:gamma error:error]:nil;idqueue=[device newCommandQueue];MPSImageLanczosScale *scale=[[MPSImageLanczosScale alloc]initWithDevice:device]; if(grayPipeline&&gammaPipeline&&queue&&scale){ entry=[NSMutableDictionary dictionaryWithDictionary:@{@"device":device,@"gray":grayPipeline,@"gamma":gammaPipeline,@"queue":queue,@"scale":scale}];_toolCaptureDevices[key]=entry; idfunction=[library newFunctionWithName:@"dyToolTrackingPreview"]; idpipeline=function?[device newComputePipelineStateWithFunction:function error:nil]:nil; entry[@"previewPipeline"]=pipeline; [pipeline release];[function release]; function=[library newFunctionWithName:@"dyToolVisionPreview"]; pipeline=function?[device newComputePipelineStateWithFunction:function error:nil]:nil; entry[@"visionPipeline"]=pipeline; [pipeline release];[function release]; } [scale release];[queue release];[grayPipeline release];[gammaPipeline release];[gray release];[gamma release];[library release]; }[devices release];if(!entry)return NO; } iddevice=entry[@"device"];idsource=[tile metalTextureForDevice:device];if(!source||source.framebufferOnly||!source.width||!source.height)return NO; double factor=MIN(1.,MIN(3840./source.width,2160./source.height));NSUInteger width=floor(source.width*factor),height=floor(source.height*factor);if(width<16||height<16||width>3840||height>2160)return NO; MTLPixelFormat format=source.pixelFormat==MTLPixelFormatRGBA32Float?MTLPixelFormatRGBA32Float:MTLPixelFormatRGBA16Float; idencoded=entry[@"encoded"],bytes=entry[@"bytes"]; if(!encoded||encoded.width!=source.width||encoded.height!=source.height||encoded.pixelFormat!=format){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:format width:source.width height:source.height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;encoded=[[device newTextureWithDescriptor:d]autorelease];if(!encoded)return NO;entry[@"encoded"]=encoded;} if(!bytes||bytes.width!=width||bytes.height!=height){MTLTextureDescriptor*d=[MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatRGBA8Unorm width:width height:height mipmapped:NO];d.storageMode=MTLStorageModePrivate;d.usage=MTLTextureUsageShaderRead|MTLTextureUsageShaderWrite;bytes=[[device newTextureWithDescriptor:d]autorelease];if(!bytes)return NO;entry[@"bytes"]=bytes;} idbuffer=nil; if(gray){ buffer=entry[@"buffer"]; if(buffer.length!=width*height){ buffer=[[device newBufferWithLength:width*height options:MTLResourceStorageModeShared]autorelease]; if(!buffer)return NO; entry[@"buffer"]=buffer; } } idcommands=[entry[@"queue"]commandBuffer];idencoder=[commands computeCommandEncoder];[encoder setComputePipelineState:entry[@"gamma"]];[encoder setTexture:source atIndex:0];[encoder setTexture:encoded atIndex:1];[encoder dispatchThreads:MTLSizeMake(source.width,source.height,1)threadsPerThreadgroup:MTLSizeMake(16,16,1)];[encoder endEncoding]; [entry[@"scale"]encodeToCommandBuffer:commands sourceTexture:encoded destinationTexture:bytes]; unsigned flip=tile.imageOrigin==kFxImageOrigin_BOTTOM_LEFT; if(gray){ vector_uint3 info={(unsigned)width,(unsigned)height,flip}; encoder=[commands computeCommandEncoder]; [encoder setComputePipelineState:entry[@"gray"]];[encoder setTexture:bytes atIndex:0]; [encoder setBuffer:buffer offset:0 atIndex:0];[encoder setBytes:&info length:sizeof(info) atIndex:1]; [encoder dispatchThreads:MTLSizeMake(width,height,1) threadsPerThreadgroup:MTLSizeMake(16,16,1)]; [encoder endEncoding]; } idpreviewBuffer=nil; CVPixelBufferRef visionImage=nullptr; CVMetalTextureRef visionTexture=nullptr; vector_uint3 previewInfo={0,0,flip}; if(preview){ double scale=std::min({1.,640./width,360./height}); previewInfo.x=std::max(1u,(unsigned)(width*scale));previewInfo.y=std::max(1u,(unsigned)(height*scale)); } if(preview&&visionFlip&&entry[@"visionPipeline"]){ visionTexture=[self newTrackingVisionTexture:entry size:previewInfo image:&visionImage]; if(visionTexture){ uint32_t reverseRows=visionFlip.boolValue; encoder=[commands computeCommandEncoder]; [encoder setComputePipelineState:entry[@"visionPipeline"]]; [encoder setTexture:bytes atIndex:0]; [encoder setTexture:CVMetalTextureGetTexture(visionTexture) atIndex:1]; [encoder setBytes:&previewInfo length:sizeof(previewInfo) atIndex:0]; [encoder setBytes:&reverseRows length:sizeof(reverseRows) atIndex:1]; [encoder dispatchThreads:MTLSizeMake(previewInfo.x,previewInfo.y,1) threadsPerThreadgroup:MTLSizeMake(16,16,1)]; [encoder endEncoding]; } } if(preview&&!visionTexture&&entry[@"previewPipeline"]){ NSUInteger size=previewInfo.x*previewInfo.y*4; previewBuffer=entry[@"previewBuffer"]; if(previewBuffer.length!=size){ previewBuffer=[[device newBufferWithLength:size options:MTLResourceStorageModeShared]autorelease]; entry[@"previewBuffer"]=previewBuffer; } if(previewBuffer){ encoder=[commands computeCommandEncoder]; [encoder setComputePipelineState:entry[@"previewPipeline"]];[encoder setTexture:bytes atIndex:0]; [encoder setBuffer:previewBuffer offset:0 atIndex:0];[encoder setBytes:&previewInfo length:sizeof(previewInfo) atIndex:1]; [encoder dispatchThreads:MTLSizeMake(previewInfo.x,previewInfo.y,1) threadsPerThreadgroup:MTLSizeMake(16,16,1)]; [encoder endEncoding]; } } if(!gray&&!previewBuffer&&!visionTexture)return NO; [commands commit];[commands waitUntilCompleted]; if(visionTexture)CFRelease(visionTexture); if(commands.status!=MTLCommandBufferStatusCompleted){ if(visionImage)CVPixelBufferRelease(visionImage); if(error)*error=commands.error; return NO; } if(previewBuffer||visionImage){ double pixelAspect=tile.pixelAspect; double aspect=double(width)/height*(std::isfinite(pixelAspect)&&pixelAspect>0?pixelAspect:1); NSString *key=visionImage?@"visionImage":@"rgba"; id pixels=visionImage?(id)visionImage:[NSData dataWithBytes:previewBuffer.contents length:previewBuffer.length]; *preview=@{key:pixels,@"width":@(previewInfo.x),@"height":@(previewInfo.y),@"aspect":@(aspect)}; if(visionImage)CVPixelBufferRelease(visionImage); } if(gray){ gray->width=int(width);gray->height=int(height); const auto *begin=static_cast(buffer.contents); gray->bytes.assign(begin,begin+width*height); } return YES; } }