feat: decode UTF-16 and UTF-32 Cube LUTs without rewriting their bytes

这个提交包含在:
mac 于 2026-11-01 15:37:26 +08:00 提交
1 父节点 6d48b653a5
当前提交 8951725be2
7 个文件已更改
+1212 -11

没有与搜索条件相匹配的文件

+40
查看文件
@@ -0,0 +1,40 @@
#!/usr/bin/env python3
"""Rebuild synthetic encoding fixtures in the fixed product cache, never user LUTs."""
from pathlib import Path
import hashlib
import json
ROOT = Path.home() / "Library/Caches/CodexBuilds/com.dayu.color/fcp-qa/编码兼容验收"
def main():
ROOT.mkdir(parents=True, exist_ok=True)
matrix = [("utf-8", False), ("utf-8", True)]
matrix += [(codec, bom) for codec in ("utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be") for bom in (False, True)]
matrix += [(codec, False) for codec in ("gb18030", "big5", "shift_jis", "cp1252", "mac_roman", "iso8859-1")]
marks = {"utf-8": b"\xef\xbb\xbf", "utf-16-le": b"\xff\xfe", "utf-16-be": b"\xfe\xff",
"utf-32-le": b"\xff\xfe\0\0", "utf-32-be": b"\0\0\xfe\xff"}
previous = ROOT / "fixtures.json"
known = json.loads(previous.read_text())["files"] if previous.exists() else []
known = {item["file"]: item["sha256"] for item in known}
rows = []
for index, (codec, bom) in enumerate(matrix):
title = "Unicode 中文 🎬" if codec.startswith("utf-") else "中文標題" if codec in ("gb18030", "big5") else "日本語" if codec == "shift_jis" else "Café"
newline = ("\n", "\r\n", "\r")[index % 3]
name = codec.upper() + ("-BOM" if bom else "-NoBOM") + ".cube"
text = [f'TITLE "{title}"', '# Encoding compatibility fixture only', 'LUT_3D_SIZE 33', 'DOMAIN_MIN 0 0 0', 'DOMAIN_MAX 1 1 1']
gain = .7 + .04 * index
text.extend(f"{x/32*gain:.7f} {y/32:.7f} {z/32:.7f}" for z in range(33) for y in range(33) for x in range(33))
data = (marks[codec] if bom else b"") + (newline.join(text) + newline).encode(codec)
target = ROOT / name
if target.exists():
assert hashlib.sha256(target.read_bytes()).hexdigest() == known.get(name), f"Existing fixture changed; preserve it: {target}"
target.write_bytes(data)
rows.append(dict(file=name, codec=codec, bom=bom, sha256=hashlib.sha256(data).hexdigest(), bytes=len(data), dimension=33))
report = dict(synthetic=True, sourceFolder=str(ROOT), files=rows, count=len(rows), bytes=sum(x["bytes"] for x in rows))
previous.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(json.dumps({k: v for k, v in report.items() if k != "files"}, ensure_ascii=False))
if __name__ == "__main__":
main()
+56
查看文件
@@ -0,0 +1,56 @@
#import <AppKit/AppKit.h>
#import "../大鱼调色/Plugin/DYLUTLibrary.h"
#include <cmath>
static NSMutableArray *checks;
static void need(BOOL pass,NSString *name){[checks addObject:@{@"name":name,@"pass":@(pass)}];if(!pass){fprintf(stderr,"FAIL %s\n",name.UTF8String);exit(1);}}
static NSString *cube(NSUInteger n){
NSMutableString *s=[NSMutableString stringWithString:@"# 中文 / 日本語 / café / 🎨\nTITLE \"Unicode 🎬\"\nLUT_3D_SIZE "];[s appendFormat:@"%lu\nDOMAIN_MIN 0 0 0\nDOMAIN_MAX 1 1 1\n",(unsigned long)n];
for(NSUInteger z=0;z<n;z++)for(NSUInteger y=0;y<n;y++)for(NSUInteger x=0;x<n;x++)[s appendFormat:@"%.7e %.7e %.7e # value\n",double(x)/(n-1)*.8,double(y)/(n-1),double(z)/(n-1)];return s;
}
static NSData *encoded(NSString *text,NSStringEncoding encoding,BOOL bom){
NSMutableData *d=[NSMutableData data];
if(bom){const uint8_t le16[]={0xff,0xfe},be16[]={0xfe,0xff},le32[]={0xff,0xfe,0,0},be32[]={0,0,0xfe,0xff},utf8[]={0xef,0xbb,0xbf};
if(encoding==NSUTF16LittleEndianStringEncoding)[d appendBytes:le16 length:2];else if(encoding==NSUTF16BigEndianStringEncoding)[d appendBytes:be16 length:2];
else if(encoding==NSUTF32LittleEndianStringEncoding)[d appendBytes:le32 length:4];else if(encoding==NSUTF32BigEndianStringEncoding)[d appendBytes:be32 length:4];else [d appendBytes:utf8 length:3];}
[d appendData:[text dataUsingEncoding:encoding]];return d;
}
static void pump(BOOL (^done)(void)){double end=NSProcessInfo.processInfo.systemUptime+90;while(!done()&&NSProcessInfo.processInfo.systemUptime<end)[[NSRunLoop mainRunLoop]runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:.01]];need(done(),@"asynchronous operation completes");}
int main(int argc,const char **argv){@autoreleasepool{
if(argc!=2)return 2;checks=[NSMutableArray new];NSFileManager *fm=NSFileManager.defaultManager;NSError *error=nil;
NSURL *root=[NSURL fileURLWithPath:[NSTemporaryDirectory()stringByAppendingPathComponent:[@"dayu-lut-encoding-" stringByAppendingString:NSUUID.UUID.UUIDString]]];NSURL *valid=[root URLByAppendingPathComponent:@"Valid"];
need([fm createDirectoryAtURL:valid withIntermediateDirectories:YES attributes:nil error:&error],@"create isolated encoding fixtures");
NSString *base=cube(2);float pixel[]={.37,.28,.81,1};NSData *input=[NSData dataWithBytes:pixel length:sizeof(pixel)];
NSURL *baseline=[root URLByAppendingPathComponent:@"Baseline.cube"];need([base writeToURL:baseline atomically:YES encoding:NSUTF8StringEncoding error:&error],@"write independent UTF-8 baseline");NSData *reference=DYLUTTransformRGBA(input,1,1,baseline.path,&error);need(reference!=nil,@"baseline color result");
NSArray *encodings=@[@(NSUTF8StringEncoding),@(NSUTF16LittleEndianStringEncoding),@(NSUTF16BigEndianStringEncoding),@(NSUTF32LittleEndianStringEncoding),@(NSUTF32BigEndianStringEncoding)];
NSUInteger count=0;NSMutableDictionary *originals=[NSMutableDictionary dictionary];
for(NSNumber *e in encodings)for(NSNumber *bom in @[@NO,@YES])for(NSString *newline in @[@"\n",@"\r\n",@"\r"]){@autoreleasepool{
NSString *name=[NSString stringWithFormat:@"Encoding-%02lu.cube",(unsigned long)count++];NSURL *url=[valid URLByAppendingPathComponent:name];NSData *data=encoded([base stringByReplacingOccurrencesOfString:@"\n" withString:newline],e.unsignedIntegerValue,bom.boolValue);
need([data writeToURL:url atomically:YES],name);originals[name]=data;
need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" exact numeric equivalence"]);
need(DYLUTThumbnail(url.path,nil,0,0,NSMakeSize(32,18),&error)!=nil,[name stringByAppendingString:@" thumbnail"]);
}}
// Unicode horizontal spacing is valid around commands and numeric tokens.
NSString *spaced=[base stringByReplacingOccurrencesOfString:@" " withString:@"\u00a0"];
for(NSNumber *e in encodings){NSString *name=[NSString stringWithFormat:@"Spacing-%lu.cube",(unsigned long)e.unsignedIntegerValue];NSData *data=encoded(spaced,e.unsignedIntegerValue,NO);NSURL *url=[valid URLByAppendingPathComponent:name];need([data writeToURL:url atomically:YES],name);originals[name]=data;need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" nonbreaking spaces"]);}
for(NSNumber *e in [encodings subarrayWithRange:NSMakeRange(1,4)]){NSString *name=[NSString stringWithFormat:@"UnicodeLines-%lu.cube",(unsigned long)e.unsignedIntegerValue];NSString *text=[base stringByReplacingOccurrencesOfString:@"\n" withString:@"\u2028"];NSData *data=encoded(text,e.unsignedIntegerValue,YES);NSURL *url=[valid URLByAppendingPathComponent:name];need([data writeToURL:url atomically:YES],name);originals[name]=data;need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" Unicode line separators"]);}
// Explicit bytes keep this independent of the decoder being exercised.
NSArray *bad=@[[NSData dataWithBytes:"\xff\xfe\x54" length:3],[NSData dataWithBytes:"\xfe\xff\x54" length:3],
[NSData dataWithBytes:"\xff\xfe\x00\xd8" length:4],[NSData dataWithBytes:"\xff\xfe\x00\xdc" length:4],
[NSData dataWithBytes:"\xff\xfe\x00\xd8\x41\x00" length:6],
[NSData dataWithBytes:"\x00\x00\xfe\xff\x00\x11\x00\x00" length:8],
[NSData dataWithBytes:"\xff\xfe\x00\x00\x00\xd8\x00\x00" length:8],
[NSData dataWithBytes:"\x00\x00\xfe\xff\x00\x00\x00" length:7],
[NSData dataWithBytes:"\xff\xfe\x00\x00\x00\x00\x00\x00" length:8]];
NSUInteger badIndex=0;for(NSData *data in bad){NSURL *url=[root URLByAppendingPathComponent:[NSString stringWithFormat:@"Malformed-%lu.cube",(unsigned long)badIndex++]];need([data writeToURL:url atomically:YES],@"write malformed Unicode fixture");std::vector<float> v;NSUInteger n=0;need(!DYLUTReadCube(url.path,v,n,&error)&&v.empty()&&n==0&&[error.localizedDescription containsString:@"编码损坏"],@"reject malformed Unicode without a lossy fallback");}
NSURL *invalid=[root URLByAppendingPathComponent:@"BadNumbers.cube"];need([encoded([base stringByAppendingString:@"NaN 0 0\n"],NSUTF32BigEndianStringEncoding,YES)writeToURL:invalid atomically:YES],@"write encoded invalid numbers");std::vector<float> v;NSUInteger n=0;need(!DYLUTReadCube(invalid.path,v,n,&error)&&v.empty()&&n==0,@"Unicode decoding does not weaken numeric validation");
// Cold large UTF-32 file exercises background conversion and cancellation.
NSURL *large=[valid URLByAppendingPathComponent:@"Large-UTF32BE.cube"];NSData *largeBytes=encoded(cube(65),NSUTF32BigEndianStringEncoding,YES);need([largeBytes writeToURL:large atomically:YES],@"write cold 65-cube UTF-32 fixture");originals[large.lastPathComponent]=largeBytes;
DYLUTLibrary *library=[[DYLUTLibrary alloc]initWithRootURL:[root URLByAppendingPathComponent:@"Library"]];__block BOOL done=NO;__block double last=NSProcessInfo.processInfo.systemUptime,gap=0;__block NSUInteger ticks=0,imported=0;
NSTimer *timer=[NSTimer scheduledTimerWithTimeInterval:.005 repeats:YES block:^(NSTimer *t){double now=NSProcessInfo.processInfo.systemUptime;gap=MAX(gap,now-last);last=now;ticks++;(void)library.entries;}];
[library importURLsAsync:@[valid] completion:^(NSArray *items,NSError *failure){need(!failure,@"mixed encoding folder imports asynchronously");imported=items.count;done=YES;}];pump(^{return done;});[timer invalidate];need(imported==originals.count,@"all encoded files imported");need(ticks>0&&gap<.15,@"main loop remains responsive during Unicode conversion");
for(NSDictionary *entry in library.entries)need([[NSData dataWithContentsOfFile:entry[@"path"]]isEqual:originals[[entry[@"path"]lastPathComponent]]],@"import preserves exact encoded bytes");
NSURL *cancelURL=[root URLByAppendingPathComponent:@"Cancel-UTF32BE.cube"];need([largeBytes writeToURL:cancelURL atomically:YES],@"write uncached cancellation fixture");done=NO;NSProgress *task=DYLUTPrepareAsync(cancelURL.path,^(NSError *failure){need(failure.code==NSUserCancelledError,@"cancelled Unicode conversion reports cancellation");done=YES;});NSTimer *cancel=[NSTimer scheduledTimerWithTimeInterval:.001 repeats:NO block:^(NSTimer *t){[task cancel];}];pump(^{return done;});[cancel invalidate];
NSDictionary *report=@{@"pass":@YES,@"checks":@(checks.count),@"results":checks,@"encodedImportFiles":@(imported),@"mainHeartbeatMaxGapMS":@(gap*1000),@"mainHeartbeatTicks":@(ticks),@"largeEncodedBytes":@(largeBytes.length),@"scope":@"Synthetic isolated UTF-8/16/32 LE/BE BOM/no-BOM, line endings, Unicode spacing, exact colors, corruption, async import and cancellation. No user files modified."};
need([[NSJSONSerialization dataWithJSONObject:report options:NSJSONWritingPrettyPrinted error:nil]writeToFile:[NSString stringWithUTF8String:argv[1]] atomically:YES],@"write encoding report");[library release];need([fm removeItemAtURL:root error:nil],@"clean encoding fixtures");printf("PASS %lu checks, main gap %.3f ms\n",(unsigned long)checks.count,gap*1000);return 0;
}}
文件差异过大,未显示。 加载差异
+1 -1
查看文件
@@ -412,7 +412,7 @@ static NSImage *DYWheelImage(void) {
self.layer.backgroundColor=[DYThemePalette()[@"background"] CGColor];
self.appearance=[NSAppearance appearanceNamed:NSAppearanceNameDarkAqua];
_brand=[DYLabel(@"◒ 大鱼调色",16) retain]; _brand.font=[NSFont boldSystemFontOfSize:16];_brand.textColor=DYInk();
_subtitle=[DYLabel([[NSBundle bundleForClass:self.class]objectForInfoDictionaryKey:@"CFBundleShortVersionString"]?:@"1.1.4",9) retain]; _subtitle.textColor=DYThemeColor(@"secondary");
_subtitle=[DYLabel([[NSBundle bundleForClass:self.class]objectForInfoDictionaryKey:@"CFBundleShortVersionString"]?:@"1.1.5",9) retain]; _subtitle.textColor=DYThemeColor(@"secondary");
_resetAll=[DYButton(@"↺",self,@selector(resetAll:)) retain];_resetAll.bordered=NO; _resetAll.toolTip=@"恢复全部调色参数与 LUT";_resetAll.accessibilityLabel=@"全部还原";
_save=[DYButton(@"保存预设",self,@selector(savePreset:)) retain];
_load=[DYButton(@"载入预设",self,@selector(loadPreset:)) retain];
+68 -6
查看文件
@@ -81,6 +81,71 @@ static NSError *DYLUTFileError(NSError *error,NSURL *url) {
info[NSFilePathErrorKey]=url.path;if(error)info[NSUnderlyingErrorKey]=error;
return [NSError errorWithDomain:error.domain?:DYLUTLibraryErrorDomain code:error?error.code:1 userInfo:info];
}
static bool DYLUTUnicodeSpace(uint32_t c) {
return c==0x20||(c>=9&&c<=13)||c==0x85||c==0xa0||c==0x1680||
(c>=0x2000&&c<=0x200a)||c==0x2028||c==0x2029||c==0x202f||c==0x205f||c==0x3000;
}
static void DYLUTAppendScalar(std::string& text,uint32_t c) {
if(DYLUTUnicodeSpace(c)) {
text+=char(c==13?'\r':c==10||c==0x85||c==0x2028||c==0x2029?'\n':' ');return;
}
if(c<0x80)text+=char(c);
else if(c<0x800){text+=char(0xc0|(c>>6));text+=char(0x80|(c&63));}
else if(c<0x10000){text+=char(0xe0|(c>>12));text+=char(0x80|((c>>6)&63));text+=char(0x80|(c&63));}
else {text+=char(0xf0|(c>>18));text+=char(0x80|((c>>12)&63));text+=char(0x80|((c>>6)&63));text+=char(0x80|(c&63));}
}
// Normalize only the working copy. Imports retain their original bytes. Narrow
// files remain byte-transparent so legacy TITLE/comment encodings still work.
static BOOL DYLUTText(NSData *bytes,std::string& text,NSProgress *progress,NSError **error) {
const uint8_t *p=(const uint8_t *)bytes.bytes;size_t length=bytes.length,offset=0,width=1;bool little=true;
if(!length)return DYLUTFail(error,@"LUT 文件为空");
auto unit=[&](size_t at,size_t w,bool le){uint32_t c=0;for(size_t k=0;k<w;k++)c|=uint32_t(p[at+k])<<(8*(le?k:w-1-k));return c;};
if(length>=4&&unit(0,4,true)==0x0000feff){width=4;offset=4;}
else if(length>=4&&unit(0,4,false)==0x0000feff){width=4;offset=4;little=false;}
else if(length>=2&&unit(0,2,true)==0xfeff){width=2;offset=2;}
else if(length>=2&&unit(0,2,false)==0xfeff){width=2;offset=2;little=false;}
else if(length>=3&&!memcmp(p,"\xef\xbb\xbf",3))offset=3;
else {
// Without a BOM, infer byte order only from a recognizable Cube prefix.
// Never remove arbitrary NUL bytes or guess an encoding from the title.
auto prefix=[&](size_t w,bool le){bool whitespace=false;
for(size_t at=0;at+w<=length&&at<128;at+=w){uint32_t c=unit(at,w,le);
if(DYLUTUnicodeSpace(c)||c==0xfeff){whitespace=true;continue;}
return c=='#'||(c>='A'&&c<='Z')||(c>='a'&&c<='z')||(c>='0'&&c<='9')||c=='+'||c=='-'||c=='.';
}return whitespace;
};
for(size_t w:{size_t(4),size_t(2)}) {
if(prefix(w,true)){width=w;break;}
if(prefix(w,false)){width=w;little=false;break;}
}
}
if(width==1){text.assign((const char *)p+offset,length-offset);return !DYLUTCancelled(progress,error);}
NSString *encoding=[NSString stringWithFormat:@"UTF-%lu%@",(unsigned long)(width*8),little?@"LE":@"BE"];
auto invalid=[&](NSString *detail){return DYLUTFail(error,[NSString stringWithFormat:@"%@ 编码损坏:%@",encoding,detail]);};
if((length-offset)%width)return invalid(@"文件结尾不完整");
text.clear();text.reserve((length-offset)/width);
size_t nextCancellationCheck=offset;
for(size_t at=offset;at<length;at+=width){
if(at>=nextCancellationCheck){if(DYLUTCancelled(progress,error))return NO;nextCancellationCheck=at+16384;}
uint32_t c=unit(at,width,little);
if(width==2&&c>=0xd800&&c<=0xdbff){
if(at+4>length)return invalid(@"缺少配对字符");uint32_t next=unit(at+2,2,little);
if(next<0xdc00||next>0xdfff)return invalid(@"字符配对无效");c=0x10000+((c-0xd800)<<10)+(next-0xdc00);at+=2;
}else if(c>=0xd800&&c<=0xdfff)return invalid(@"孤立的配对字符");
if(c>0x10ffff||c==0)return invalid(@"含无效字符");
DYLUTAppendScalar(text,c);
}
return !DYLUTCancelled(progress,error);
}
static std::string_view DYLUTTrim(std::string_view v) {
while(!v.empty()) {
unsigned char c=v.front();size_t count=0;
if(c==' '||(c>=9&&c<=13))count=1;
else if(v.size()>=2&&v.substr(0,2)=="\xc2\xa0")count=2;
else if(v.size()>=3&&(v.substr(0,3)=="\xe1\x9a\x80"||v.substr(0,3)=="\xe2\x80\xaf"||v.substr(0,3)=="\xe2\x81\x9f"||v.substr(0,3)=="\xe3\x80\x80"||(v.substr(0,2)=="\xe2\x80"&&(unsigned char)v[2]>=0x80&&(unsigned char)v[2]<=0x8a)))count=3;
if(!count)break;v.remove_prefix(count);
}return v;
}
static BOOL DYLUTParse(NSString *path,std::vector<float>& data,NSUInteger& size,NSProgress *progress,NSError **error) {
data.clear();size=0;
NSDictionary *attributes=[[NSFileManager defaultManager]attributesOfItemAtPath:path error:error];
@@ -99,20 +164,17 @@ static BOOL DYLUTParse(NSString *path,std::vector<float>& data,NSUInteger& size,
for(float v:data)if(!std::isfinite(v))return DYLUTFail(error,@"输入 LUT 含无效数字");
size=n;return YES;
}
// Cube commands and numbers use ASCII. Descriptive TITLE/comments can
// legitimately use legacy encodings; they never affect the numeric table.
if(!bytes.length)return DYLUTFail(error,@"LUT 文件为空");
std::string storage((const char *)bytes.bytes,bytes.length);
std::string storage;if(!DYLUTText(bytes,storage,progress,error))return NO;
static locale_t numericLocale;static dispatch_once_t localeOnce;dispatch_once(&localeOnce,^{numericLocale=newlocale(LC_NUMERIC_MASK,"C",NULL);});
float domainMin[3]={0,0,0},domainMax[3]={1,1,1};BOOL foundSize=NO;
std::string_view remaining(storage);NSUInteger rows=0;
if(remaining.size()>=3&&remaining.substr(0,3)=="\xef\xbb\xbf")remaining.remove_prefix(3);
auto trim=[](std::string_view v){while(!v.empty()&&(v.front()==' '||v.front()=='\t'||v.front()=='\r'))v.remove_prefix(1);return v;};
auto trim=DYLUTTrim;
auto number=[&](std::string_view& v,float& value){v=trim(v);if(v.empty())return false;char *end=nullptr;value=strtof_l(v.data(),&end,numericLocale);if(end==v.data()||end>v.data()+v.size())return false;v.remove_prefix(end-v.data());return true;};
while(!remaining.empty()) {
if((rows++&1023)==0&&DYLUTCancelled(progress,error))return NO;
size_t end=remaining.find_first_of("\r\n");std::string_view line=remaining.substr(0,end);
if(end==std::string_view::npos)remaining={};else remaining.remove_prefix(end+1);
if(end==std::string_view::npos)remaining={};else {size_t count=end+1;if(remaining[end]=='\r'&&count<remaining.size()&&remaining[count]=='\n')count++;remaining.remove_prefix(count);}
line=line.substr(0,line.find('#'));line=trim(line);if(line.empty())continue;
float r=0,g=0,b=0;
auto starts=[&](std::string_view prefix){return line.substr(0,prefix.size())==prefix;};
+2 -2
查看文件
@@ -17,9 +17,9 @@
<key>CFBundlePackageType</key>
<string>XPC!</string>
<key>CFBundleShortVersionString</key>
<string>1.1.4</string>
<string>1.1.5</string>
<key>CFBundleVersion</key>
<string>15</string>
<string>16</string>
<key>NSHumanReadableCopyright</key>
<string></string>
<key>PlugInKit</key>
+2 -2
查看文件
@@ -17,9 +17,9 @@
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
<string>1.1.4</string>
<string>1.1.5</string>
<key>CFBundleVersion</key>
<string>15</string>
<string>16</string>
<key>LSMinimumSystemVersion</key>
<string>$(MACOSX_DEPLOYMENT_TARGET)</string>
<key>NSHumanReadableCopyright</key>