feat: decode UTF-16 and UTF-32 Cube LUTs without rewriting their bytes
这个提交包含在:
1 父节点
6d48b653a5
当前提交
8951725be2
7 个文件已更改
+1212
-11
没有与搜索条件相匹配的文件
@@ -0,0 +1,40 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Rebuild synthetic encoding fixtures in the fixed product cache, never user LUTs."""
|
||||
from pathlib import Path
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
ROOT = Path.home() / "Library/Caches/CodexBuilds/com.dayu.color/fcp-qa/编码兼容验收"
|
||||
|
||||
|
||||
def main():
|
||||
ROOT.mkdir(parents=True, exist_ok=True)
|
||||
matrix = [("utf-8", False), ("utf-8", True)]
|
||||
matrix += [(codec, bom) for codec in ("utf-16-le", "utf-16-be", "utf-32-le", "utf-32-be") for bom in (False, True)]
|
||||
matrix += [(codec, False) for codec in ("gb18030", "big5", "shift_jis", "cp1252", "mac_roman", "iso8859-1")]
|
||||
marks = {"utf-8": b"\xef\xbb\xbf", "utf-16-le": b"\xff\xfe", "utf-16-be": b"\xfe\xff",
|
||||
"utf-32-le": b"\xff\xfe\0\0", "utf-32-be": b"\0\0\xfe\xff"}
|
||||
previous = ROOT / "fixtures.json"
|
||||
known = json.loads(previous.read_text())["files"] if previous.exists() else []
|
||||
known = {item["file"]: item["sha256"] for item in known}
|
||||
rows = []
|
||||
for index, (codec, bom) in enumerate(matrix):
|
||||
title = "Unicode 中文 🎬" if codec.startswith("utf-") else "中文標題" if codec in ("gb18030", "big5") else "日本語" if codec == "shift_jis" else "Café"
|
||||
newline = ("\n", "\r\n", "\r")[index % 3]
|
||||
name = codec.upper() + ("-BOM" if bom else "-NoBOM") + ".cube"
|
||||
text = [f'TITLE "{title}"', '# Encoding compatibility fixture only', 'LUT_3D_SIZE 33', 'DOMAIN_MIN 0 0 0', 'DOMAIN_MAX 1 1 1']
|
||||
gain = .7 + .04 * index
|
||||
text.extend(f"{x/32*gain:.7f} {y/32:.7f} {z/32:.7f}" for z in range(33) for y in range(33) for x in range(33))
|
||||
data = (marks[codec] if bom else b"") + (newline.join(text) + newline).encode(codec)
|
||||
target = ROOT / name
|
||||
if target.exists():
|
||||
assert hashlib.sha256(target.read_bytes()).hexdigest() == known.get(name), f"Existing fixture changed; preserve it: {target}"
|
||||
target.write_bytes(data)
|
||||
rows.append(dict(file=name, codec=codec, bom=bom, sha256=hashlib.sha256(data).hexdigest(), bytes=len(data), dimension=33))
|
||||
report = dict(synthetic=True, sourceFolder=str(ROOT), files=rows, count=len(rows), bytes=sum(x["bytes"] for x in rows))
|
||||
previous.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(json.dumps({k: v for k, v in report.items() if k != "files"}, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,56 @@
|
||||
#import <AppKit/AppKit.h>
|
||||
#import "../大鱼调色/Plugin/DYLUTLibrary.h"
|
||||
#include <cmath>
|
||||
|
||||
static NSMutableArray *checks;
|
||||
static void need(BOOL pass,NSString *name){[checks addObject:@{@"name":name,@"pass":@(pass)}];if(!pass){fprintf(stderr,"FAIL %s\n",name.UTF8String);exit(1);}}
|
||||
static NSString *cube(NSUInteger n){
|
||||
NSMutableString *s=[NSMutableString stringWithString:@"# 中文 / 日本語 / café / 🎨\nTITLE \"Unicode 🎬\"\nLUT_3D_SIZE "];[s appendFormat:@"%lu\nDOMAIN_MIN 0 0 0\nDOMAIN_MAX 1 1 1\n",(unsigned long)n];
|
||||
for(NSUInteger z=0;z<n;z++)for(NSUInteger y=0;y<n;y++)for(NSUInteger x=0;x<n;x++)[s appendFormat:@"%.7e %.7e %.7e # value\n",double(x)/(n-1)*.8,double(y)/(n-1),double(z)/(n-1)];return s;
|
||||
}
|
||||
static NSData *encoded(NSString *text,NSStringEncoding encoding,BOOL bom){
|
||||
NSMutableData *d=[NSMutableData data];
|
||||
if(bom){const uint8_t le16[]={0xff,0xfe},be16[]={0xfe,0xff},le32[]={0xff,0xfe,0,0},be32[]={0,0,0xfe,0xff},utf8[]={0xef,0xbb,0xbf};
|
||||
if(encoding==NSUTF16LittleEndianStringEncoding)[d appendBytes:le16 length:2];else if(encoding==NSUTF16BigEndianStringEncoding)[d appendBytes:be16 length:2];
|
||||
else if(encoding==NSUTF32LittleEndianStringEncoding)[d appendBytes:le32 length:4];else if(encoding==NSUTF32BigEndianStringEncoding)[d appendBytes:be32 length:4];else [d appendBytes:utf8 length:3];}
|
||||
[d appendData:[text dataUsingEncoding:encoding]];return d;
|
||||
}
|
||||
static void pump(BOOL (^done)(void)){double end=NSProcessInfo.processInfo.systemUptime+90;while(!done()&&NSProcessInfo.processInfo.systemUptime<end)[[NSRunLoop mainRunLoop]runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:.01]];need(done(),@"asynchronous operation completes");}
|
||||
int main(int argc,const char **argv){@autoreleasepool{
|
||||
if(argc!=2)return 2;checks=[NSMutableArray new];NSFileManager *fm=NSFileManager.defaultManager;NSError *error=nil;
|
||||
NSURL *root=[NSURL fileURLWithPath:[NSTemporaryDirectory()stringByAppendingPathComponent:[@"dayu-lut-encoding-" stringByAppendingString:NSUUID.UUID.UUIDString]]];NSURL *valid=[root URLByAppendingPathComponent:@"Valid"];
|
||||
need([fm createDirectoryAtURL:valid withIntermediateDirectories:YES attributes:nil error:&error],@"create isolated encoding fixtures");
|
||||
NSString *base=cube(2);float pixel[]={.37,.28,.81,1};NSData *input=[NSData dataWithBytes:pixel length:sizeof(pixel)];
|
||||
NSURL *baseline=[root URLByAppendingPathComponent:@"Baseline.cube"];need([base writeToURL:baseline atomically:YES encoding:NSUTF8StringEncoding error:&error],@"write independent UTF-8 baseline");NSData *reference=DYLUTTransformRGBA(input,1,1,baseline.path,&error);need(reference!=nil,@"baseline color result");
|
||||
NSArray *encodings=@[@(NSUTF8StringEncoding),@(NSUTF16LittleEndianStringEncoding),@(NSUTF16BigEndianStringEncoding),@(NSUTF32LittleEndianStringEncoding),@(NSUTF32BigEndianStringEncoding)];
|
||||
NSUInteger count=0;NSMutableDictionary *originals=[NSMutableDictionary dictionary];
|
||||
for(NSNumber *e in encodings)for(NSNumber *bom in @[@NO,@YES])for(NSString *newline in @[@"\n",@"\r\n",@"\r"]){@autoreleasepool{
|
||||
NSString *name=[NSString stringWithFormat:@"Encoding-%02lu.cube",(unsigned long)count++];NSURL *url=[valid URLByAppendingPathComponent:name];NSData *data=encoded([base stringByReplacingOccurrencesOfString:@"\n" withString:newline],e.unsignedIntegerValue,bom.boolValue);
|
||||
need([data writeToURL:url atomically:YES],name);originals[name]=data;
|
||||
need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" exact numeric equivalence"]);
|
||||
need(DYLUTThumbnail(url.path,nil,0,0,NSMakeSize(32,18),&error)!=nil,[name stringByAppendingString:@" thumbnail"]);
|
||||
}}
|
||||
// Unicode horizontal spacing is valid around commands and numeric tokens.
|
||||
NSString *spaced=[base stringByReplacingOccurrencesOfString:@" " withString:@"\u00a0"];
|
||||
for(NSNumber *e in encodings){NSString *name=[NSString stringWithFormat:@"Spacing-%lu.cube",(unsigned long)e.unsignedIntegerValue];NSData *data=encoded(spaced,e.unsignedIntegerValue,NO);NSURL *url=[valid URLByAppendingPathComponent:name];need([data writeToURL:url atomically:YES],name);originals[name]=data;need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" nonbreaking spaces"]);}
|
||||
for(NSNumber *e in [encodings subarrayWithRange:NSMakeRange(1,4)]){NSString *name=[NSString stringWithFormat:@"UnicodeLines-%lu.cube",(unsigned long)e.unsignedIntegerValue];NSString *text=[base stringByReplacingOccurrencesOfString:@"\n" withString:@"\u2028"];NSData *data=encoded(text,e.unsignedIntegerValue,YES);NSURL *url=[valid URLByAppendingPathComponent:name];need([data writeToURL:url atomically:YES],name);originals[name]=data;need([DYLUTTransformRGBA(input,1,1,url.path,&error)isEqual:reference],[name stringByAppendingString:@" Unicode line separators"]);}
|
||||
// Explicit bytes keep this independent of the decoder being exercised.
|
||||
NSArray *bad=@[[NSData dataWithBytes:"\xff\xfe\x54" length:3],[NSData dataWithBytes:"\xfe\xff\x54" length:3],
|
||||
[NSData dataWithBytes:"\xff\xfe\x00\xd8" length:4],[NSData dataWithBytes:"\xff\xfe\x00\xdc" length:4],
|
||||
[NSData dataWithBytes:"\xff\xfe\x00\xd8\x41\x00" length:6],
|
||||
[NSData dataWithBytes:"\x00\x00\xfe\xff\x00\x11\x00\x00" length:8],
|
||||
[NSData dataWithBytes:"\xff\xfe\x00\x00\x00\xd8\x00\x00" length:8],
|
||||
[NSData dataWithBytes:"\x00\x00\xfe\xff\x00\x00\x00" length:7],
|
||||
[NSData dataWithBytes:"\xff\xfe\x00\x00\x00\x00\x00\x00" length:8]];
|
||||
NSUInteger badIndex=0;for(NSData *data in bad){NSURL *url=[root URLByAppendingPathComponent:[NSString stringWithFormat:@"Malformed-%lu.cube",(unsigned long)badIndex++]];need([data writeToURL:url atomically:YES],@"write malformed Unicode fixture");std::vector<float> v;NSUInteger n=0;need(!DYLUTReadCube(url.path,v,n,&error)&&v.empty()&&n==0&&[error.localizedDescription containsString:@"编码损坏"],@"reject malformed Unicode without a lossy fallback");}
|
||||
NSURL *invalid=[root URLByAppendingPathComponent:@"BadNumbers.cube"];need([encoded([base stringByAppendingString:@"NaN 0 0\n"],NSUTF32BigEndianStringEncoding,YES)writeToURL:invalid atomically:YES],@"write encoded invalid numbers");std::vector<float> v;NSUInteger n=0;need(!DYLUTReadCube(invalid.path,v,n,&error)&&v.empty()&&n==0,@"Unicode decoding does not weaken numeric validation");
|
||||
// Cold large UTF-32 file exercises background conversion and cancellation.
|
||||
NSURL *large=[valid URLByAppendingPathComponent:@"Large-UTF32BE.cube"];NSData *largeBytes=encoded(cube(65),NSUTF32BigEndianStringEncoding,YES);need([largeBytes writeToURL:large atomically:YES],@"write cold 65-cube UTF-32 fixture");originals[large.lastPathComponent]=largeBytes;
|
||||
DYLUTLibrary *library=[[DYLUTLibrary alloc]initWithRootURL:[root URLByAppendingPathComponent:@"Library"]];__block BOOL done=NO;__block double last=NSProcessInfo.processInfo.systemUptime,gap=0;__block NSUInteger ticks=0,imported=0;
|
||||
NSTimer *timer=[NSTimer scheduledTimerWithTimeInterval:.005 repeats:YES block:^(NSTimer *t){double now=NSProcessInfo.processInfo.systemUptime;gap=MAX(gap,now-last);last=now;ticks++;(void)library.entries;}];
|
||||
[library importURLsAsync:@[valid] completion:^(NSArray *items,NSError *failure){need(!failure,@"mixed encoding folder imports asynchronously");imported=items.count;done=YES;}];pump(^{return done;});[timer invalidate];need(imported==originals.count,@"all encoded files imported");need(ticks>0&&gap<.15,@"main loop remains responsive during Unicode conversion");
|
||||
for(NSDictionary *entry in library.entries)need([[NSData dataWithContentsOfFile:entry[@"path"]]isEqual:originals[[entry[@"path"]lastPathComponent]]],@"import preserves exact encoded bytes");
|
||||
NSURL *cancelURL=[root URLByAppendingPathComponent:@"Cancel-UTF32BE.cube"];need([largeBytes writeToURL:cancelURL atomically:YES],@"write uncached cancellation fixture");done=NO;NSProgress *task=DYLUTPrepareAsync(cancelURL.path,^(NSError *failure){need(failure.code==NSUserCancelledError,@"cancelled Unicode conversion reports cancellation");done=YES;});NSTimer *cancel=[NSTimer scheduledTimerWithTimeInterval:.001 repeats:NO block:^(NSTimer *t){[task cancel];}];pump(^{return done;});[cancel invalidate];
|
||||
NSDictionary *report=@{@"pass":@YES,@"checks":@(checks.count),@"results":checks,@"encodedImportFiles":@(imported),@"mainHeartbeatMaxGapMS":@(gap*1000),@"mainHeartbeatTicks":@(ticks),@"largeEncodedBytes":@(largeBytes.length),@"scope":@"Synthetic isolated UTF-8/16/32 LE/BE BOM/no-BOM, line endings, Unicode spacing, exact colors, corruption, async import and cancellation. No user files modified."};
|
||||
need([[NSJSONSerialization dataWithJSONObject:report options:NSJSONWritingPrettyPrinted error:nil]writeToFile:[NSString stringWithUTF8String:argv[1]] atomically:YES],@"write encoding report");[library release];need([fm removeItemAtURL:root error:nil],@"clean encoding fixtures");printf("PASS %lu checks, main gap %.3f ms\n",(unsigned long)checks.count,gap*1000);return 0;
|
||||
}}
|
||||
在新工单中引用
屏蔽一个用户