From 3f7a06363065b417824eb5d1a7f388d2b5fef332 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 18:01:21 +0800 Subject: [PATCH] Index exported document envelopes for seekable record decoding --- src/lib/canvas-runtime/import-document.ts | 42 +++++++++++++++++++++++ tests/headless/import-document.test.mjs | 6 ++++ 2 files changed, 48 insertions(+) create mode 100644 src/lib/canvas-runtime/import-document.ts create mode 100644 tests/headless/import-document.test.mjs diff --git a/src/lib/canvas-runtime/import-document.ts b/src/lib/canvas-runtime/import-document.ts new file mode 100644 index 0000000..80353da --- /dev/null +++ b/src/lib/canvas-runtime/import-document.ts @@ -0,0 +1,42 @@ +import type {ImportSource,ImportLine} from './imports.ts'; +import {fault,validJson} from '../canvas-core/values.ts'; +type Range={start:number;end:number}; +/** Index the envelope without decoding element payloads. Original bytes remain the source. + * Preparation scans O(bytes) and retains O(records) ranges; this is not constant memory. */ +export async function openDocumentImportSource(options:{bytes:Blob;fingerprint:string;signal:AbortSignal;maxRecordBytes?:number;yieldTask?:()=>Promise}):Promise{ + const {bytes,signal}=options,max=options.maxRecordBytes??256*1024; + if(!(bytes instanceof Blob)||bytes.size>64*1024*1024||!options.fingerprint||!Number.isSafeInteger(max)||max<1)throw fault('invalid_import_source'); + let cursor=0,cache={start:-1,bytes:new Uint8Array()},work=0; + const yieldTask=options.yieldTask??(()=>new Promise(resolve=>setTimeout(resolve,0))),space=(b:number)=>[9,10,13,32].includes(b); + async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor=cache.start+cache.bytes.length){cache={start:cursor,bytes:new Uint8Array(await bytes.slice(cursor,cursor+16384).arrayBuffer())};if(signal.aborted)throw fault('import_aborted');}return cache.bytes[cursor-cache.start];} + async function step(){cursor++;if(++work>=16384){work=0;await yieldTask();if(signal.aborted)throw fault('import_aborted');}} + async function spaces(){while(space(await peek()))await step();} + async function take(byte:number){await spaces();if(await peek()!==byte)throw fault('invalid_document_import');await step();} + async function value():Promise{await spaces();const start=cursor,stack:number[]=[];let quoted=false,escaped=false; + while(true){const b=await peek();if(b<0){if(quoted||stack.length)throw fault('invalid_document_import');break;}if(!quoted&&!stack.length&&(b===44||b===93||b===125||space(b)))break; + if(quoted){if(escaped)escaped=false;else if(b===92)escaped=true;else if(b===34)quoted=false;}else if(b===34)quoted=true;else if(b===91||b===123){stack.push(b);if(stack.length>256)throw fault('json_nesting_limit');}else if(b===93||b===125){if(stack.pop()!==(b===93?91:123))throw fault('invalid_document_import');} + const ended=!quoted&&stack.length===0&&(b===34||b===93||b===125);await step();if(ended)break; + }if(cursor===start)throw fault('invalid_document_import');return{start,end:cursor}; + } + async function decode(range:Range,limit=max){if(range.end-range.start>limit)throw fault('record_too_large');let text:string;try{text=new TextDecoder('utf-8',{fatal:true}).decode(await bytes.slice(range.start,range.end).arrayBuffer());}catch{throw fault('invalid_utf8');}try{const result=JSON.parse(text);if(!validJson(result))throw Error();return result;}catch{throw fault('invalid_json');}} + const fields=new Map(),elements:Range[]=[]; + await take(123);await spaces();if(await peek()===125)throw fault('invalid_document_import'); + while(true){const key=await decode(await value(),1024);if(typeof key!=='string'||fields.has(key))throw fault('invalid_document_import');await take(58); + if(key==='elements'){fields.set(key,true);await take(91);await spaces();if(await peek()!==93)while(true){elements.push(await value());await spaces();if(await peek()===93)break;await take(44);}await take(93);} + else fields.set(key,await decode(await value(),4*1024*1024)); + await spaces();if(await peek()===125){await step();break;}await take(44); + } + await spaces();if(cursor!==bytes.size||fields.get('documentFormatVersion')!==1||typeof fields.get('projectId')!=='string'||!fields.get('projectId')||!Array.isArray(fields.get('canvases'))||!fields.has('elements')||[...fields.keys()].some(key=>!['documentFormatVersion','projectId','canvases','elements','tombstones'].includes(key)))throw fault('invalid_document_import'); + const canvases=fields.get('canvases') as {id:string;parent:null|{canvasId:string;entryElementId:string}}[],ids=new Set(); + for(const canvas of canvases){if(!canvas||typeof canvas.id!=='string'||!canvas.id||ids.has(canvas.id)||canvas.parent!==null&&(!canvas.parent||typeof canvas.parent.canvasId!=='string'||typeof canvas.parent.entryElementId!=='string'))throw fault('invalid_document_import');ids.add(canvas.id);} + const roots=canvases.filter(canvas=>canvas.parent===null);if(roots.length!==1)throw fault('invalid_document_import');const rootId=roots[0].id; + const byId=new Map(canvases.map(canvas=>[canvas.id,canvas]));let topologyWork=0; + for(const canvas of canvases){const seen=new Set();let current=canvas;while(current.parent){if(seen.has(current.id))throw fault('invalid_document_import');seen.add(current.id);if(++topologyWork%128===0){await yieldTask();if(signal.aborted)throw fault('import_aborted');}const parent=byId.get(current.parent.canvasId);if(!parent)throw fault('invalid_document_import');current=parent;}} + const tombstones=fields.get('tombstones');if(tombstones!==undefined&&(!tombstones||typeof tombstones!=='object'||Array.isArray(tombstones)||Object.values(tombstones).some(n=>!Number.isSafeInteger(n)||(n as number)<1)))throw fault('invalid_document_import'); + const children=canvases.filter(canvas=>canvas.id!==rootId),count=children.length+elements.length; + return Object.freeze({fingerprint:options.fingerprint,parserVersion:4,async read(position:number,readSignal:AbortSignal):Promise{ + if(readSignal.aborted)throw fault('import_aborted');if(!Number.isSafeInteger(position)||position<0||position>count)throw fault('invalid_import_cursor');if(position===count)return null; + const base={cursor:position,nextCursor:position+1,key:`record:${position}`};let record:any,kind:string; + try{if(positionnew AbortController().signal,source=value=>new Blob([typeof value==='string'?value:JSON.stringify(value)]),document={documentFormatVersion:1,projectId:'source',canvases:[{id:'root',parent:null},{id:'child',parent:{canvasId:'root',entryElementId:'frame'}}],elements:[{id:'frame',data:{title:'中文 [,] "'}},{id:'inside',ownerCanvasId:'child'}],tombstones:{old:2}}; +const open=bytes=>openDocumentImportSource({bytes,fingerprint:'original-file',signal:signal()}); +test('export envelope retains nested canvas records, emits root alias and seeks payloads by stable ordinal',async()=>{const reader=await open(source(document));assert.equal(reader.parserVersion,4);const a=await reader.read(0,signal()),b=await reader.read(1,signal());assert.equal(a.value.kind,'canvas');assert.equal(a.value.record.id,'child');assert.equal(a.value.sourceRootId,'root');assert.deepEqual(b.value.record,document.elements[0]);const reopened=await open(source(document));assert.deepEqual(await reopened.read(2,signal()),await reader.read(2,signal()));assert.equal(await reader.read(3,signal()),null);}); +test('envelope fields may be reordered, while future versions, duplicate keys and broken topology reject',async()=>{const reordered={elements:document.elements,tombstones:{},canvases:document.canvases,projectId:'source',documentFormatVersion:1};assert.equal((await(await open(source(reordered))).read(1,signal())).value.record.id,'frame');for(const bad of [{...document,documentFormatVersion:2},{...document,canvases:[...document.canvases,{id:'other',parent:null}]},{...document,canvases:[{id:'root',parent:null},{id:'child',parent:{canvasId:'missing',entryElementId:'e'}}]},{...document,tombstones:{x:0}},'{"documentFormatVersion":1,"documentFormatVersion":1}'])await assert.rejects(open(source(bad)));}); +test('payload errors are per record and large records are never decoded; malformed envelope tails reject',async()=>{const reader=await open(source({...document,elements:[{id:'huge',text:'x'.repeat(300000)},document.elements[1]]}));assert.equal((await reader.read(1,signal())).error,'record_too_large');assert.equal((await reader.read(2,signal())).value.record.id,'inside');await assert.rejects(open(source(JSON.stringify(document)+'garbage')));const controller=new AbortController();controller.abort();await assert.rejects(openDocumentImportSource({bytes:source(document),fingerprint:'x',signal:controller.signal}),/import_aborted/);});