Yield native import scans by elapsed time and bounded work

This commit is contained in:
2026-11-09 19:41:00 +08:00
parent fd6c166f4b
commit 180f66b2af
9 changed files with 67 additions and 16 deletions
+1
View File
@@ -8,3 +8,4 @@ import {prepareNativeImport} from '../../src/lib/canvas-runtime/import-native.ts
import {createImportJob} from '../../src/lib/canvas-runtime/imports.ts';
import {createRegistry,createDocument} from '../../src/lib/canvas-core/index.ts';
test('document root aliases also remap component-declared canvas references without reserving a duplicate root',()=>{const registry=createRegistry([{id:'ref',dataVersion:1,structure:'node',features:[],validate:()=>true,copy:{references:[{path:['canvasRef'],target:'canvas',external:'reject'}],omit:[]}}]),target=createDocument('p',[{id:'target',parent:null}],[]),job=createImportJob({id:'job',projectId:'p',parserVersion:4,sourceFingerprint:'file',source:{kind:'native-document',fileId:'file',byteLength:100,targetCanvasId:'target'}}),reserved=[],record={id:'a',incarnation:1,ownerCanvasId:'source',componentId:'ref',dataVersion:1,structure:'node',geometry:{x:0,y:0,width:10,height:10,rotation:0},data:{canvasRef:'source'}},unit=prepareNativeImport({cursor:0,nextCursor:1,key:'record:0',value:{kind:'element',key:'a',record,sourceRootId:'source'}},job,key=>{reserved.push(key);return'new-'+key;},target,registry);assert.equal(unit.changes[0].after.data.canvasRef,'target');assert.equal(unit.changes[0].after.ownerCanvasId,'target');assert.ok(!reserved.includes('canvas:source'));});
test('document preparation yields by elapsed time even for files below the byte ceiling',async()=>{const controller=new AbortController();let clock=0,yields=0;await assert.rejects(openDocumentImportSource({bytes:source({...document,elements:[{id:'slow',text:'x'.repeat(2048)}]}),fingerprint:'slow',signal:controller.signal,now:()=>clock+=10,frameBudgetMs:6,yieldTask:async()=>{yields++;controller.abort();}}),/import_aborted/);assert.equal(yields,1);});
+2 -1
View File
@@ -5,4 +5,5 @@ async function all(reader){let cursor=0;const rows=[];for(let i=0;i<100;i++){con
test('array reader handles UTF8, escaped strings and nested delimiters across one-byte chunks and resumes from the same byte cursor',async()=>{const text=' [ {"title":"图\\\"片,]","nested":[1,{"x":"]"}]},\n {"title":"second"} ] ',reader=source(text,{chunkBytes:1}),rows=await all(reader);assert.deepEqual(rows.map(r=>r.value),JSON.parse(text));assert.ok(rows.every(r=>!r.error));const reopened=source(text,{chunkBytes:3});assert.deepEqual(await reopened.read(rows[1].cursor,signal()),rows[1]);assert.equal(await reopened.read(rows[1].nextCursor,signal()),null);});
test('structurally malformed tails report failure after the already accepted prefix',async()=>{for(const tail of [',]',',','','] garbage']){const rows=await all(source('[{"id":1},{"id":2}'+tail));assert.deepEqual(rows[0].value,{id:1});assert.ok(rows.at(-1).error,tail);}assert.deepEqual(await all(source('[]')),[]);await assert.rejects(all(source('')),/invalid_json_array/);assert.equal((await all(source('[1,]')))[0].error,'invalid_json_array');});
test('oversized records skip bounded payload and allow a later valid record; bad UTF8 remains explicit',async()=>{const rows=await all(source(JSON.stringify(['x'.repeat(200),{ok:true}]),{maxRecordBytes:32,chunkBytes:8}));assert.equal(rows[0].error,'record_too_large');assert.deepEqual(rows[1].value,{ok:true});const reader=createJsonArrayImportSource({bytes:new Blob([new Uint8Array([91,34,255,34,93])]),fingerprint:'bad'});assert.equal((await all(reader))[0].error,'invalid_utf8');});
test('long scans yield and honor abort without publishing a completed record',async()=>{const controller=new AbortController();let yields=0;const reader=source(JSON.stringify(['x'.repeat(40000)]),{yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(reader.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);await assert.rejects(reader.read(-1,signal()),/invalid_import_cursor/);});
test('long scans yield and honor abort without publishing a completed record',async()=>{const controller=new AbortController();let yields=0;const reader=source(JSON.stringify(['x'.repeat(40000)]),{now:()=>0,yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(reader.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);await assert.rejects(reader.read(-1,signal()),/invalid_import_cursor/);});
test('slow array scans yield by elapsed time before the fixed byte ceiling and abort at that boundary',async()=>{const controller=new AbortController();let clock=0,yields=0;const reader=source(JSON.stringify(['x'.repeat(2048)]),{now:()=>clock+=10,frameBudgetMs:6,yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(reader.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);});
+2
View File
@@ -141,3 +141,5 @@ test('a ready parser publication observes host backpressure before admitting its
test('async commit preparation cannot publish while waiting and cancellation fences a late successful result',async()=>{const gate=deferred(),entered=deferred(),f=fixture([{id:'image'}],{ports:{async beforeCommit(units,job,signal){assert.equal(f.root.document.elements.size,0);assert.ok(job.pending.length);entered.resolve();await gate.promise;return{enabled:true};}}});const running=f.runner.start();await entered.promise;assert.equal(f.commits.length,0);assert.equal(f.root.job.committedSourceKeys.length,0);await f.runner.cancel();gate.resolve();await running;await f.runner.settle();assert.equal(f.root.job.status,'cancelled');assert.equal(f.commits.length,0);assert.equal(f.root.document.elements.size,0);await f.runner.close();});
test('async preparation denial isolates independent groups without publishing the denied record',async()=>{const prepared=[],f=fixture([{id:'denied'},{id:'allowed'}],{ports:{async beforeCommit(units){prepared.push(units.map(unit=>unit.key));return units.some(unit=>unit.key==='denied')?{enabled:false,code:'asset_forbidden',targets:[]}:{enabled:true};}}});await f.runner.start();assert.equal(f.root.job.status,'partial');assert.ok(f.root.job.failures.some(row=>row.key==='denied'&&row.code==='asset_forbidden'));assert.equal(f.root.document.elements.size,1);assert.equal([...f.root.document.elements.values()][0].data.title,'allowed');assert.ok(prepared.length>=2);assert.equal(f.errors.length,0);await f.runner.close();});
test('a generation change during async preparation prevents the old attempt from committing or changing progress',async()=>{const gate=deferred(),entered=deferred(),f=fixture([{id:'later'}],{ports:{async beforeCommit(){entered.resolve();await gate.promise;return{enabled:true};}}});const running=f.runner.start();await entered.promise;f.root={...f.root,job:{...f.root.job,generation:f.root.job.generation+1}};const generation=f.root.job.generation;gate.resolve();await running;assert.equal(f.commits.length,0);assert.equal(f.root.job.generation,generation);assert.equal(f.root.job.committedSourceKeys.length,0);await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.document.elements.size,1);await f.runner.close();});
test('NDJSON checks elapsed time before returning a short scanned record and honors cancellation',async()=>{const controller=new AbortController();let clock=0,yields=0;const source=createNdjsonImportSource({bytes:new Blob([JSON.stringify({text:'x'.repeat(2048)})+'\n']),fingerprint:'timed',chunkBytes:256,now:()=>clock+=10,frameBudgetMs:6,yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(source.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);});