Decode duplicate payloads on demand from indexed snapshots
This commit is contained in:
@@ -404,7 +404,7 @@ async function duplicatePlannerRecoveryCase(name,assert){
|
||||
try{
|
||||
engine=await openProjectEngine(options);const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);
|
||||
await new Promise((resolve,reject)=>{const timer=setTimeout(()=>{release();reject(new Error('no durable DFS checkpoint'));},30000);const release=engine.editor.subscribe((_,snapshot)=>{const planner=snapshot.runtime.imports.get(id)?.preparation?.planner;if(planner?.phase==='dfs'&&planner.work.length){clearTimeout(timer);release();resolve();}});});
|
||||
await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id),saved=JSON.stringify(before.preparation.planner);assert(before.preparation.version===3&&before.preparation.planner.phase==='dfs','did not pause within DFS');assert(before.preparation.planner.sourceToken===before.source.sha256,'checkpoint lost source binding');assert(before.committedSourceKeys.length===0,'preparation published content prematurely');await engine.close();engine=await openProjectEngine(options);
|
||||
await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id),saved=JSON.stringify(before.preparation.planner);assert(before.preparation.version===4&&before.preparation.planner.phase==='dfs','did not pause within DFS');assert(before.preparation.planner.sourceToken===before.source.sha256,'checkpoint lost source binding');assert(before.committedSourceKeys.length===0,'preparation published content prematurely');await engine.close();engine=await openProjectEngine(options);
|
||||
assert(JSON.stringify(engine.imports.getJobs().get(id).preparation.planner)===saved,'IndexedDB changed the DFS checkpoint');let resumed=false;const release=engine.editor.subscribe((_,snapshot)=>{const planner=snapshot.runtime.imports.get(id)?.preparation?.planner;if(planner){assert(planner.steps>=before.preparation.steps,'resume regressed the cursor');assert(!['selection','canvases','members','roots','seed'].includes(planner.phase),'resume repeated an earlier preparation phase');resumed=true;}});
|
||||
await engine.imports.resume(id);await engine.settle();release();const job=engine.imports.getJobs().get(id);assert(resumed&&job.status==='completed'&&job.failures.length===0,'DFS recovery did not complete');assert(job.committedSourceKeys.length===502&&Object.keys(job.stableIds).length===502,'DFS recovery lost a source record');assert(document.elements.size===1002&&document.canvases.size===3,'DFS recovery duplicated or lost content');const copiedChild=job.stableIds[`canvas:${child}`];for(let i=0;i<500;i++)assert(document.elements.get(job.stableIds[`element:member-${i}`])?.ownerCanvasId===copiedChild,'DFS sibling lost its copied owner');
|
||||
}finally{await engine?.close();storage.close();indexedDB.deleteDatabase(database);}
|
||||
@@ -610,7 +610,7 @@ export async function run() {
|
||||
await referenceCycleCase(name,assert);
|
||||
results.push('native frame reference cycles and copied port mappings commit atomically through protocol 2 and real IndexedDB undo/redo/reopening');
|
||||
await duplicatePlannerRecoveryCase(name,assert);
|
||||
results.push('preparation v2 persists a DFS stack/cursor in real IndexedDB and resumes 500 siblings without restarting scans or duplicating identities');
|
||||
results.push('preparation v4 persists a DFS stack/cursor in real IndexedDB and resumes 500 siblings without restarting scans or duplicating identities');
|
||||
await duplicateRecoveryCase(name,assert);
|
||||
results.push('deep copy source/mapping survive IndexedDB pause/reopen/resume, retain C to F-prime and shared assets, preserve edits and do not revive undone content');
|
||||
await nativeImportRecoveryCase(name,assert);
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createDocument,createRegistry,builtinComponents,planDuplicate,validateDocument} from '../../src/lib/canvas-core/index.ts';
|
||||
import {encodeIndexedCopySnapshot,openIndexedCopySnapshot} from '../../src/lib/canvas-runtime/copy-snapshot-index.ts';
|
||||
import {prepareDuplicateStream} from '../../src/lib/canvas-runtime/duplicate.ts';
|
||||
import {copyRegistrySignature,decodeCopySnapshot} from '../../src/lib/canvas-runtime/copy-snapshot.ts';
|
||||
import {createDuplicatePlanSource} from '../../src/lib/canvas-runtime/duplicate-source.ts';
|
||||
const registry=createRegistry(builtinComponents()),signal=()=>new AbortController().signal,node=(id,owner='P',componentId='text')=>({id,incarnation:2,structure:'node',ownerCanvasId:owner,componentId,dataVersion:1,data:{title:id,text:'unparsed-payload-秘密',extra:{retained:true}},geometry:{x:10,y:20,width:200,height:100,rotation:0}});
|
||||
const document=createDocument('project',[{id:'P',parent:null},{id:'S',parent:{canvasId:'P',entryElementId:'F'}}],[node('C'),{...node('F','P','frame'),data:{...node('F').data,ports:[{id:'port',nodeId:'A'}],presentation:{unitScale:2,contentScale:.75,other:'not-needed-in-index'}},geometry:{...node('F').geometry,rotation:35}},node('A','S','image'),{id:'edge',incarnation:1,structure:'connection',ownerCanvasId:'P',componentId:'connection',dataVersion:1,data:{retained:true},source:{elementId:'C'},target:{elementId:'A'},path:{type:'bezier-chain',knots:[{id:'k',mode:'corner',position:{x:70,y:80},in:{x:1,y:2},out:{x:3,y:4}}]}}],new Map([['deleted',4]]));
|
||||
const lines=async bytes=>(await bytes.text()).trimEnd().split('\n');
|
||||
|
||||
test('indexed snapshots read topology before payload, preserve raw records on demand and reproduce exact mapped output',async()=>{
|
||||
const bytes=await encodeIndexedCopySnapshot(document,signal()),snapshot=await openIndexedCopySnapshot(bytes,'project',registry,signal());assert.equal(JSON.stringify([...snapshot.document.elements]).includes('unparsed-payload'),false);assert.equal(snapshot.document.elements.get('F').data.ports,undefined);assert.deepEqual(snapshot.document.elements.get('F').data.presentation,{unitScale:2,contentScale:.75});assert.deepEqual([...snapshot.document.tombstones],[...document.tombstones]);assert.throws(()=>validateDocument(snapshot.document,registry));
|
||||
const loaded=[];let next=0;const state=await prepareDuplicateStream(snapshot.document,registry,[{id:'F',incarnation:2}],{createId:()=>`copy-${++next}`},{sourceToken:'a'.repeat(64),schemaToken:copyRegistrySignature(snapshot.document,registry),topologyOnly:true});assert.equal(loaded.length,0);const source=createDuplicatePlanSource(snapshot.document,registry,state,'indexed',async(id,abort)=>{loaded.push(id);return snapshot.readElement(id,abort);});let reference=0;const expected=planDuplicate(document,registry,[{id:'F',incarnation:2}],{createId:()=>`copy-${++reference}`});
|
||||
const last=await source.read(expected.changes.length-1,signal());assert.deepEqual(last.value.change,expected.changes.at(-1));assert.deepEqual(loaded,['edge'],'edge geometry decoded unrelated node payload');
|
||||
for(let i=0;i<expected.changes.length;i++)assert.deepEqual((await source.read(i,signal())).value.change,expected.changes[i]);for(const[id,record]of document.elements)assert.deepEqual(await snapshot.readElement(id,signal()),record);await assert.rejects(decodeCopySnapshot(bytes,'project',registry,signal()),/invalid_copy_snapshot/);
|
||||
});
|
||||
|
||||
test('index decoding defers corrupt or invalid component content until the corresponding record is requested',async()=>{
|
||||
const rows=await lines(await encodeIndexedCopySnapshot(document,signal())),last=rows.length-1;rows[last]='!'+rows[last].slice(1);const corrupt=new Blob([rows.join('\n')+'\n']),snapshot=await openIndexedCopySnapshot(corrupt,'project',registry,signal());assert.equal((await snapshot.readElement('A',signal())).id,'A');await assert.rejects(snapshot.readElement('edge',signal()),/invalid_copy_snapshot/);
|
||||
const clean=await lines(await encodeIndexedCopySnapshot(document,signal())),i=clean.findIndex(row=>row.startsWith('{"kind":"element","record":{"id":"A"'));clean[i]=clean[i].replace('"id":"A"','"id":"Z"');const mismatch=await openIndexedCopySnapshot(new Blob([clean.join('\n')+'\n']),'project',registry,signal());await assert.rejects(mismatch.readElement('A',signal()),/invalid_copy_snapshot/);
|
||||
});
|
||||
|
||||
test('indexed source rejects locator gaps, geometry corruption, truncation, foreign versions and aborted reads',async()=>{
|
||||
const bytes=await encodeIndexedCopySnapshot(document,signal()),rows=await lines(bytes),index=rows.findIndex(row=>JSON.parse(row).kind==='element-index');
|
||||
for(const change of [row=>({...row,offset:1}),row=>({...row,byteLength:0}),row=>({...row,record:{...row.record,geometry:{...row.record.geometry,width:0}}})]){const bad=[...rows];bad[index]=JSON.stringify(change(JSON.parse(bad[index])));await assert.rejects(openIndexedCopySnapshot(new Blob([bad.join('\n')+'\n']),'project',registry,signal()));}
|
||||
await assert.rejects(openIndexedCopySnapshot(bytes.slice(0,bytes.size-1),'project',registry,signal()));await assert.rejects(openIndexedCopySnapshot(new Blob([await bytes.text(),'x']),'project',registry,signal()));await assert.rejects(openIndexedCopySnapshot(bytes,'foreign',registry,signal()));const controller=new AbortController();controller.abort();await assert.rejects(encodeIndexedCopySnapshot(document,controller.signal),/copy_aborted/);await assert.rejects(openIndexedCopySnapshot(bytes,'project',registry,controller.signal),/copy_aborted/);const snapshot=await openIndexedCopySnapshot(bytes,'project',registry,signal());await assert.rejects(snapshot.readElement('A',controller.signal),/copy_aborted/);
|
||||
});
|
||||
|
||||
test('bounded raw windows preserve sequential parser batching and evict old bytes without caching decoded content',async()=>{
|
||||
const source=createDocument('project',[{id:'P',parent:null}],Array.from({length:120},(_,i)=>({...node(`record-${i}`),data:{title:`record-${i}`,text:'x'.repeat(3500)}}))),bytes=await encodeIndexedCopySnapshot(source,signal()),reads=[];
|
||||
class ObservedBlob extends Blob{slice(...args){reads.push(args);return super.slice(...args);}}
|
||||
const snapshot=await openIndexedCopySnapshot(new ObservedBlob([bytes]),'project',registry,signal());reads.length=0;
|
||||
for(let i=0;i<120;i++)assert.equal((await snapshot.readElement(`record-${i}`,signal())).data.title,`record-${i}`);assert.ok(reads.length>=5&&reads.length<120/4,`unexpected per-record Blob reads: ${reads.length}`);const count=reads.length;assert.equal((await snapshot.readElement('record-0',signal())).data.text.length,3500);assert.equal(reads.length,count+1,'first window did not leave the bounded cache');
|
||||
});
|
||||
@@ -15,6 +15,6 @@ test('snapshot decoding rejects malformed, future, duplicate, invalid and trunca
|
||||
});
|
||||
test('preparation checkpoints require a matching interpreter, valid selection, unique allocations and no published progress',()=>{
|
||||
const base={...createImportJob({id:'job',projectId:'project',sourceFingerprint:'snapshot',parserVersion:1,source:{kind:'duplicate-snapshot',sha256:'a'.repeat(64),fileId:'file',byteLength:10,targetCanvasId:'P',copy:{sourceRevision:0,roots:[]}}}),status:'preparing',preparation:{version:1,registrySignature:'registry-v1',selection:[{id:'A',incarnation:2}],worldOffset:{x:32,y:32},allocationIds:['new-A'],steps:512}};validateImportJob(base);
|
||||
for(const patch of [{preparation:undefined},{preparation:{...base.preparation,version:4}},{preparation:{...base.preparation,allocationIds:['same','same']}},{preparation:{...base.preparation,selection:[]}},{cursor:1},{committedSourceKeys:['element:A']},{status:'running'}])assert.throws(()=>validateImportJob({...base,...patch}));
|
||||
for(const patch of [{preparation:undefined},{preparation:{...base.preparation,version:5}},{preparation:{...base.preparation,allocationIds:['same','same']}},{preparation:{...base.preparation,selection:[]}},{cursor:1},{committedSourceKeys:['element:A']},{status:'running'},{source:{...base.source,snapshotVersion:3}},{source:{...base.source,snapshotVersion:2}}])assert.throws(()=>validateImportJob({...base,...patch}));
|
||||
assert.throws(()=>createImportRunner({getJob:()=>base}),/import_interpreter_unavailable/);
|
||||
});
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import {encodeCopySnapshot} from '../../src/lib/canvas-runtime/copy-snapshot.ts';
|
||||
import {importSourceDigest} from '../../src/lib/canvas-runtime/import-integrity.ts';
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createDocument,createRegistry,builtinComponents,applyChanges} from '../../src/lib/canvas-core/index.ts';
|
||||
@@ -130,7 +132,7 @@ test('read-only policy rejects copy/import admission without storing a source or
|
||||
test('preparing copy resumes saved planning cursors and allocated identities without repeating earlier phases',async(t)=>{
|
||||
const f=fixture(),save=f.storage.commit;let durableCuts=0;f.storage.commit=async(...args)=>{durableCuts++;return save(...args);};let engine=await f.open();try{
|
||||
await seedCopy(engine);assert.equal(engine.editor.transaction(tx=>{for(let i=0;i<1000;i++)tx.dispatch({type:'element.create',element:copyNode(`extra-${i}`,'child')});}).status,'applied');await engine.settle();
|
||||
const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>(snapshot.runtime.imports.get(id).preparation?.allocationIds.length??0)>0);await engine.imports.pause(id);await engine.settle();const checkpoint=engine.imports.getJobs().get(id);assert.equal(checkpoint.status,'paused');assert.equal(checkpoint.source.kind,'duplicate-snapshot');assert.ok(checkpoint.preparation.allocationIds.length>0);assert.ok(checkpoint.preparation.steps>0);assert.equal(checkpoint.preparation.version,3);assert.ok(checkpoint.preparation.planner);assert.equal(checkpoint.preparation.planner.sourceToken,checkpoint.source.sha256);assert.equal(checkpoint.committedSourceKeys.length,0);decodeRecovery(f.image(),registry);
|
||||
const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>(snapshot.runtime.imports.get(id).preparation?.allocationIds.length??0)>0);await engine.imports.pause(id);await engine.settle();const checkpoint=engine.imports.getJobs().get(id);assert.equal(checkpoint.status,'paused');assert.equal(checkpoint.source.kind,'duplicate-snapshot');assert.ok(checkpoint.preparation.allocationIds.length>0);assert.ok(checkpoint.preparation.steps>0);assert.equal(checkpoint.preparation.version,4);assert.ok(checkpoint.preparation.planner);assert.equal(checkpoint.preparation.planner.sourceToken,checkpoint.source.sha256);assert.equal(checkpoint.committedSourceKeys.length,0);decodeRecovery(f.image(),registry);
|
||||
engine.editor.dispatch({type:'element.data',target:{id:'F',incarnation:1},patch:{title:'new source title'}});await engine.settle();await engine.close();engine=await f.open();assert.equal(engine.imports.getJobs().get(id).status,'paused');assert.deepEqual(engine.imports.getJobs().get(id).preparation.allocationIds,checkpoint.preparation.allocationIds);assert.deepEqual(engine.imports.getJobs().get(id).preparation.planner,checkpoint.preparation.planner);const observed=[],stop=engine.editor.subscribe((_,snapshot)=>{const state=snapshot.runtime.imports.get(id)?.preparation?.planner;if(state)observed.push({phase:state.phase,steps:state.steps});});await engine.imports.resume(id);await engine.settle();stop();assert.ok(observed.some(state=>state.steps>checkpoint.preparation.steps));assert.ok(observed.every(state=>state.steps>=checkpoint.preparation.steps&&!['selection','canvases','members','roots','seed','dfs','edges'].includes(state.phase)));await engine.settle();
|
||||
const job=engine.imports.getJobs().get(id);assert.equal(job.status,'completed');assert.deepEqual(Object.values(job.stableIds).slice(0,checkpoint.preparation.allocationIds.length),checkpoint.preparation.allocationIds);assert.equal(engine.getSnapshot().document.elements.get(job.stableIds['element:F']).data.title,'F');assert.equal(engine.getSnapshot().document.elements.get('F').data.title,'new source title');assert.equal(job.preparation,undefined);assert.equal(f.errors.length,0);t.diagnostic(JSON.stringify({imported:job.committedSourceKeys.length,durableCuts,requests:f.requests.length}));
|
||||
}finally{await engine.close();}
|
||||
@@ -195,8 +197,10 @@ test('legacy unfinished sources remain readable but cannot acquire a new content
|
||||
});
|
||||
|
||||
|
||||
async function legacyCopySource(f,job,document){const signal=new AbortController().signal,bytes=await encodeCopySnapshot(document,signal);delete job.source.snapshotVersion;job.source.byteLength=bytes.size;job.source.sha256=await importSourceDigest(bytes,signal);job.sourceFingerprint=`local-file:${job.source.fileId}:${bytes.size}`;f.files.set(job.source.fileId,bytes);}
|
||||
|
||||
test('legacy preparation version 1 still replays its reserved identities instead of allocating replacements',async()=>{
|
||||
const f=fixture();let engine=await f.open();try{await seedCopy(engine);const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();await engine.close();const job=f.image().payload.runtime.imports.find(([key])=>key===id)[1];job.preparation.version=1;job.preparation.steps=1;job.preparation.allocationIds=['legacy-frame-id'];delete job.preparation.planner;engine=await f.open();await engine.imports.resume(id);await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(engine.imports.getJobs().get(id).stableIds['element:F'],'legacy-frame-id');assert.equal(f.errors.length,0);}finally{await engine.close();}
|
||||
const f=fixture();let engine=await f.open();try{await seedCopy(engine);const sourceDocument=engine.getSnapshot().document,id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();await engine.close();const job=f.image().payload.runtime.imports.find(([key])=>key===id)[1];job.preparation.version=1;job.preparation.steps=1;job.preparation.allocationIds=['legacy-frame-id'];delete job.preparation.planner;await legacyCopySource(f,job,sourceDocument);engine=await f.open();await engine.imports.resume(id);await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(engine.imports.getJobs().get(id).stableIds['element:F'],'legacy-frame-id');assert.equal(f.errors.length,0);}finally{await engine.close();}
|
||||
});
|
||||
|
||||
|
||||
@@ -206,8 +210,8 @@ test('copy converts and publishes a prefix before the output tail, preserves it
|
||||
}finally{await engine.close();}
|
||||
});
|
||||
|
||||
test('legacy preparation version 2 retains its complete-plan source path after reopening',async()=>{
|
||||
const f=fixture();let engine=await f.open();try{await seedCopy(engine);const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();await engine.close();const job=f.image().payload.runtime.imports.find(([key])=>key===id)[1];job.preparation.version=2;engine=await f.open();await engine.imports.resume(id);await engine.settle();const done=engine.imports.getJobs().get(id);assert.equal(done.status,'completed');assert.equal(done.source.kind,'duplicate-ndjson');assert.equal(done.parserVersion,1);assert.equal(f.errors.length,0);}finally{await engine.close();}
|
||||
test('legacy preparation versions 2 and 3 retain their source interpreters after reopening',async()=>{
|
||||
for(const version of [2,3]){const f=fixture();let engine=await f.open();try{await seedCopy(engine);const sourceDocument=engine.getSnapshot().document,id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();await engine.close();const job=f.image().payload.runtime.imports.find(([key])=>key===id)[1];job.preparation.version=version;await legacyCopySource(f,job,sourceDocument);engine=await f.open();await engine.imports.resume(id);await engine.settle();const done=engine.imports.getJobs().get(id);assert.equal(done.status,'completed');assert.equal(done.source.kind,version===2?'duplicate-ndjson':'duplicate-plan');assert.equal(done.parserVersion,version===2?1:2);assert.equal(f.errors.length,0);}finally{await engine.close();}}
|
||||
});
|
||||
|
||||
test('late planned-source decoding respects cancellation and canonical recovery holds',async()=>{
|
||||
@@ -221,3 +225,13 @@ test('late planned-source decoding respects cancellation and canonical recovery
|
||||
}finally{gate.resolve();await engine.close();}
|
||||
}
|
||||
});
|
||||
|
||||
test('indexed copy displays a prefix while a later raw payload read is still blocked',async()=>{
|
||||
const source=createDocument('project',[{id:'canvas',parent:null},{id:'child',parent:{canvasId:'canvas',entryElementId:'F'}}],[copyNode('F','canvas','frame'),...Array.from({length:100},(_,i)=>({...copyNode(`lazy-${i}`,'child'),data:{title:`lazy-${i}`,text:'x'.repeat(3000)}}))]),f=fixture(source);let engine=await f.open();const entered=deferred(),gate=deferred();try{
|
||||
const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id);await engine.close();const bytes=f.files.get(before.source.fileId),rows=(await bytes.text()).trimEnd().split('\n');let payload=0;
|
||||
for(const text of rows){const row=JSON.parse(text);payload+=new TextEncoder().encode(text+'\n').byteLength;if(row.kind==='payload')break;}
|
||||
class DelayedChunk extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}
|
||||
class DelayedPayload extends Blob{slice(start,end,...rest){const part=super.slice(start,end,...rest);return start>=payload+64*1024?new DelayedChunk([part]):part;}}
|
||||
f.files.set(before.source.fileId,new DelayedPayload([bytes]));engine=await f.open();await engine.imports.resume(id);await entered.promise;const cut=engine.getSnapshot(),job=cut.runtime.imports.get(id);assert.equal(job.source.snapshotVersion,2);assert.equal(job.status,'running');assert.ok(job.committedSourceKeys.length>0&&job.committedSourceKeys.length<102);assert.ok(cut.document.elements.size>source.elements.size&&cut.document.elements.size<202,'raw tail read gated the entire copy');gate.resolve();await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(engine.getSnapshot().document.elements.size,202);assert.equal(f.errors.length,0);
|
||||
}finally{gate.resolve();await engine.close();}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user