Bind import recovery checkpoints to source content digests

This commit is contained in:
2026-11-08 19:34:32 +08:00
parent 627a043d92
commit a51d189b8e
9 changed files with 86 additions and 16 deletions
+14
View File
@@ -319,6 +319,18 @@ async function resourceRecoveryCase(name, assert) {
}
async function sourceIntegrityCase(name,assert){
const root=crypto.randomUUID(),workspace=crypto.randomUUID(),registry=createRegistry(builtinComponents()),databaseName=`${name}-source-integrity`,storage=createBrowserStorage({name:databaseName}),scope={accountId:crypto.randomUUID(),workspaceId:workspace,projectId:root},original=createDocument(root,[{id:root,parent:null}],[]);let document=original,versions=new Map([[root,1]]),engine,calls=0,raw;
const transport=async request=>{calls++;const body=JSON.parse(request.body);for(const[id,version]of Object.entries(body.expected_versions))assert(version===(versions.get(id)??0),'integrity lost a version fence');for(const operation of body.operations)document=applyChanges(document,operation.changes);for(const id of Object.keys(body.expected_versions))versions.set(id,(versions.get(id)??0)+1);return{status:200,body:{protocol_version:2,request_id:request.requestId,status:'applied',operation_ids:body.operations.map(row=>row.id),model:{documentFormatVersion:1,projectId:root,complete:false,removedCanvasIds:[],canvases:[...document.canvases.values()],elements:[...document.elements.values()],tombstones:Object.fromEntries(document.tombstones),versions:Object.fromEntries(versions)}}};};
const node=id=>({kind:'element',key:id,record:{id,incarnation:1,structure:'node',componentId:'text',dataVersion:1,ownerCanvasId:'$target',data:{title:id,text:''},geometry:{x:0,y:0,width:200,height:100,rotation:0}}}),bytes=new Blob([[node('first'),node('last')].map(row=>JSON.stringify(row)).join('\n')]),options={scope,registry,document:original,versions:new Map(versions),storage,persistence:createFoundryPersistence({workspaceId:workspace,rootId:root,registry,transport})};
try{
engine=await openProjectEngine(options);const id=engine.imports.startNdjson(bytes,root);await new Promise(resolve=>{const stop=engine.editor.subscribe((_,snapshot)=>{if(snapshot.runtime.imports.get(id).committedSourceKeys.length===1){stop();resolve();}});});await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id),first=before.stableIds['element:first'];assert(/^[a-f0-9]{64}$/.test(before.source.sha256),'source hash was not bound before publication');await engine.close();const sent=calls;
raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(databaseName);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const key=[JSON.stringify([scope.accountId,scope.workspaceId,scope.projectId]),before.source.fileId];
const write=async value=>{const tx=raw.transaction('files','readwrite'),done=new Promise((resolve,reject)=>{tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);});tx.objectStore('files').put(value,key);await done;};const text=await bytes.text(),changed=text.replaceAll('last','evil');assert(new Blob([changed]).size===bytes.size,'corruption fixture changed its byte length');await write(new Blob([changed]));engine=await openProjectEngine(options);await engine.imports.resume(id);await engine.settle();assert(engine.imports.getJobs().get(id).error==='import_source_changed','same-length source tampering was accepted');assert(engine.getSnapshot().document.elements.size===1&&engine.getSnapshot().document.elements.has(first)&&calls===sent,'source tampering changed the preserved prefix or sent new work');
await write(bytes);await engine.imports.resume(id);await engine.settle();assert(engine.imports.getJobs().get(id).status==='completed'&&document.elements.size===2&&engine.imports.getJobs().get(id).stableIds['element:first']===first,'restoring exact bytes did not resume the original task');
}finally{raw?.close();await engine?.close();storage.close();indexedDB.deleteDatabase(databaseName);}
}
async function deltaJournalCase(name,assert){
const databaseName=`${name}-delta-journal`,scope={accountId:'delta-account',workspaceId:'workspace',projectId:'project'},key=JSON.stringify(Object.values(scope)),rows=Array.from({length:100},(_,i)=>({id:String(i),text:'x'.repeat(1000)})),image=revision=>({formatVersion:1,rootRevision:revision,localRevision:revision,payload:{rows:rows.map((row,i)=>i===0?{...row,text:`version-${revision}`}:row),ledger:{frozenBody:'exact original request bytes'}}});
const done=tx=>new Promise((resolve,reject)=>{tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);}),read=request=>new Promise((resolve,reject)=>{request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
@@ -573,6 +585,8 @@ export async function run() {
results.push('owned immutable recovery images survive IndexedDB checksums; mutable images and file descriptors are captured before asynchronous work');
await cooperativeCopyCase(assert);
results.push('cooperative duplicate planning yields to actual browser tasks and cancellation without mutating the source');
await sourceIntegrityCase(name,assert);
results.push('source SHA256 detects same-length Blob tampering before recovery adds or sends a tail; exact bytes resume with the original prefix and IDs');
await deltaJournalCase(name,assert);
results.push('delta journals migrate legacy checkpoints, retain atomic files, verify hashes/base chains, reject old readers and compact through serialized immutable commits');
await importBackpressureCase(name,assert);
+1 -1
View File
@@ -14,7 +14,7 @@ test('snapshot decoding rejects malformed, future, duplicate, invalid and trunca
const controller=new AbortController();controller.abort();await assert.rejects(encodeCopySnapshot(document,controller.signal),/copy_aborted/);await assert.rejects(decodeCopySnapshot(new Blob([original]),'project',registry,controller.signal),/copy_aborted/);
});
test('preparation checkpoints require a matching interpreter, valid selection, unique allocations and no published progress',()=>{
const base={...createImportJob({id:'job',projectId:'project',sourceFingerprint:'snapshot',parserVersion:1,source:{kind:'duplicate-snapshot',fileId:'file',byteLength:10,targetCanvasId:'P',copy:{sourceRevision:0,roots:[]}}}),status:'preparing',preparation:{version:1,registrySignature:'registry-v1',selection:[{id:'A',incarnation:2}],worldOffset:{x:32,y:32},allocationIds:['new-A'],steps:512}};validateImportJob(base);
const base={...createImportJob({id:'job',projectId:'project',sourceFingerprint:'snapshot',parserVersion:1,source:{kind:'duplicate-snapshot',sha256:'a'.repeat(64),fileId:'file',byteLength:10,targetCanvasId:'P',copy:{sourceRevision:0,roots:[]}}}),status:'preparing',preparation:{version:1,registrySignature:'registry-v1',selection:[{id:'A',incarnation:2}],worldOffset:{x:32,y:32},allocationIds:['new-A'],steps:512}};validateImportJob(base);
for(const patch of [{preparation:undefined},{preparation:{...base.preparation,version:2}},{preparation:{...base.preparation,allocationIds:['same','same']}},{preparation:{...base.preparation,selection:[]}},{cursor:1},{committedSourceKeys:['element:A']},{status:'running'}])assert.throws(()=>validateImportJob({...base,...patch}));
assert.throws(()=>createImportRunner({getJob:()=>base}),/import_interpreter_unavailable/);
});
+13
View File
@@ -0,0 +1,13 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {importSourceDigest} from '../../src/lib/canvas-runtime/import-integrity.ts';
import {createImportJob,validateImportJob} from '../../src/lib/canvas-runtime/imports.ts';
test('source SHA256 binds complete bytes and cancellation fences both sides of asynchronous reading',async()=>{
assert.equal(await importSourceDigest(new Blob(['abc'])),'ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad');assert.notEqual(await importSourceDigest(new Blob(['abc'])),await importSourceDigest(new Blob(['abd'])));
const controller=new AbortController();controller.abort();await assert.rejects(importSourceDigest(new Blob(['abc']),controller.signal),/import_aborted/);let release;const gate=new Promise(resolve=>{release=resolve;}),later=new AbortController(),bytes=new class extends Blob{async arrayBuffer(){await gate;return super.arrayBuffer();}}(['abc']),result=importSourceDigest(bytes,later.signal);later.abort();release();await assert.rejects(result,/import_aborted/);
});
test('new source jobs declare integrity version and reject progress without a bound hash; legacy data remains readable',()=>{
const job=createImportJob({id:'i',projectId:'p',sourceFingerprint:'file',parserVersion:1,source:{kind:'native-ndjson',fileId:'file',targetCanvasId:'canvas',byteLength:10}});assert.equal(job.version,3);assert.equal(job.source.integrityVersion,1);validateImportJob(job);
assert.throws(()=>validateImportJob({...job,cursor:1}),/import_source_integrity_missing/);assert.throws(()=>validateImportJob({...job,stableIds:{'element:A':'new-A'}}),/import_source_integrity_missing/);assert.throws(()=>validateImportJob({...job,source:{...job.source,sha256:'invalid'}}),/invalid_import_source/);assert.throws(()=>validateImportJob({...job,source:{...job.source,integrityVersion:2}}),/import_source_integrity_interpreter_unavailable/);
const source={...job.source};delete source.integrityVersion;validateImportJob({...job,version:2,source,cursor:1});validateImportJob({...job,source:{...job.source,sha256:'a'.repeat(64)},cursor:1});
});
+19
View File
@@ -174,3 +174,22 @@ test('blocked source persistence admits several local batches then applies bound
gate.resolve();await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(f.server().elements.size,500);assert.equal(f.errors.length,0);assert.equal(f.files.size,1);decodeRecovery(f.image(),registry);
}finally{gate.resolve();await engine.close();}
});
test('same-length changes to an admitted copy snapshot are rejected before preparation allocates or publishes content',async()=>{
const f=fixture();let engine=await f.open();try{await seedCopy(engine);const size=engine.getSnapshot().document.elements.size,id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();const job=engine.imports.getJobs().get(id),raw=await f.files.get(job.source.fileId).text(),changed=raw.replace('source','tamper');assert.equal(changed.length,raw.length);await engine.close();const sent=f.requests.length;f.files.set(job.source.fileId,new Blob([changed]));engine=await f.open();await engine.imports.resume(id);await engine.settle();const blocked=engine.imports.getJobs().get(id);assert.equal(blocked.status,'paused');assert.equal(blocked.error,'import_source_changed');assert.equal(blocked.committedSourceKeys.length,0);assert.equal(engine.getSnapshot().document.elements.size,size);assert.equal(f.requests.length,sent);assert.equal(blocked.source.sha256,job.source.sha256);}finally{await engine.close();}
});
test('a changed unparsed tail cannot pass native import recovery or alter its preserved prefix',async()=>{
const f=fixture();let engine=await f.open();try{const id=engine.imports.startNdjson(file([node('one'),node('two'),node('three')]),'canvas');await until(engine,snapshot=>snapshot.runtime.imports.get(id).committedSourceKeys.length===1);await engine.imports.pause(id);await engine.settle();const job=engine.imports.getJobs().get(id),raw=await f.files.get(job.source.fileId).text(),changed=raw.replaceAll('three','other');assert.equal(changed.length,raw.length);const first=job.stableIds['element:one'];await engine.close();const sent=f.requests.length;f.files.set(job.source.fileId,new Blob([changed]));engine=await f.open();await engine.imports.resume(id);await engine.settle();assert.equal(engine.imports.getJobs().get(id).error,'import_source_changed');assert.equal(engine.getSnapshot().document.elements.size,1);assert.ok(engine.getSnapshot().document.elements.has(first));assert.equal(f.requests.length,sent);}finally{await engine.close();}
});
test('late source digest cannot revive a cancelled job or publish into a held recovery root',async()=>{
for(const cancel of [true,false]){
const f=fixture(),engine=await f.open(),entered=deferred(),gate=deferred(),bytes=new class extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}([JSON.stringify(node('one'))]);
try{const id=engine.imports.startNdjson(bytes,'canvas');await entered.promise;if(cancel){await engine.imports.cancel(id);gate.resolve();await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'cancelled');assert.equal(engine.getSnapshot().document.elements.size,0);assert.equal(engine.imports.getJobs().get(id).source.sha256,undefined);}else{const preparing=engine.prepareRecovery();gate.resolve();const session=await preparing,revision=engine.getSnapshot().rootRevision;await new Promise(resolve=>setTimeout(resolve,10));assert.equal(engine.getSnapshot().rootRevision,revision);assert.equal((await session.apply()).status,'applied');await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(f.server().elements.size,1);}assert.equal(f.errors.length,0);}finally{gate.resolve();await engine.close();}
}
});
test('legacy unfinished sources remain readable but cannot acquire a new content identity silently',async()=>{
const f=fixture();let engine=await f.open();try{const id=engine.imports.startNdjson(file([node('one'),node('two')]),'canvas');await until(engine,snapshot=>snapshot.runtime.imports.get(id).committedSourceKeys.length===1);await engine.imports.pause(id);await engine.settle();await engine.close();const image=f.image(),job=image.payload.runtime.imports.find(([key])=>key===id)[1];job.version=2;delete job.source.sha256;delete job.source.integrityVersion;const sent=f.requests.length;engine=await f.open();assert.equal(engine.imports.getJobs().get(id).status,'paused');await engine.imports.resume(id);await engine.settle();assert.equal(engine.imports.getJobs().get(id).error,'import_source_integrity_unavailable');assert.equal(engine.getSnapshot().document.elements.size,1);assert.equal(f.requests.length,sent);decodeRecovery(f.image(),registry);}finally{await engine.close();}
});
+3 -3
View File
@@ -64,7 +64,7 @@ test('dependency backpressure pauses with locators intact and a larger budget re
});
test('future interpreter versions and malformed job cuts fail before any source read',()=>{
const job=createImportJob({id:'i',projectId:'p',sourceFingerprint:'s',parserVersion:1});assert.throws(()=>validateImportJob({...job,version:3}),/import_interpreter_unavailable/);assert.throws(()=>validateImportJob({...job,stableIds:{a:'same',b:'same'}}),/invalid_import_job/);assert.throws(()=>validateImportJob({...job,status:'completed'}),/invalid_import_job/);
const job=createImportJob({id:'i',projectId:'p',sourceFingerprint:'s',parserVersion:1});assert.throws(()=>validateImportJob({...job,version:4}),/import_interpreter_unavailable/);assert.throws(()=>validateImportJob({...job,stableIds:{a:'same',b:'same'}}),/invalid_import_job/);assert.throws(()=>validateImportJob({...job,status:'completed'}),/invalid_import_job/);
});
test('fast independent records stage bounded publications and commits while the first record remains immediate',async()=>{
@@ -105,8 +105,8 @@ test('a durable pending cycle resumes without reallocating IDs or exposing a par
});
test('new jobs carry interpreter version 2 and a resumed legacy job upgrades in its first published cut',async()=>{
const f=fixture([{id:'one'}]);assert.equal(f.root.job.version,2);f.root={...f.root,job:{...f.root.job,version:1}};validateImportJob(f.root.job);const publish=f.ports.publish;f.ports.publish=(job,expected)=>{assert.equal(job.version,2);return publish(job,expected);};await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.job.version,2);
test('new jobs carry interpreter version 3 and a resumed legacy job upgrades in its first published cut',async()=>{
const f=fixture([{id:'one'}]);assert.equal(f.root.job.version,3);f.root={...f.root,job:{...f.root.job,version:1}};validateImportJob(f.root.job);const publish=f.ports.publish;f.ports.publish=(job,expected)=>{assert.equal(job.version,3);return publish(job,expected);};await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.job.version,3);
});
test('optimistic local checkpoints use host backpressure without declaring completion ahead of persistence',async()=>{