Inventory scoped files and atomically collect verified legacy orphans
This commit is contained in:
@@ -317,6 +317,22 @@ async function importSourceCollectionCase(name,assert){
|
||||
}finally{await engine?.close();storage.close();indexedDB.deleteDatabase(database);}
|
||||
}
|
||||
|
||||
async function orphanInventoryCase(name,assert){
|
||||
const database=`${name}-orphan-inventory`,storage=createBrowserStorage({name:database,checkpointEvery:3}),scope={accountId:'account',workspaceId:'workspace',projectId:'project'},registry=createRegistry(builtinComponents()),document=createDocument('project',[{id:'P',parent:null}],[{id:'image',incarnation:1,ownerCanvasId:'P',structure:'node',componentId:'image',dataVersion:1,data:{title:'image',text:'',assetId:'z-pinned'},geometry:{x:0,y:0,width:100,height:100,rotation:0}}]);let engine,owner,other,now=1000;
|
||||
const options={scope,registry,document,versions:new Map([['P',1]]),storage:{...storage,claim:async scope=>{owner=await storage.claim(scope);return owner;}},persistence:{compile(){throw Error('unexpected write');},async submit(){throw Error('unexpected write');},async query(){throw Error('unexpected write');},canonical(){throw Error('unexpected write');}},clock:{now:()=>now,setTimer:(fn,ms)=>setTimeout(fn,ms),clearTimer:id=>clearTimeout(id)}};
|
||||
try{
|
||||
engine=await openProjectEngine(options);await engine.persist();await engine.close();let legacy=await storage.claim(scope),cut=await storage.recover(legacy);await storage.commit(legacy,{image:{...cut,rootRevision:cut.rootRevision+1},files:[{id:'a-orphan',bytes:new Blob(['orphan bytes'])},{id:'z-pinned',bytes:new Blob(['pinned bytes'])}]});await legacy.release();
|
||||
other=await storage.claim({...scope,projectId:'other'});await storage.commit(other,{image:{formatVersion:1,rootRevision:0,localRevision:0,payload:{}},files:[{id:'a-foreign',bytes:new Blob(['other project'])}]});engine=await openProjectEngine(options);
|
||||
const before=engine.getSnapshot(),first=await engine.inventoryLocalFiles({limit:1});assert(first.files.length===1&&first.files[0].id==='a-orphan'&&first.nextAfter==='a-orphan','inventory paging/scope prefix is wrong');assert(engine.getSnapshot()===before,'read-only inventory mutated the Root');const last=await engine.inventoryLocalFiles({after:first.nextAfter,limit:1});assert(last.files[0].id==='z-pinned'&&last.files[0].pins.length>0&&last.nextAfter===undefined,'referenced unregistered bytes lost their pin');
|
||||
await engine.collectOrphanFiles({retentionMs:100});now+=100;const nativeDelete=IDBObjectStore.prototype.delete;let injected=false;
|
||||
IDBObjectStore.prototype.delete=function(key){const request=nativeDelete.call(this,key);if(!injected&&this.name==='files'&&Array.isArray(key)&&key[1]==='a-orphan'){injected=true;throw Error('orphan_gc_abort');}return request;};
|
||||
let failure;try{await engine.collectOrphanFiles({retentionMs:100});}catch(error){failure=error.message;}finally{IDBObjectStore.prototype.delete=nativeDelete;}
|
||||
assert(injected&&failure==='orphan_gc_abort','orphan GC did not reach post-delete rollback');assert(await(await storage.readFile(owner,'a-orphan')).text()==='orphan bytes','orphan byte rollback failed');cut=await storage.recover(owner);assert(cut.formatVersion===5&&cut.payload.runtime.orphanFiles[0][1].retirement.purgedAt===undefined,'aborted orphan GC advanced its cut');await engine.persist();assert(await storage.readFile(owner,'a-orphan')===undefined,'orphan deletion retry failed');assert(await(await storage.readFile(owner,'z-pinned')).text()==='pinned bytes','orphan GC deleted referenced bytes');assert(await(await storage.readFile(other,'a-foreign')).text()==='other project','orphan GC crossed its project scope');
|
||||
cut=await storage.recover(owner);assert(decodeRecovery(cut,registry).runtime.orphanFiles.get('a-orphan').retirement.purgedAt===1100,'orphan v5 record failed recovery');let stale;try{await storage.scanFiles(owner,{expectedRootRevision:before.rootRevision,limit:1});}catch(error){stale=error.code;}assert(stale==='stale_file_inventory','old inventory revision was accepted');
|
||||
await engine.close();legacy=await storage.claim(scope);let reuse;try{await storage.commit(legacy,{image:{...cut,rootRevision:cut.rootRevision+1},files:[{id:'a-orphan',bytes:new Blob(['reused identity'])}]});}catch(error){reuse=error.code;}assert(reuse==='resource_identity_conflict','retired file identity was reused');await legacy.release();engine=await openProjectEngine(options);let retired;try{await engine.displayResource('a-orphan');}catch(error){retired=error.code;}assert(retired==='local_file_retired','retired orphan became a guessed remote asset');assert(engine.getSnapshot().document.elements.has('image'),'orphan GC mutated the document');
|
||||
}finally{await engine?.close();await other?.release();storage.close();indexedDB.deleteDatabase(database);}
|
||||
}
|
||||
|
||||
async function resourceRecoveryCase(name, assert) {
|
||||
const scope = { accountId: crypto.randomUUID(), workspaceId: crypto.randomUUID(), projectId: crypto.randomUUID() };
|
||||
const registry = createRegistry(builtinComponents()), document = createDocument(scope.projectId, [{ id: scope.projectId, parent: null }], []);
|
||||
@@ -607,7 +623,7 @@ export async function run() {
|
||||
await storage.commit(leader, { image: image(1) });
|
||||
await rejects(() => storage.commit(leader, { image: { ...image(1), payload: null } }), 'storage_revision_conflict');
|
||||
await rejects(() => storage.commit(leader, { image: image(3) }), 'storage_revision_gap');
|
||||
await rejects(() => storage.commit(leader, { image: { ...image(2), formatVersion: 5 } }), 'unsupported_recovery_version');
|
||||
await rejects(() => storage.commit(leader, { image: { ...image(2), formatVersion: 6 } }), 'unsupported_recovery_version');
|
||||
results.push('same-cut retry is idempotent; conflicting, skipped and future-format cuts reject');
|
||||
|
||||
// The first put succeeds before a later duplicate resource aborts the whole transaction.
|
||||
@@ -652,6 +668,8 @@ export async function run() {
|
||||
results.push('atomic local file collection rolls back injected failure after IDB delete, captures descriptors, rejects missing markers and survives reopening');
|
||||
await importSourceCollectionCase(name,assert);
|
||||
results.push('terminal import source GC atomically rolls back a post-delete fault, retries its v4 cut, preserves partial reports and reopens with undo/redo independent of source bytes');
|
||||
await orphanInventoryCase(name,assert);
|
||||
results.push('scoped paged inventory finds legacy orphans without hiding references; orphan v5 deletion rolls back faults, retries, isolates projects and forbids byte identity reuse');
|
||||
await resourceRecoveryCase(name, assert);
|
||||
results.push('staged file and image recover from real IndexedDB; lost upload ACK queries stable resource identity');
|
||||
await digestWorkerCase(assert);
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
import {test} from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import {createDocument,createRegistry,builtinComponents,applyChanges} from '../../src/lib/canvas-core/index.ts';
|
||||
import {openProjectEngine} from '../../src/lib/canvas-runtime/engine.ts';
|
||||
import {decodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts';
|
||||
import {orphanCoverage} from '../../src/lib/canvas-runtime/orphan-files.ts';
|
||||
const registry=createRegistry(builtinComponents()),scope={accountId:'a',workspaceId:'w',projectId:'p'},node=(id,data={})=>({id,incarnation:1,structure:'node',ownerCanvasId:'C',componentId:'image',dataVersion:1,geometry:{x:0,y:0,width:100,height:100,rotation:0},data:{title:id,text:'',...data}}),original=createDocument('p',[{id:'C',parent:null}],[node('A')]);
|
||||
const deferred=()=>{let resolve;const promise=new Promise(yes=>resolve=yes);return{promise,resolve};};
|
||||
function fixture(files=new Map(),document=original){
|
||||
let image,owner,next=0,server=document,version=1,now=1000;const batches=[],requests=[],receipts=new Map();
|
||||
const storage={fileDeletion:'atomic-v1',async claim(){assert.equal(owner,undefined);const leader={scope,fence:1,active:true,async release(){owner=undefined;leader.active=false;}};owner=leader;return leader;},async recover(){return image?structuredClone(image):null;},async commit(leader,batch){assert.equal(leader,owner);for(const file of batch.files??[])files.set(file.id,file.bytes);for(const id of batch.deleteFiles??[])files.delete(id);image=structuredClone(batch.image);batches.push(batch);return{durableRootRevision:image.rootRevision};},async readFile(leader,id){assert.equal(leader,owner);return files.get(id);},async scanFiles(leader,{expectedRootRevision,after,limit}){assert.equal(leader,owner);if(image.rootRevision!==expectedRootRevision)throw Error('stale_file_inventory');const rows=[...files].sort(([a],[b])=>a<b?-1:a>b?1:0).filter(([id])=>after===undefined||id>after),page=rows.slice(0,limit).map(([id,blob])=>({id,byteLength:blob.size,mimeType:blob.type}));return{files:page,rootRevision:image.rootRevision,...(rows.length>limit?{nextAfter:page.at(-1).id}:{})};},close(){}};
|
||||
const port={compile:(intent,id)=>({payload:{intent,id},canvasIds:intent.canvasIds,expectedVersions:{C:version}}),async submit(request){requests.push(request);server=applyChanges(server,JSON.parse(request.body).intent.changes);receipts.set(request.id,{document:server,versions:new Map([['C',++version]])});return{outcome:'applied'};},async query(request){return{outcome:receipts.has(request.id)?'applied':'unresolved'};},canonical:request=>receipts.get(request.id)};
|
||||
return{files,batches,requests,storage,advance:ms=>{now+=ms;},image:()=>image,open:()=>openProjectEngine({scope,registry,document,versions:new Map([['C',1]]),storage,persistence:port,createId:()=>`id-${++next}`,clock:{now:()=>now,setTimer:(fn,ms)=>setTimeout(fn,ms),clearTimer:id=>clearTimeout(id)}})};
|
||||
}
|
||||
test('inventory is bounded and read-only; unregistered but referenced files stay protected',async()=>{
|
||||
const doc=createDocument('p',[{id:'C',parent:null}],[node('A',{assetId:'used'})]),f=fixture(new Map([['old',new Blob(['old'])],['used',new Blob(['used'])]]),doc),engine=await f.open();try{await engine.persist();const root=engine.getSnapshot(),first=await engine.inventoryLocalFiles({limit:1});assert.equal(engine.getSnapshot(),root);assert.equal(first.files[0].id,'old');assert.equal(first.files[0].ownership,'unregistered');assert.equal(first.nextAfter,'old');const second=await engine.inventoryLocalFiles({after:first.nextAfter,limit:1});assert.equal(second.files[0].id,'used');assert.ok(second.files[0].pins.some(owner=>owner.startsWith('element:')));assert.equal(second.nextAfter,undefined);assert.equal(f.requests.length,0);
|
||||
const marked=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(marked.markedIds,['old']);assert.deepEqual(marked.purgedIds,[]);assert.equal(f.image().formatVersion,5);f.advance(100);const result=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(result.purgedIds,['old']);assert.equal(f.files.has('old'),false);assert.equal(f.files.has('used'),true);assert.equal(engine.getSnapshot().document,root.document);assert.equal(engine.getSnapshot().history,root.history);assert.equal(f.requests.length,0);decodeRecovery(f.image(),registry);
|
||||
}finally{await engine.close();}
|
||||
});
|
||||
test('retired orphan identity and v5 recovery survive reopening and reject new references or silent downgrade',async()=>{
|
||||
const f=fixture(new Map([['old',new Blob(['old'])]]));let engine=await f.open();try{await engine.collectOrphanFiles({retentionMs:0});const file=engine.getSnapshot().runtime.orphanFiles.get('old');assert.ok(file.retirement.purgedAt!==undefined);for(const formatVersion of [1,2,3,4])assert.throws(()=>decodeRecovery({...f.image(),formatVersion},registry),/unsupported_recovery_version/);await engine.close();engine=await f.open();await assert.rejects(engine.displayResource('old'),/local_file_retired/);assert.equal(engine.editor.dispatch({type:'element.create',element:node('bad',{assetId:'old'})}).code,'local_file_retired');assert.equal(engine.editor.updateRuntime(previous=>({...previous,orphanFiles:new Map()})).code,'local_file_retired');assert.equal(engine.editor.dispatch({type:'element.data',target:{id:'A',incarnation:1},patch:{text:'still editable'}}).status,'applied');await engine.settle();assert.equal(f.files.has('old'),false);
|
||||
}finally{await engine.close();}
|
||||
});
|
||||
test('changed orphan bytes reset the retention interval even when size and MIME stay equal',async()=>{
|
||||
const f=fixture(new Map([['old',new Blob(['aa'])]])),engine=await f.open();try{await engine.collectOrphanFiles({retentionMs:100});const digest=engine.getSnapshot().runtime.orphanFiles.get('old').digest;f.advance(100);f.files.set('old',new Blob(['bb']));const changed=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(changed.changedIds,['old']);assert.deepEqual(changed.purgedIds,[]);const record=engine.getSnapshot().runtime.orphanFiles.get('old');assert.notEqual(record.digest,digest);assert.equal(record.retirement.unreferencedAt,1100);f.advance(100);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:100})).purgedIds,['old']);}finally{await engine.close();}
|
||||
});
|
||||
test('display borrowing and explicit owners reset soft markers before collection',async()=>{
|
||||
const f=fixture(new Map([['old',new Blob(['data'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile;try{await engine.collectOrphanFiles({retentionMs:100});f.advance(100);f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const opening=engine.displayResource('old');await entered.promise;assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:0})).purgedIds,[]);gate.resolve();const handle=await opening;handle.release();await engine.collectOrphanFiles({retentionMs:100});engine.editor.updateRuntime(previous=>({...previous,resourcePins:new Map([['export',['old']]])}));assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:0})).purgedIds,[]);}finally{gate.resolve();await engine.close();}
|
||||
});
|
||||
test('hash budgets preserve pagination progress and a changed Root invalidates read-only verification',async()=>{
|
||||
const f=fixture(new Map([['a',new Blob(['a'])],['b',new Blob(['bb'])],['c',new Blob(['c'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile;try{const first=await engine.collectOrphanFiles({retentionMs:100,maxHashBytes:1});assert.deepEqual(first.markedIds,['a']);assert.equal(first.nextAfter,'a');assert.equal(first.hashBudgetExhausted,true);assert.equal(first.scanFinished,false);const second=await engine.collectOrphanFiles({retentionMs:100,after:first.nextAfter,maxHashBytes:3});assert.deepEqual(second.markedIds,['b','c']);assert.equal(second.scanFinished,true);
|
||||
f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const collecting=engine.collectOrphanFiles({retentionMs:0});await entered.promise;assert.equal(engine.editor.dispatch({type:'element.data',target:{id:'A',incarnation:1},patch:{text:'changed while hashing'}}).status,'applied');gate.resolve();await assert.rejects(collecting,/stale_file_inventory/);assert.equal(f.files.size,3);
|
||||
}finally{gate.resolve();await engine.close();}
|
||||
});
|
||||
test('cancelling verification never publishes late byte results and failures retain the original purge cut',async()=>{
|
||||
const f=fixture(new Map([['old',new Blob(['old'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile,cancel=new AbortController();let fail=false;const commit=f.storage.commit;try{f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const task=engine.collectOrphanFiles({retentionMs:0,signal:cancel.signal});await entered.promise;cancel.abort();await assert.rejects(task,/file_inventory_aborted/);gate.resolve();await Promise.resolve();assert.equal(engine.getSnapshot().runtime.orphanFiles?.size??0,0);f.storage.readFile=read;f.storage.commit=async(...args)=>{if(fail&&args[1].deleteFiles?.length)throw Error('disk failure');return commit(...args);};fail=true;await assert.rejects(engine.collectOrphanFiles({retentionMs:0}),/disk failure/);assert.ok(f.files.has('old'));await assert.rejects(engine.displayResource('old'),/resource_collection_in_progress/);fail=false;await engine.persist();assert.equal(f.files.has('old'),false);assert.ok(decodeRecovery(f.image(),registry).runtime.orphanFiles.get('old').retirement.purgedAt!==undefined);
|
||||
}finally{fail=false;gate.resolve();await engine.close();}
|
||||
});
|
||||
test('opaque producers block orphan collection even when the known local resource map is empty',async()=>{
|
||||
const f=fixture(),engine=await f.open();try{const root=engine.getSnapshot(),coverage=runtime=>orphanCoverage({...root,runtime},registry,new Map(),new Map());assert.equal(coverage({...root.runtime,runs:new Map([['run',{id:'run'}]])}).complete,false);assert.ok(coverage({...root.runtime,recoveries:new Map([['audit',{}]])}).blockers.includes('recovery_audits'));assert.ok(coverage({...root.runtime,imports:new Map([['job',{id:'job',status:'paused',source:{fileId:'source'}}]])}).blockers.includes('unfinished_imports'));assert.ok(orphanCoverage(root,registry,new Map(),new Map([['copy-admission',[]]])).blockers.includes('copy_admission'));
|
||||
}finally{await engine.close();}
|
||||
});
|
||||
|
||||
test('a cancelled copy admission resets orphan retention before any ImportJob is published',async()=>{
|
||||
const f=fixture(new Map([['old',new Blob(['old'])]])),engine=await f.open(),cancel=new AbortController();try{await engine.collectOrphanFiles({retentionMs:100});f.advance(100);cancel.abort();await assert.rejects(engine.imports.duplicate([{id:'A',incarnation:1}],{signal:cancel.signal}),/copy_aborted/);assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:100})).purgedIds,[]);assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement.unreferencedAt,1100);}finally{await engine.close();}
|
||||
});
|
||||
@@ -24,5 +24,5 @@ test('cached subtree height cannot bypass the depth bound and omitted object fie
|
||||
const cycle={};cycle.self=cycle;assert.throws(()=>encodeRecoveryValue(cycle),/invalid_recovery_payload/);assert.throws(()=>encodeRecoveryValue({big:1n}));
|
||||
});
|
||||
test('only factory-owned images are recognized; immutable-looking impostors retain defensive-copy handling',()=>{
|
||||
const mutable={title:'before'},image=sealRecoveryImage(2,3,2,{mutable});assert.equal(isSealedRecoveryImage(image),true);assert.equal(isSealedRecoveryImage(structuredClone(image)),false);assert.equal(isSealedRecoveryImage(Object.freeze({...image})),false);mutable.title='after';assert.equal(image.payload.mutable.title,'before');assert.ok(Object.isFrozen(image.payload.mutable));assert.throws(()=>sealRecoveryImage(2,1,2,{}),/corrupt_recovery_image/);assert.equal(isSealedRecoveryImage(sealRecoveryImage(3,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(4,1,1,{})),true);assert.throws(()=>sealRecoveryImage(5,1,1,{}),/corrupt_recovery_image/);
|
||||
const mutable={title:'before'},image=sealRecoveryImage(2,3,2,{mutable});assert.equal(isSealedRecoveryImage(image),true);assert.equal(isSealedRecoveryImage(structuredClone(image)),false);assert.equal(isSealedRecoveryImage(Object.freeze({...image})),false);mutable.title='after';assert.equal(image.payload.mutable.title,'before');assert.ok(Object.isFrozen(image.payload.mutable));assert.throws(()=>sealRecoveryImage(2,1,2,{}),/corrupt_recovery_image/);assert.equal(isSealedRecoveryImage(sealRecoveryImage(3,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(4,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(5,1,1,{})),true);assert.throws(()=>sealRecoveryImage(6,1,1,{}),/corrupt_recovery_image/);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user