From 3ccb8dbc5c2aa896b9cad8d6db72421b35611d43 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 8 Nov 2026 23:21:27 +0800 Subject: [PATCH] Inventory scoped files and atomically collect verified legacy orphans --- src/lib/canvas-runtime/contracts.ts | 2 + src/lib/canvas-runtime/engine.ts | 86 ++++++++++++++++++--- src/lib/canvas-runtime/orphan-files.ts | 43 +++++++++++ src/lib/canvas-runtime/recovery-encoding.ts | 4 +- src/lib/canvas-runtime/recovery.ts | 14 ++-- src/lib/canvas-runtime/storage.ts | 38 +++++++-- tests/headless/browser-storage-cases.mjs | 20 ++++- tests/headless/orphan-files.test.mjs | 46 +++++++++++ tests/headless/recovery-encoding.test.mjs | 2 +- 9 files changed, 232 insertions(+), 23 deletions(-) create mode 100644 src/lib/canvas-runtime/orphan-files.ts create mode 100644 tests/headless/orphan-files.test.mjs diff --git a/src/lib/canvas-runtime/contracts.ts b/src/lib/canvas-runtime/contracts.ts index 02cf898..558ca2a 100644 --- a/src/lib/canvas-runtime/contracts.ts +++ b/src/lib/canvas-runtime/contracts.ts @@ -1,3 +1,4 @@ +import type {OrphanFile} from './orphan-files.ts'; import type { ImportJob } from './imports.ts'; import type { Document, Id, Json, OperationIntent, Snapshot } from '../canvas-core/types.ts'; import type { LedgerSnapshot, FrozenRequest } from './ledger.ts'; @@ -7,6 +8,7 @@ import type { ResourceRecord } from './resources.ts'; import type { RecoveryAudit } from './conflicts.ts'; export type RuntimeState = { + readonly orphanFiles?:ReadonlyMap; readonly imports?: ReadonlyMap; readonly ledger: LedgerSnapshot; readonly runs: ReadonlyMap; diff --git a/src/lib/canvas-runtime/engine.ts b/src/lib/canvas-runtime/engine.ts index 30b7ed8..70c14d9 100644 --- a/src/lib/canvas-runtime/engine.ts +++ b/src/lib/canvas-runtime/engine.ts @@ -1,3 +1,4 @@ +import {orphanCoverage,prepareOrphanFiles,abortableFileRead,type OrphanFile} from './orphan-files.ts'; import {importFiles,terminalImport,markImportFiles,prepareImportFileRetention,type FileRetirement} from './import-file-retention.ts'; import {resourceRetentionPins,prepareResourceRetention} from './resource-retention.ts'; import { encodeIndexedCopySnapshot, openIndexedCopySnapshot, type IndexedCopySnapshot } from './copy-snapshot-index.ts'; @@ -66,6 +67,9 @@ export async function openProjectEngine(options: EngineOptions) { const displayPins=new Map(),collectionReservations=new Set(),durablyPurged=new Set([...(recovered?.runtime.resources.values()??[])].filter(resource=>resource.retirement?.purgedAt!==undefined).map(resource=>resource.id)); type CollectionReport={readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly complete:boolean;readonly checkedRootRevision:number;readonly rootRevision:number;readonly durableRootRevision:number}; let sourceCollection:Promise|undefined; + let orphanCollection:Promise<{readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly changedIds:readonly Id[];readonly complete:boolean;readonly blockers:readonly string[];readonly checkedRootRevision:number;readonly scanned:number;readonly nextAfter?:Id;readonly hashBudgetExhausted:boolean;readonly scanFinished:boolean;readonly durableRootRevision:number}>|undefined; + let orphanController:AbortController|undefined; + for(const [id,file]of recovered?.runtime.orphanFiles??[])if(file.retirement?.purgedAt!==undefined)durablyPurged.add(id); for(const [id,entry]of importFiles(recovered?.runtime.imports))if(entry.file.retirement?.purgedAt!==undefined)durablyPurged.add(id); let collection:Promise|undefined; const fileWrites = new Map(), digestTasks = new Map>(); @@ -85,7 +89,9 @@ export async function openProjectEngine(options: EngineOptions) { onListenerError: report, prepareRuntime:(previous,next)=>{ const runtime=prepareResourceRetention(next,options.registry,displayPins,collectionReservations,admissionPins(next)); - return prepareImportFileRetention(previous.runtime,runtime,()=>resourceRetentionPins({...next,runtime},options.registry,displayPins,admissionPins({...next,runtime})),collectionReservations,previous.document!==next.document||previous.history!==next.history||previous.runtime.ledger!==runtime.ledger||previous.runtime.resourcePins!==runtime.resourcePins); + const changed=previous.document!==next.document||previous.history!==next.history||previous.runtime.ledger!==runtime.ledger||previous.runtime.resourcePins!==runtime.resourcePins; + const imports=prepareImportFileRetention(previous.runtime,runtime,()=>resourceRetentionPins({...next,runtime},options.registry,displayPins,admissionPins({...next,runtime})),collectionReservations,changed); + return prepareOrphanFiles(previous.runtime,imports,()=>orphanCoverage({...next,runtime:imports},options.registry,displayPins,admissionPins({...next,runtime:imports})),collectionReservations,changed); }, prepareIntent: intent => applying && intent.origin==='user' ? {...intent,operationType:'run.apply',action:applying} : intent, nextOperationSequence(runtime, fallback) { let next = fallback; for (const record of runtime.ledger.operations.values()) next = Math.max(next, record.intent.sequence + 1); return next + staged.length; }, @@ -126,7 +132,7 @@ export async function openProjectEngine(options: EngineOptions) { const [revision, snapshot] = pending.entries().next().value!; const sourceFiles=importFiles(snapshot.runtime.imports); const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||sourceFiles.has(id)).map(([id, bytes]) => ({ id, bytes })); - const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)]; + const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id),...[...(snapshot.runtime.orphanFiles??[])].filter(([id,file])=>file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)]; const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles }); if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt'); durableRootRevision = revision; pending.delete(revision); @@ -218,6 +224,64 @@ export async function openProjectEngine(options: EngineOptions) { if(closing||closed)throw fault('engine_closing');return Object.freeze({bytes,release}); }catch(error){release();if(closing||closed)throw fault('engine_closing');throw error;} } + async function scanLocalFiles(root:EngineSnapshot,after:Id|undefined,limit:number){ + if(!options.storage.scanFiles)throw fault('file_inventory_unavailable'); + const page=await options.storage.scanFiles(leader,{expectedRootRevision:root.rootRevision,after,limit}); + if(closing||closed)throw fault('engine_closing');if(editor.getSnapshot()!==root)throw fault('stale_file_inventory'); + let previous=after; + if(page.rootRevision!==root.rootRevision||!Array.isArray(page.files)||page.files.length>limit)throw fault('invalid_file_inventory'); + for(const file of page.files){if(!file.id||typeof file.id!=='string'||previous!==undefined&&file.id<=previous||!Number.isSafeInteger(file.byteLength)||file.byteLength<0||typeof file.mimeType!=='string')throw fault('invalid_file_inventory');previous=file.id;} + if(page.nextAfter!==undefined&&(!page.files.length||page.nextAfter!==previous))throw fault('invalid_file_inventory');return page; + } + async function inventoryLocalFiles(settings:{after?:Id;limit?:number}={}){ + assertWritable();const limit=settings.limit??128;if(!Number.isSafeInteger(limit)||limit<1||limit>128||settings.after!==undefined&&(typeof settings.after!=='string'||!settings.after))throw fault('invalid_file_inventory'); + await persist();assertWritable();const root=editor.getSnapshot(),page=await scanLocalFiles(root,settings.after,limit),sources=importFiles(root.runtime.imports),pins=orphanCoverage(root,options.registry,displayPins,admissionPins(root)); + return Object.freeze({rootRevision:root.rootRevision,complete:pins.complete,blockers:pins.blockers,nextAfter:page.nextAfter,files:Object.freeze(page.files.map(file=>{const resource=root.runtime.resources.get(file.id),source=sources.get(file.id),orphan=root.runtime.orphanFiles?.get(file.id);return Object.freeze({...file,ownership:resource?'resource' as const:source?'import-source' as const:orphan?'registered-orphan' as const:'unregistered' as const,...(source?{jobId:source.jobId,current:source.current}:{}),retired:(resource?.retirement??source?.file.retirement??orphan?.retirement)?.purgedAt!==undefined,pins:Object.freeze([...(pins.pins.get(file.id)??[])])});}))}); + } + function collectOrphanFiles(settings:{retentionMs:number;maxFiles?:number;after?:Id;pageSize?:number;maxHashBytes?:number;signal?:AbortSignal}){ + assertWritable();if(options.storage.fileDeletion!=='atomic-v1'||!options.storage.scanFiles)throw fault('file_inventory_unavailable'); + const retentionMs=settings.retentionMs,maxFiles=settings.maxFiles??64,pageSize=settings.pageSize??128,maxHashBytes=settings.maxHashBytes??64*1024*1024; + if(!Number.isSafeInteger(retentionMs)||retentionMs<0||![maxFiles,pageSize].every(value=>Number.isSafeInteger(value)&&value>=1&&value<=128)||!Number.isSafeInteger(maxHashBytes)||maxHashBytes<1||settings.after!==undefined&&(typeof settings.after!=='string'||!settings.after))throw fault('invalid_file_inventory'); + if(orphanCollection)return orphanCollection; + const controller=new AbortController(),signal=settings.signal?AbortSignal.any([controller.signal,settings.signal]):controller.signal;orphanController=controller; + const check=()=>{assertWritable();if(signal.aborted)throw fault('file_inventory_aborted');}; + const task=(async()=>{ + await persist();check();const now=runClock.now();if(!Number.isSafeInteger(now)||now<0)throw fault('invalid_resource_collection_clock'); + let root=editor.getSnapshot(),pins=orphanCoverage(root,options.registry,displayPins,admissionPins(root)),scanned=0,nextAfter=settings.after,hashBudgetExhausted=false,scanFinished=false; + const markedIds:Id[]=[],purgedIds:Id[]=[],changedIds:Id[]=[],verified=new Map(); + const result=()=>Object.freeze({markedIds:Object.freeze(markedIds),purgedIds:Object.freeze(purgedIds),changedIds:Object.freeze(changedIds),complete:pins.complete,blockers:pins.blockers,checkedRootRevision:root.rootRevision,scanned,nextAfter,hashBudgetExhausted,scanFinished,durableRootRevision}); + if(!pins.complete)return result(); + const page=await abortableFileRead(scanLocalFiles(root,settings.after,pageSize),signal),sources=importFiles(root.runtime.imports);let hashed=0,finished=true; + for(const file of page.files){ + check();const registered=root.runtime.orphanFiles?.get(file.id),candidate=!root.runtime.resources.has(file.id)&&!sources.has(file.id)&®istered?.retirement?.purgedAt===undefined&&!pins.pins.has(file.id)&&!fileWrites.has(file.id); + if(candidate){ + if(verified.size>=maxFiles){finished=false;break;}if(file.byteLength>maxHashBytes-hashed){hashBudgetExhausted=true;finished=false;break;} + const bytes=await abortableFileRead(options.storage.readFile(leader,file.id),signal);check();if(editor.getSnapshot()!==root)throw fault('stale_file_inventory'); + if(!(bytes instanceof Blob)||bytes.size!==file.byteLength||bytes.type!==file.mimeType)throw fault('file_inventory_changed',[file.id]); + const digest=await abortableFileRead(sourceDigest(bytes,signal),signal);check();if(editor.getSnapshot()!==root)throw fault('stale_file_inventory');hashed+=bytes.size; + verified.set(file.id,freeze({version:1 as const,id:file.id,byteLength:file.byteLength,mimeType:file.mimeType,digest})); + } + scanned++;nextAfter=file.id; + } + if(finished)nextAfter=page.nextAfter;scanFinished=finished&&page.nextAfter===undefined; + pins=orphanCoverage(root,options.registry,displayPins,admissionPins(root));if(!pins.complete)return result(); + const updates=new Map(); + for(const [id,file]of verified){if(pins.pins.has(id))continue;const old=root.runtime.orphanFiles?.get(id),changed=!!old&&(old.digest!==file.digest||old.byteLength!==file.byteLength||old.mimeType!==file.mimeType); + if(!old?.retirement||changed){updates.set(id,freeze({...file,retirement:{version:1 as const,unreferencedAt:now}}));markedIds.push(id);if(changed)changedIds.push(id);} + } + if(updates.size){const update=editor.updateRuntime(previous=>{const files=new Map(previous.orphanFiles);for(const [id,file]of updates)files.set(id,file);return Object.freeze({...previous,orphanFiles:readMap(files)});},root.rootRevision);if(update.status==='rejected')throw fault(update.code,update.targets);await persist();check();} + root=editor.getSnapshot();pins=orphanCoverage(root,options.registry,displayPins,admissionPins(root));if(!pins.complete)return result(); + const eligible=[...verified].filter(([id,proof])=>{const file=root.runtime.orphanFiles?.get(id);return file?.retirement&&file.retirement.purgedAt===undefined&&now-file.retirement.unreferencedAt>=retentionMs&&file.digest===proof.digest&&file.byteLength===proof.byteLength&&file.mimeType===proof.mimeType&&!pins.pins.has(id)&&!root.runtime.resources.has(id)&&!importFiles(root.runtime.imports).has(id)&&!fileWrites.has(id);}); + if(eligible.length){ + for(const [id]of eligible)collectionReservations.add(id); + const update=editor.updateRuntime(previous=>{const files=new Map(previous.orphanFiles);for(const [id]of eligible){const file=files.get(id)!;files.set(id,freeze({...file,retirement:{...file.retirement!,purgedAt:now}}));}return Object.freeze({...previous,orphanFiles:readMap(files)});},root.rootRevision); + if(update.status==='rejected'){for(const [id]of eligible)collectionReservations.delete(id);throw fault(update.code,update.targets);} + await persist();purgedIds.push(...eligible.map(([id])=>id)); + } + return result(); + })(); + orphanCollection=task.finally(()=>{orphanCollection=undefined;if(orphanController===controller)orphanController=undefined;});return orphanCollection; + } function mutate(reduce: (ledger: ReturnType) => void) { const result = editor.updateRuntime(previous => { const ledger = createLedger(previous.ledger); reduce(ledger); return Object.freeze({ ...previous, ledger: ledger.getSnapshot() }); }); if (result.status === 'rejected') throw fault(result.code, result.targets); @@ -500,7 +564,7 @@ export async function openProjectEngine(options: EngineOptions) { openSource:openImportSource, startNdjson(bytes:Blob,targetCanvasId:Id):Id{ assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);const canvas=editor.getSnapshot().document.canvases.get(targetCanvasId);if(!canvas)throw fault('canvas_not_loaded',[targetCanvasId]);if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source'); - const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict'); + const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||current.orphanFiles?.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict'); const entry=canvas.parent?editor.getSnapshot().document.elements.get(canvas.parent.entryElementId)!:undefined; const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'native-ndjson',name:((bytes as Blob&{name?:string}).name||'画布数据').slice(0,255),fileId,byteLength:bytes.size,targetCanvasId,...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'running' as const}); fileWrites.set(fileId,bytes); @@ -510,8 +574,8 @@ export async function openProjectEngine(options: EngineOptions) { async duplicate(selection:readonly Target[],settings:{readonly worldOffset?:Point;readonly signal?:AbortSignal;readonly viewCanvasId?:Id}={}):Promise{ assertWritable();if(collectionReservations.size)throw fault('resource_collection_in_progress',[...collectionReservations]); const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets); - if([...editor.getSnapshot().runtime.resources.values()].some(resource=>resource.retirement&&resource.retirement.purgedAt===undefined)){ - const reset=editor.updateRuntime(previous=>{const resources=new Map(previous.resources);for(const [id,resource]of resources)if(resource.retirement&&resource.retirement.purgedAt===undefined){const {retirement:_,...kept}=resource;resources.set(id,freeze(kept));}return Object.freeze({...previous,resources:readMap(resources)});});if(reset.status==='rejected')throw fault(reset.code,reset.targets); + if([...editor.getSnapshot().runtime.resources.values(),...(editor.getSnapshot().runtime.orphanFiles?.values()??[])].some(file=>file.retirement&&file.retirement.purgedAt===undefined)){ + const reset=editor.updateRuntime(previous=>{const resources=new Map(previous.resources);for(const [id,resource]of resources)if(resource.retirement&&resource.retirement.purgedAt===undefined){const {retirement:_,...kept}=resource;resources.set(id,freeze(kept));}const files=new Map(previous.orphanFiles);for(const [id,file]of files)if(file.retirement&&file.retirement.purgedAt===undefined){const {retirement:_,...kept}=file;files.set(id,freeze(kept));}return Object.freeze({...previous,resources:readMap(resources),...(previous.orphanFiles?{orphanFiles:readMap(files)}:{})});});if(reset.status==='rejected')throw fault(reset.code,reset.targets); } const captured=editor.getSnapshot(),target=settings.viewCanvasId??(selection[0]?captured.document.elements.get(selection[0].id)?.ownerCanvasId:undefined),signal=settings.signal,controller=new AbortController(),abort=()=>controller.abort(); const targets=freeze(clone(selection)),offset=freeze(clone(settings.worldOffset??{x:32,y:32})); @@ -522,7 +586,7 @@ export async function openProjectEngine(options: EngineOptions) { const bytes=await encodeIndexedCopySnapshot(captured.document,controller.signal);assertWritable();if(controller.signal.aborted)throw fault('copy_aborted'); const canvas=captured.document.canvases.get(target);if(!canvas)throw fault('canvas_not_loaded',[target]); const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime; - if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict'); + if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||current.orphanFiles?.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict'); const entry=canvas.parent?captured.document.elements.get(canvas.parent.entryElementId)!:undefined; const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'duplicate-snapshot',snapshotVersion:2,name:'复制选中内容',fileId,byteLength:bytes.size,targetCanvasId:target,copy:{sourceRevision:captured.revision,roots:[]},...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'preparing' as const,preparation:{version:4 as const,registrySignature:copyRegistrySignature(captured.document,options.registry),selection:targets,worldOffset:offset,allocationIds:[],steps:0}}); fileWrites.set(fileId,bytes);const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});}); @@ -537,7 +601,7 @@ export async function openProjectEngine(options: EngineOptions) { scheduler.start(); prepareDigests(); runTracker.notify([...initial.runs.keys()]); for(const job of initial.imports?.values()??[])if(['running','preparing'].includes(job.status))void loadImport(job.id); return Object.freeze({ - editor, sessions, imports, collectResources,collectImportSources, + editor, sessions, imports, collectResources,collectImportSources,inventoryLocalFiles,collectOrphanFiles, getResourcePins:()=>resourceRetentionPins(editor.getSnapshot(),options.registry,displayPins,admissionPins(editor.getSnapshot())), /** Read/preview under a scheduling hold. No request is rewritten or declared resolved here. */ async prepareRecovery(): Promise { @@ -677,7 +741,7 @@ export async function openProjectEngine(options: EngineOptions) { if (!(file.bytes instanceof Blob) || !file.bytes.size || file.bytes.size > maxResourceBytes || !file.bytes.type.startsWith('image/') || typeof file.name !== 'string' || !file.name) throw fault('invalid_staged_file'); return freeze({ id: createId(), operationId: createId(), name: file.name, mimeType: file.bytes.type, byteLength: file.bytes.size }); }); - if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || fileWrites.has(resource.id) || importFiles(editor.getSnapshot().runtime.imports).has(resource.id))) throw fault('resource_identity_conflict'); + if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || editor.getSnapshot().runtime.orphanFiles?.has(resource.id) || fileWrites.has(resource.id) || importFiles(editor.getSnapshot().runtime.imports).has(resource.id))) throw fault('resource_identity_conflict'); staged = resources; resources.forEach((resource, index) => fileWrites.set(resource.id, files[index].bytes)); try { const handles = freeze(resources.map(resource => ({ id: resource.id, src: options.assets!.sourceURL(resource.id) }))); @@ -695,6 +759,8 @@ export async function openProjectEngine(options: EngineOptions) { displays.add(release); try{ const retained=editor.getSnapshot().runtime.resources.get(id); + const orphan=editor.getSnapshot().runtime.orphanFiles?.get(id);if(orphan?.retirement?.purgedAt!==undefined)throw fault('local_file_retired',[id]); + if(orphan?.retirement){const result=editor.updateRuntime(previous=>Object.freeze({...previous}));if(result.status==='rejected')throw fault(result.code,result.targets);} if(retained?.retirement&&retained.retirement.purgedAt===undefined){ const result=editor.updateRuntime(previous=>{const resources=new Map(previous.resources),{retirement:_,...kept}=retained;resources.set(id,freeze(kept));return Object.freeze({...previous,resources:readMap(resources)});}); if(result.status==='rejected')throw fault(result.code,result.targets); @@ -706,6 +772,8 @@ export async function openProjectEngine(options: EngineOptions) { } const bytes = fileWrites.get(id) ?? await options.storage.readFile(leader, id); if (closing || closed) throw fault('engine_closing'); + if(orphan&&(!bytes||bytes.size!==orphan.byteLength||bytes.type!==orphan.mimeType||await sourceDigest(bytes)!==orphan.digest))throw fault('file_inventory_changed',[id]); + if(closing||closed)throw fault('engine_closing'); const resource=editor.getSnapshot().runtime.resources.get(id); if(!bytes&&resource?.retirement?.purgedAt!==undefined&&!resource.stored)throw fault('resource_bytes_unavailable',[id]); if (!bytes && !options.assets) throw fault('resource_missing', [id]); @@ -720,7 +788,7 @@ export async function openProjectEngine(options: EngineOptions) { async settle() { await Promise.resolve();await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); }, async close() { if (closed) return; - closing = true;await Promise.allSettled([collection,sourceCollection].filter(Boolean));digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop(); + closing = true;orphanController?.abort();await Promise.allSettled([collection,sourceCollection,orphanCollection].filter(Boolean));digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop(); await Promise.all([...importRunners.values()].map(runner=>runner.close()));while(importLoads.size)await Promise.all([...importLoads.values()]);while(digestTasks.size)await Promise.allSettled([...digestTasks.values()]); recovery?.controller.abort();recovery?.release(); for (const release of [...displays]) release(); diff --git a/src/lib/canvas-runtime/orphan-files.ts b/src/lib/canvas-runtime/orphan-files.ts new file mode 100644 index 0000000..9cebe71 --- /dev/null +++ b/src/lib/canvas-runtime/orphan-files.ts @@ -0,0 +1,43 @@ +import type {Id,Registry} from '../canvas-core/types.ts'; +import {equal,fault,freeze,readMap} from '../canvas-core/values.ts'; +import type {EngineSnapshot,RuntimeState} from './contracts.ts'; +import {importFiles,terminalImport,validFileRetirement,type FileRetirement} from './import-file-retention.ts'; +import {resourceRetentionPins} from './resource-retention.ts'; + +export type OrphanFile={readonly version:1;readonly id:Id;readonly byteLength:number;readonly mimeType:string;readonly digest:string;readonly retirement?:FileRetirement}; +export function validateOrphanFile(value:unknown,id:Id):asserts value is OrphanFile { + const file=value as OrphanFile; + if(!file||file.version!==1||file.id!==id||!id||!Number.isSafeInteger(file.byteLength)||file.byteLength<0||typeof file.mimeType!=='string'||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest)||file.retirement!==undefined&&!validFileRetirement(file.retirement))throw fault('invalid_orphan_file'); +} +export function orphanCoverage(snapshot:EngineSnapshot,registry:Registry,displays:ReadonlyMap,temporary:ReadonlyMap) { + const pins=resourceRetentionPins(snapshot,registry,displays,temporary),blockers:string[]=[]; + if(!pins.complete)blockers.push('opaque_references'); + if(snapshot.runtime.runs.size)blockers.push('opaque_run_records'); + if(snapshot.runtime.recoveries.size)blockers.push('recovery_audits'); + if([...(snapshot.runtime.imports?.values()??[])].some(job=>!terminalImport(job.status)))blockers.push('unfinished_imports'); + if(temporary.has('copy-admission'))blockers.push('copy_admission'); + if([...snapshot.runtime.ledger.operations.values()].some(record=>record.phase!=='resolved'&&!['document.change','asset.upload'].includes(record.intent.operationType)))blockers.push('opaque_pending_actions'); + return Object.freeze({...pins,complete:blockers.length===0,blockers:Object.freeze(blockers)}); +} +/** Registered orphan identities survive repinning. Purged identities cannot be forgotten or reassigned. */ +export function prepareOrphanFiles(previous:RuntimeState,next:RuntimeState,coverage:()=>{complete:boolean;pins:ReadonlyMap},reserved:ReadonlySet,referencesChanged:boolean):RuntimeState { + if(!previous.orphanFiles?.size&&!next.orphanFiles?.size)return next; + const hasSoft=[...(next.orphanFiles?.values()??[])].some(file=>file.retirement&&file.retirement.purgedAt===undefined); + if(previous.orphanFiles===next.orphanFiles&&previous.resources===next.resources&&previous.imports===next.imports&&!reserved.size&&!referencesChanged&&!hasSoft)return next; + const sources=importFiles(next.imports); + for(const [id,file]of next.orphanFiles??[]){validateOrphanFile(file,id);if(next.resources.has(id)||sources.has(id))throw fault('local_file_identity_conflict',[id]);} + for(const [id,file]of previous.orphanFiles??[]){const after=next.orphanFiles?.get(id);if(!after||file.retirement?.purgedAt!==undefined&&!equal(file,after))throw fault('local_file_retired',[id]);} + const pins=coverage(),clear:Id[]=[]; + for(const [id,file]of next.orphanFiles??[]){ + if(reserved.has(id)&&(!pins.complete||pins.pins.has(id)))throw fault('resource_collection_in_progress',[id]); + if(file.retirement?.purgedAt!==undefined){if(pins.pins.has(id))throw fault('local_file_retired',[id]);} + else if(file.retirement&&(!pins.complete||pins.pins.has(id)))clear.push(id); + } + if(!clear.length)return next; + const files=new Map(next.orphanFiles);for(const id of clear){const {retirement:_,...file}=files.get(id)!;files.set(id,freeze(file));}return Object.freeze({...next,orphanFiles:readMap(files)}); +} + +/** Read-only I/O may finish late; cancellation detaches its result without publishing it. */ +export function abortableFileRead(task:Promise,signal:AbortSignal):Promise{ + return new Promise((resolve,reject)=>{const abort=()=>{signal.removeEventListener('abort',abort);reject(fault('file_inventory_aborted'));};signal.addEventListener('abort',abort,{once:true});if(signal.aborted)abort();task.then(value=>{signal.removeEventListener('abort',abort);if(!signal.aborted)resolve(value);},error=>{signal.removeEventListener('abort',abort);if(!signal.aborted)reject(error);});}); +} diff --git a/src/lib/canvas-runtime/recovery-encoding.ts b/src/lib/canvas-runtime/recovery-encoding.ts index 6fcd75c..42c30b1 100644 --- a/src/lib/canvas-runtime/recovery-encoding.ts +++ b/src/lib/canvas-runtime/recovery-encoding.ts @@ -52,8 +52,8 @@ export function encodeRecoveryValue(input:unknown):Json { } /** Only codec-owned, validated immutable images qualify for the storage copy fast path. */ -export function sealRecoveryImage(formatVersion:1|2|3|4,rootRevision:number,localRevision:number,payload:unknown):RecoveryImage { - if(![1,2,3,4].includes(formatVersion)||!Number.isSafeInteger(rootRevision)||rootRevision<0||!Number.isSafeInteger(localRevision)||localRevision<0||localRevision>rootRevision)throw fault('corrupt_recovery_image'); +export function sealRecoveryImage(formatVersion:1|2|3|4|5,rootRevision:number,localRevision:number,payload:unknown):RecoveryImage { + if(![1,2,3,4,5].includes(formatVersion)||!Number.isSafeInteger(rootRevision)||rootRevision<0||!Number.isSafeInteger(localRevision)||localRevision<0||localRevision>rootRevision)throw fault('corrupt_recovery_image'); const image=Object.freeze({formatVersion,rootRevision,localRevision,payload:encodeRecoveryValue(payload)});sealed.add(image);return image; } export function isSealedRecoveryImage(image:RecoveryImage):boolean{return sealed.has(image);} diff --git a/src/lib/canvas-runtime/recovery.ts b/src/lib/canvas-runtime/recovery.ts index a5bfaf7..3c19750 100644 --- a/src/lib/canvas-runtime/recovery.ts +++ b/src/lib/canvas-runtime/recovery.ts @@ -1,3 +1,4 @@ +import {validateOrphanFile,type OrphanFile} from './orphan-files.ts'; import {importFiles} from './import-file-retention.ts'; import {resourceRetentionPins} from './resource-retention.ts'; import { sealRecoveryImage } from './recovery-encoding.ts'; @@ -22,9 +23,9 @@ export function encodeRecovery(snapshot: EngineSnapshot): RecoveryImage { document: documentValue(snapshot.document), history: { undo: snapshot.history.undo, redo: snapshot.history.redo, entries: [...snapshot.history.entries] }, operations: [...snapshot.operations], - runtime: { imports:[...(snapshot.runtime.imports??[])], recoveries:[...(snapshot.runtime.recoveries??[])], runs: [...snapshot.runtime.runs], resources: [...snapshot.runtime.resources], resourcePins: [...snapshot.runtime.resourcePins], acknowledged: documentValue(snapshot.runtime.acknowledged), versions: [...snapshot.runtime.versions], receipts: [...snapshot.runtime.receipts], conflicts: [...snapshot.runtime.conflicts], ledger: { ...snapshot.runtime.ledger, operations: [...snapshot.runtime.ledger.operations], requests: [...snapshot.runtime.ledger.requests] } }, + runtime: { ...(snapshot.runtime.orphanFiles?.size?{orphanFiles:[...snapshot.runtime.orphanFiles]}:{}), imports:[...(snapshot.runtime.imports??[])], recoveries:[...(snapshot.runtime.recoveries??[])], runs: [...snapshot.runtime.runs], resources: [...snapshot.runtime.resources], resourcePins: [...snapshot.runtime.resourcePins], acknowledged: documentValue(snapshot.runtime.acknowledged), versions: [...snapshot.runtime.versions], receipts: [...snapshot.runtime.receipts], conflicts: [...snapshot.runtime.conflicts], ledger: { ...snapshot.runtime.ledger, operations: [...snapshot.runtime.ledger.operations], requests: [...snapshot.runtime.ledger.requests] } }, }; - return sealRecoveryImage([...(snapshot.runtime.imports?.values()??[])].some(job=>job.source?.retirement||job.source?.previousFiles?.length)?4:[...snapshot.runtime.resources.values()].some(resource=>resource.retirement)?3:snapshot.runtime.imports?.size ? 2 : 1,snapshot.rootRevision,snapshot.revision,payload); + return sealRecoveryImage(snapshot.runtime.orphanFiles?.size?5:[...(snapshot.runtime.imports?.values()??[])].some(job=>job.source?.retirement||job.source?.previousFiles?.length)?4:[...snapshot.runtime.resources.values()].some(resource=>resource.retirement)?3:snapshot.runtime.imports?.size ? 2 : 1,snapshot.rootRevision,snapshot.revision,payload); } type ObjectValue = Record; function object(value: unknown): ObjectValue { if (!value || Array.isArray(value) || typeof value !== 'object') throw fault('corrupt_recovery_image'); return value as ObjectValue; } @@ -87,7 +88,7 @@ function readHistory(history:ObjectValue,operations:ReadonlyMap image.rootRevision || !validJson(image.payload)) throw fault('corrupt_recovery_image'); // Own the parsed records; mutating a caller's RecoveryImage must not mutate live state. @@ -227,13 +228,16 @@ export function decodeRecovery(image: RecoveryImage, registry: Registry): CoreSn return freeze({...value as RunRecord,outputEvents:array(raw.outputEvents ?? []) as never}); }); const imports=table(runtime.imports??[],(value,id)=>{const job=value as ImportJob;validateImportJob(job);if(job.id!==id||job.projectId!==document.projectId)throw fault('corrupt_recovery_image');return freeze(job);}); - for(const [id,entry]of importFiles(imports)){if(resources.has(id))throw fault('import_file_identity_conflict',[id]);if((entry.file.retirement||imports.get(entry.jobId)?.source?.previousFiles?.length)&&image.formatVersion!==4)throw fault('unsupported_recovery_version');} + for(const [id,entry]of importFiles(imports)){if(resources.has(id))throw fault('import_file_identity_conflict',[id]);if((entry.file.retirement||imports.get(entry.jobId)?.source?.previousFiles?.length)&&image.formatVersion<4)throw fault('unsupported_recovery_version');} + const orphanFiles=table(runtime.orphanFiles??[],(file,id)=>{validateOrphanFile(file,id);return freeze(file);}); + if(orphanFiles.size&&image.formatVersion!==5)throw fault('unsupported_recovery_version');for(const id of orphanFiles.keys())if(resources.has(id)||importFiles(imports).has(id))throw fault('local_file_identity_conflict',[id]); const restoredLedger: LedgerSnapshot = Object.freeze({ generation, durableGeneration, operations: records, requests }); const snapshot=Object.freeze({ revision: image.localRevision, rootRevision: image.rootRevision, document, history:restoredHistory, operations, - runtime: Object.freeze({ imports, recoveries, runs, resources, resourcePins, ledger: restoredLedger, acknowledged, versions, receipts, conflicts: table(runtime.conflicts, string) }), + runtime: Object.freeze({ orphanFiles, imports, recoveries, runs, resources, resourcePins, ledger: restoredLedger, acknowledged, versions, receipts, conflicts: table(runtime.conflicts, string) }), }); if([...resources.values()].some(resource=>resource.retirement?.purgedAt!==undefined&&!resource.stored)){ const pins=resourceRetentionPins(snapshot,registry);for(const [id,resource]of resources)if(resource.retirement?.purgedAt!==undefined&&!resource.stored&&pins.pins.has(id))throw fault('corrupt_recovery_image'); } + if([...orphanFiles.values()].some(file=>file.retirement?.purgedAt!==undefined)){const pins=resourceRetentionPins(snapshot,registry);for(const [id,file]of orphanFiles)if(file.retirement?.purgedAt!==undefined&&pins.pins.has(id))throw fault('corrupt_recovery_image');} return snapshot; } diff --git a/src/lib/canvas-runtime/storage.ts b/src/lib/canvas-runtime/storage.ts index 2b3ac3f..ac3b531 100644 --- a/src/lib/canvas-runtime/storage.ts +++ b/src/lib/canvas-runtime/storage.ts @@ -4,10 +4,13 @@ import type { Json } from '../canvas-core/types.ts'; import { clone, equal, fault, freeze, validJson } from '../canvas-core/values.ts'; export type StorageScope = { readonly accountId: string; readonly workspaceId: string; readonly projectId: string }; -export type RecoveryImage = { readonly formatVersion: 1 | 2 | 3 | 4; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json }; +export type RecoveryImage = { readonly formatVersion: 1 | 2 | 3 | 4 | 5; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json }; export type DurableBatch = { readonly image: RecoveryImage; readonly deleteFiles?:readonly string[]; readonly files?: readonly { readonly id: string; readonly bytes: Blob }[] }; export type StorageLeader = { readonly scope: StorageScope; readonly fence: number; readonly active: boolean; release(): Promise }; +export type StoredFileInfo={readonly id:string;readonly byteLength:number;readonly mimeType:string}; +export type FileInventoryPage={readonly files:readonly StoredFileInfo[];readonly rootRevision:number;readonly nextAfter?:string}; export type StoragePort = { + scanFiles?(leader:StorageLeader,input:{readonly expectedRootRevision:number;readonly after?:string;readonly limit:number}):Promise; readonly fileDeletion?: 'atomic-v1'; claim(scope: StorageScope): Promise; recover(leader: StorageLeader): Promise; @@ -33,8 +36,14 @@ const keyFor = (scope: StorageScope) => { if (![scope.accountId, scope.workspaceId, scope.projectId].every(value => typeof value === 'string' && value.length > 0)) throw fault('invalid_storage_scope'); return JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]); }; +function purgedFileIds(image:RecoveryImage):Set{ + const result=new Set(),payload=image.payload as {runtime?:{resources?:[string,{retirement?:{purgedAt?:number}}][];orphanFiles?:[string,{retirement?:{purgedAt?:number}}][];imports?:[string,{source?:{fileId:string;retirement?:{purgedAt?:number};previousFiles?:{fileId:string;retirement?:{purgedAt?:number}}[]}}][]}}; + for(const rows of [payload?.runtime?.resources,payload?.runtime?.orphanFiles])if(Array.isArray(rows))for(const row of rows)if(Array.isArray(row)&&typeof row[0]==='string'&&row[1]?.retirement?.purgedAt!==undefined)result.add(row[0]); + if(Array.isArray(payload?.runtime?.imports))for(const [,job]of payload.runtime.imports){if(!job?.source)continue;for(const file of [job.source,...(job.source.previousFiles??[])])if(file.retirement?.purgedAt!==undefined)result.add(file.fileId);} + return result; +} function validateImage(image: RecoveryImage) { - if (![1,2,3,4].includes(image.formatVersion)) throw fault('unsupported_recovery_version'); + if (![1,2,3,4,5].includes(image.formatVersion)) throw fault('unsupported_recovery_version'); if (!Number.isSafeInteger(image.rootRevision) || image.rootRevision < 0 || !Number.isSafeInteger(image.localRevision) || image.localRevision < 0 || image.localRevision > image.rootRevision || !validJson(image.payload)) throw fault('corrupt_recovery_image'); } const requestValue = (request: IDBRequest) => new Promise((resolve, reject) => { request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); }); @@ -134,18 +143,20 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: // Detach mutable callers and descriptors before the first await, including queued commits. const image=isSealedRecoveryImage(batch.image)?batch.image:freeze(clone(batch.image)),files=(batch.files??[]).map(file=>({id:file.id,bytes:file.bytes})),deleteFiles=[...(batch.deleteFiles??[])];if(deleteFiles.length>128||new Set(deleteFiles).size!==deleteFiles.length||deleteFiles.some(id=>typeof id!=='string'||!id||files.some(file=>file.id===id)))throw fault('invalid_file_deletion');if(!isSealedRecoveryImage(image))validateImage(image); if(deleteFiles.length){ - type FileProof={fileId?:string;retirement?:{version:number;unreferencedAt:number;purgedAt?:number}}; - const payload=image.payload as {runtime?:{resources?:[string,FileProof][];imports?:[string,{status:string;source?:FileProof&{previousFiles?:FileProof[]}}][]}}; + type FileProof={id?:string;version?:number;byteLength?:number;mimeType?:string;digest?:string;fileId?:string;retirement?:{version:number;unreferencedAt:number;purgedAt?:number}}; + const payload=image.payload as {runtime?:{orphanFiles?:[string,FileProof][];resources?:[string,FileProof][];imports?:[string,{status:string;source?:FileProof&{previousFiles?:FileProof[]}}][]}}; const proofs=new Map(),owned=new Set(); const own=(id:string,file:FileProof,eligible:boolean)=>{if(typeof id!=='string'||!id||owned.has(id))throw fault('invalid_file_deletion');owned.add(id);if(eligible)proofs.set(id,file);}; for(const [id,resource]of payload?.runtime?.resources??[])own(id,resource,true); - if(image.formatVersion===4)for(const [,job]of payload?.runtime?.imports??[]){if(job.source){own(job.source.fileId!,job.source,['completed','partial','cancelled'].includes(job.status));for(const file of job.source.previousFiles??[])own(file.fileId!,file,true);}} + if(image.formatVersion>=4)for(const [,job]of payload?.runtime?.imports??[]){if(job.source){own(job.source.fileId!,job.source,['completed','partial','cancelled'].includes(job.status));for(const file of job.source.previousFiles??[])own(file.fileId!,file,true);}} + if(image.formatVersion===5)for(const [id,file]of payload?.runtime?.orphanFiles??[]){if(file.version!==1||file.id!==id||!Number.isSafeInteger(file.byteLength)||file.byteLength!<0||typeof file.mimeType!=='string'||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest))throw fault('invalid_file_deletion');own(id,file,true);} if(image.formatVersion<3||deleteFiles.some(id=>{const marker=proofs.get(id)?.retirement;return !marker||marker.version!==1||!Number.isSafeInteger(marker.unreferencedAt)||marker.unreferencedAt<0||!Number.isSafeInteger(marker.purgedAt)||marker.purgedAt!{}).then(async()=>{ let base=lastCuts.get(leader)??await recoverCut(leader); if(base?.image.rootRevision===image.rootRevision)base=await recoverCut(leader); + if(files.length&&base){const retired=purgedFileIds(base.image);if(files.some(file=>retired.has(file.id)))throw fault('resource_identity_conflict');} const digest=await imageDigest(image),stored:StoredImage={image,checksum:digest.checksum};let candidate:StoredCut=stored; if(base&&image.rootRevision===base.image.rootRevision+1){ const patches=diffJournal(base.image.payload,image.payload); @@ -171,6 +182,23 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: }); writeTails.set(leader,task);return task; }, + scanFiles(leader:StorageLeader,input:{expectedRootRevision:number;after?:string;limit:number}) { + const {expectedRootRevision,after,limit}=input; + if(!Number.isSafeInteger(expectedRootRevision)||expectedRootRevision<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))return Promise.reject(fault('invalid_file_inventory')); + return execute(leader,['files'],'readonly',async(transaction,metadata,key)=>{ + if(metadata.throughRevision!==expectedRootRevision)throw fault('stale_file_inventory'); + const rows=await new Promise((resolve,reject)=>{ + const files:StoredFileInfo[]=[],range=IDBKeyRange.bound(after===undefined?[key]:[key,after],[key,[]],after!==undefined,true),request=transaction.objectStore('files').openCursor(range); + request.onerror=()=>reject(request.error); + request.onsuccess=()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key,bytes=cursor.value; + if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]||!(bytes instanceof Blob)){reject(fault('corrupt_file_store'));return;} + files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue(); + }; + }); + const more=rows.length>limit,files=more?rows.slice(0,limit):rows; + return Object.freeze({files:Object.freeze(files),rootRevision:expectedRootRevision,...(more?{nextAfter:files.at(-1)!.id}:{})}); + }); + }, readFile(leader: StorageLeader, id: string) { return execute(leader, ['files'], 'readonly', (transaction, _metadata, key) => requestValue(transaction.objectStore('files').get([key, id]))); }, close() { closed = true; for (const handle of handles) void handle.release(); void opening.then(database => database.close()).catch(() => {}); }, }); diff --git a/tests/headless/browser-storage-cases.mjs b/tests/headless/browser-storage-cases.mjs index e2ce4d0..f9b9b99 100644 --- a/tests/headless/browser-storage-cases.mjs +++ b/tests/headless/browser-storage-cases.mjs @@ -317,6 +317,22 @@ async function importSourceCollectionCase(name,assert){ }finally{await engine?.close();storage.close();indexedDB.deleteDatabase(database);} } +async function orphanInventoryCase(name,assert){ + const database=`${name}-orphan-inventory`,storage=createBrowserStorage({name:database,checkpointEvery:3}),scope={accountId:'account',workspaceId:'workspace',projectId:'project'},registry=createRegistry(builtinComponents()),document=createDocument('project',[{id:'P',parent:null}],[{id:'image',incarnation:1,ownerCanvasId:'P',structure:'node',componentId:'image',dataVersion:1,data:{title:'image',text:'',assetId:'z-pinned'},geometry:{x:0,y:0,width:100,height:100,rotation:0}}]);let engine,owner,other,now=1000; + const options={scope,registry,document,versions:new Map([['P',1]]),storage:{...storage,claim:async scope=>{owner=await storage.claim(scope);return owner;}},persistence:{compile(){throw Error('unexpected write');},async submit(){throw Error('unexpected write');},async query(){throw Error('unexpected write');},canonical(){throw Error('unexpected write');}},clock:{now:()=>now,setTimer:(fn,ms)=>setTimeout(fn,ms),clearTimer:id=>clearTimeout(id)}}; + try{ + engine=await openProjectEngine(options);await engine.persist();await engine.close();let legacy=await storage.claim(scope),cut=await storage.recover(legacy);await storage.commit(legacy,{image:{...cut,rootRevision:cut.rootRevision+1},files:[{id:'a-orphan',bytes:new Blob(['orphan bytes'])},{id:'z-pinned',bytes:new Blob(['pinned bytes'])}]});await legacy.release(); + other=await storage.claim({...scope,projectId:'other'});await storage.commit(other,{image:{formatVersion:1,rootRevision:0,localRevision:0,payload:{}},files:[{id:'a-foreign',bytes:new Blob(['other project'])}]});engine=await openProjectEngine(options); + const before=engine.getSnapshot(),first=await engine.inventoryLocalFiles({limit:1});assert(first.files.length===1&&first.files[0].id==='a-orphan'&&first.nextAfter==='a-orphan','inventory paging/scope prefix is wrong');assert(engine.getSnapshot()===before,'read-only inventory mutated the Root');const last=await engine.inventoryLocalFiles({after:first.nextAfter,limit:1});assert(last.files[0].id==='z-pinned'&&last.files[0].pins.length>0&&last.nextAfter===undefined,'referenced unregistered bytes lost their pin'); + await engine.collectOrphanFiles({retentionMs:100});now+=100;const nativeDelete=IDBObjectStore.prototype.delete;let injected=false; + IDBObjectStore.prototype.delete=function(key){const request=nativeDelete.call(this,key);if(!injected&&this.name==='files'&&Array.isArray(key)&&key[1]==='a-orphan'){injected=true;throw Error('orphan_gc_abort');}return request;}; + let failure;try{await engine.collectOrphanFiles({retentionMs:100});}catch(error){failure=error.message;}finally{IDBObjectStore.prototype.delete=nativeDelete;} + assert(injected&&failure==='orphan_gc_abort','orphan GC did not reach post-delete rollback');assert(await(await storage.readFile(owner,'a-orphan')).text()==='orphan bytes','orphan byte rollback failed');cut=await storage.recover(owner);assert(cut.formatVersion===5&&cut.payload.runtime.orphanFiles[0][1].retirement.purgedAt===undefined,'aborted orphan GC advanced its cut');await engine.persist();assert(await storage.readFile(owner,'a-orphan')===undefined,'orphan deletion retry failed');assert(await(await storage.readFile(owner,'z-pinned')).text()==='pinned bytes','orphan GC deleted referenced bytes');assert(await(await storage.readFile(other,'a-foreign')).text()==='other project','orphan GC crossed its project scope'); + cut=await storage.recover(owner);assert(decodeRecovery(cut,registry).runtime.orphanFiles.get('a-orphan').retirement.purgedAt===1100,'orphan v5 record failed recovery');let stale;try{await storage.scanFiles(owner,{expectedRootRevision:before.rootRevision,limit:1});}catch(error){stale=error.code;}assert(stale==='stale_file_inventory','old inventory revision was accepted'); + await engine.close();legacy=await storage.claim(scope);let reuse;try{await storage.commit(legacy,{image:{...cut,rootRevision:cut.rootRevision+1},files:[{id:'a-orphan',bytes:new Blob(['reused identity'])}]});}catch(error){reuse=error.code;}assert(reuse==='resource_identity_conflict','retired file identity was reused');await legacy.release();engine=await openProjectEngine(options);let retired;try{await engine.displayResource('a-orphan');}catch(error){retired=error.code;}assert(retired==='local_file_retired','retired orphan became a guessed remote asset');assert(engine.getSnapshot().document.elements.has('image'),'orphan GC mutated the document'); + }finally{await engine?.close();await other?.release();storage.close();indexedDB.deleteDatabase(database);} +} + async function resourceRecoveryCase(name, assert) { const scope = { accountId: crypto.randomUUID(), workspaceId: crypto.randomUUID(), projectId: crypto.randomUUID() }; const registry = createRegistry(builtinComponents()), document = createDocument(scope.projectId, [{ id: scope.projectId, parent: null }], []); @@ -607,7 +623,7 @@ export async function run() { await storage.commit(leader, { image: image(1) }); await rejects(() => storage.commit(leader, { image: { ...image(1), payload: null } }), 'storage_revision_conflict'); await rejects(() => storage.commit(leader, { image: image(3) }), 'storage_revision_gap'); - await rejects(() => storage.commit(leader, { image: { ...image(2), formatVersion: 5 } }), 'unsupported_recovery_version'); + await rejects(() => storage.commit(leader, { image: { ...image(2), formatVersion: 6 } }), 'unsupported_recovery_version'); results.push('same-cut retry is idempotent; conflicting, skipped and future-format cuts reject'); // The first put succeeds before a later duplicate resource aborts the whole transaction. @@ -652,6 +668,8 @@ export async function run() { results.push('atomic local file collection rolls back injected failure after IDB delete, captures descriptors, rejects missing markers and survives reopening'); await importSourceCollectionCase(name,assert); results.push('terminal import source GC atomically rolls back a post-delete fault, retries its v4 cut, preserves partial reports and reopens with undo/redo independent of source bytes'); + await orphanInventoryCase(name,assert); + results.push('scoped paged inventory finds legacy orphans without hiding references; orphan v5 deletion rolls back faults, retries, isolates projects and forbids byte identity reuse'); await resourceRecoveryCase(name, assert); results.push('staged file and image recover from real IndexedDB; lost upload ACK queries stable resource identity'); await digestWorkerCase(assert); diff --git a/tests/headless/orphan-files.test.mjs b/tests/headless/orphan-files.test.mjs new file mode 100644 index 0000000..4e4e546 --- /dev/null +++ b/tests/headless/orphan-files.test.mjs @@ -0,0 +1,46 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createDocument,createRegistry,builtinComponents,applyChanges} from '../../src/lib/canvas-core/index.ts'; +import {openProjectEngine} from '../../src/lib/canvas-runtime/engine.ts'; +import {decodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts'; +import {orphanCoverage} from '../../src/lib/canvas-runtime/orphan-files.ts'; +const registry=createRegistry(builtinComponents()),scope={accountId:'a',workspaceId:'w',projectId:'p'},node=(id,data={})=>({id,incarnation:1,structure:'node',ownerCanvasId:'C',componentId:'image',dataVersion:1,geometry:{x:0,y:0,width:100,height:100,rotation:0},data:{title:id,text:'',...data}}),original=createDocument('p',[{id:'C',parent:null}],[node('A')]); +const deferred=()=>{let resolve;const promise=new Promise(yes=>resolve=yes);return{promise,resolve};}; +function fixture(files=new Map(),document=original){ + let image,owner,next=0,server=document,version=1,now=1000;const batches=[],requests=[],receipts=new Map(); + const storage={fileDeletion:'atomic-v1',async claim(){assert.equal(owner,undefined);const leader={scope,fence:1,active:true,async release(){owner=undefined;leader.active=false;}};owner=leader;return leader;},async recover(){return image?structuredClone(image):null;},async commit(leader,batch){assert.equal(leader,owner);for(const file of batch.files??[])files.set(file.id,file.bytes);for(const id of batch.deleteFiles??[])files.delete(id);image=structuredClone(batch.image);batches.push(batch);return{durableRootRevision:image.rootRevision};},async readFile(leader,id){assert.equal(leader,owner);return files.get(id);},async scanFiles(leader,{expectedRootRevision,after,limit}){assert.equal(leader,owner);if(image.rootRevision!==expectedRootRevision)throw Error('stale_file_inventory');const rows=[...files].sort(([a],[b])=>ab?1:0).filter(([id])=>after===undefined||id>after),page=rows.slice(0,limit).map(([id,blob])=>({id,byteLength:blob.size,mimeType:blob.type}));return{files:page,rootRevision:image.rootRevision,...(rows.length>limit?{nextAfter:page.at(-1).id}:{})};},close(){}}; + const port={compile:(intent,id)=>({payload:{intent,id},canvasIds:intent.canvasIds,expectedVersions:{C:version}}),async submit(request){requests.push(request);server=applyChanges(server,JSON.parse(request.body).intent.changes);receipts.set(request.id,{document:server,versions:new Map([['C',++version]])});return{outcome:'applied'};},async query(request){return{outcome:receipts.has(request.id)?'applied':'unresolved'};},canonical:request=>receipts.get(request.id)}; + return{files,batches,requests,storage,advance:ms=>{now+=ms;},image:()=>image,open:()=>openProjectEngine({scope,registry,document,versions:new Map([['C',1]]),storage,persistence:port,createId:()=>`id-${++next}`,clock:{now:()=>now,setTimer:(fn,ms)=>setTimeout(fn,ms),clearTimer:id=>clearTimeout(id)}})}; +} +test('inventory is bounded and read-only; unregistered but referenced files stay protected',async()=>{ + const doc=createDocument('p',[{id:'C',parent:null}],[node('A',{assetId:'used'})]),f=fixture(new Map([['old',new Blob(['old'])],['used',new Blob(['used'])]]),doc),engine=await f.open();try{await engine.persist();const root=engine.getSnapshot(),first=await engine.inventoryLocalFiles({limit:1});assert.equal(engine.getSnapshot(),root);assert.equal(first.files[0].id,'old');assert.equal(first.files[0].ownership,'unregistered');assert.equal(first.nextAfter,'old');const second=await engine.inventoryLocalFiles({after:first.nextAfter,limit:1});assert.equal(second.files[0].id,'used');assert.ok(second.files[0].pins.some(owner=>owner.startsWith('element:')));assert.equal(second.nextAfter,undefined);assert.equal(f.requests.length,0); + const marked=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(marked.markedIds,['old']);assert.deepEqual(marked.purgedIds,[]);assert.equal(f.image().formatVersion,5);f.advance(100);const result=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(result.purgedIds,['old']);assert.equal(f.files.has('old'),false);assert.equal(f.files.has('used'),true);assert.equal(engine.getSnapshot().document,root.document);assert.equal(engine.getSnapshot().history,root.history);assert.equal(f.requests.length,0);decodeRecovery(f.image(),registry); + }finally{await engine.close();} +}); +test('retired orphan identity and v5 recovery survive reopening and reject new references or silent downgrade',async()=>{ + const f=fixture(new Map([['old',new Blob(['old'])]]));let engine=await f.open();try{await engine.collectOrphanFiles({retentionMs:0});const file=engine.getSnapshot().runtime.orphanFiles.get('old');assert.ok(file.retirement.purgedAt!==undefined);for(const formatVersion of [1,2,3,4])assert.throws(()=>decodeRecovery({...f.image(),formatVersion},registry),/unsupported_recovery_version/);await engine.close();engine=await f.open();await assert.rejects(engine.displayResource('old'),/local_file_retired/);assert.equal(engine.editor.dispatch({type:'element.create',element:node('bad',{assetId:'old'})}).code,'local_file_retired');assert.equal(engine.editor.updateRuntime(previous=>({...previous,orphanFiles:new Map()})).code,'local_file_retired');assert.equal(engine.editor.dispatch({type:'element.data',target:{id:'A',incarnation:1},patch:{text:'still editable'}}).status,'applied');await engine.settle();assert.equal(f.files.has('old'),false); + }finally{await engine.close();} +}); +test('changed orphan bytes reset the retention interval even when size and MIME stay equal',async()=>{ + const f=fixture(new Map([['old',new Blob(['aa'])]])),engine=await f.open();try{await engine.collectOrphanFiles({retentionMs:100});const digest=engine.getSnapshot().runtime.orphanFiles.get('old').digest;f.advance(100);f.files.set('old',new Blob(['bb']));const changed=await engine.collectOrphanFiles({retentionMs:100});assert.deepEqual(changed.changedIds,['old']);assert.deepEqual(changed.purgedIds,[]);const record=engine.getSnapshot().runtime.orphanFiles.get('old');assert.notEqual(record.digest,digest);assert.equal(record.retirement.unreferencedAt,1100);f.advance(100);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:100})).purgedIds,['old']);}finally{await engine.close();} +}); +test('display borrowing and explicit owners reset soft markers before collection',async()=>{ + const f=fixture(new Map([['old',new Blob(['data'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile;try{await engine.collectOrphanFiles({retentionMs:100});f.advance(100);f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const opening=engine.displayResource('old');await entered.promise;assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:0})).purgedIds,[]);gate.resolve();const handle=await opening;handle.release();await engine.collectOrphanFiles({retentionMs:100});engine.editor.updateRuntime(previous=>({...previous,resourcePins:new Map([['export',['old']]])}));assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:0})).purgedIds,[]);}finally{gate.resolve();await engine.close();} +}); +test('hash budgets preserve pagination progress and a changed Root invalidates read-only verification',async()=>{ + const f=fixture(new Map([['a',new Blob(['a'])],['b',new Blob(['bb'])],['c',new Blob(['c'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile;try{const first=await engine.collectOrphanFiles({retentionMs:100,maxHashBytes:1});assert.deepEqual(first.markedIds,['a']);assert.equal(first.nextAfter,'a');assert.equal(first.hashBudgetExhausted,true);assert.equal(first.scanFinished,false);const second=await engine.collectOrphanFiles({retentionMs:100,after:first.nextAfter,maxHashBytes:3});assert.deepEqual(second.markedIds,['b','c']);assert.equal(second.scanFinished,true); + f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const collecting=engine.collectOrphanFiles({retentionMs:0});await entered.promise;assert.equal(engine.editor.dispatch({type:'element.data',target:{id:'A',incarnation:1},patch:{text:'changed while hashing'}}).status,'applied');gate.resolve();await assert.rejects(collecting,/stale_file_inventory/);assert.equal(f.files.size,3); + }finally{gate.resolve();await engine.close();} +}); +test('cancelling verification never publishes late byte results and failures retain the original purge cut',async()=>{ + const f=fixture(new Map([['old',new Blob(['old'])]])),engine=await f.open(),gate=deferred(),entered=deferred(),read=f.storage.readFile,cancel=new AbortController();let fail=false;const commit=f.storage.commit;try{f.storage.readFile=async(...args)=>{const bytes=await read(...args);entered.resolve();await gate.promise;return bytes;};const task=engine.collectOrphanFiles({retentionMs:0,signal:cancel.signal});await entered.promise;cancel.abort();await assert.rejects(task,/file_inventory_aborted/);gate.resolve();await Promise.resolve();assert.equal(engine.getSnapshot().runtime.orphanFiles?.size??0,0);f.storage.readFile=read;f.storage.commit=async(...args)=>{if(fail&&args[1].deleteFiles?.length)throw Error('disk failure');return commit(...args);};fail=true;await assert.rejects(engine.collectOrphanFiles({retentionMs:0}),/disk failure/);assert.ok(f.files.has('old'));await assert.rejects(engine.displayResource('old'),/resource_collection_in_progress/);fail=false;await engine.persist();assert.equal(f.files.has('old'),false);assert.ok(decodeRecovery(f.image(),registry).runtime.orphanFiles.get('old').retirement.purgedAt!==undefined); + }finally{fail=false;gate.resolve();await engine.close();} +}); +test('opaque producers block orphan collection even when the known local resource map is empty',async()=>{ + const f=fixture(),engine=await f.open();try{const root=engine.getSnapshot(),coverage=runtime=>orphanCoverage({...root,runtime},registry,new Map(),new Map());assert.equal(coverage({...root.runtime,runs:new Map([['run',{id:'run'}]])}).complete,false);assert.ok(coverage({...root.runtime,recoveries:new Map([['audit',{}]])}).blockers.includes('recovery_audits'));assert.ok(coverage({...root.runtime,imports:new Map([['job',{id:'job',status:'paused',source:{fileId:'source'}}]])}).blockers.includes('unfinished_imports'));assert.ok(orphanCoverage(root,registry,new Map(),new Map([['copy-admission',[]]])).blockers.includes('copy_admission')); + }finally{await engine.close();} +}); + +test('a cancelled copy admission resets orphan retention before any ImportJob is published',async()=>{ + const f=fixture(new Map([['old',new Blob(['old'])]])),engine=await f.open(),cancel=new AbortController();try{await engine.collectOrphanFiles({retentionMs:100});f.advance(100);cancel.abort();await assert.rejects(engine.imports.duplicate([{id:'A',incarnation:1}],{signal:cancel.signal}),/copy_aborted/);assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement,undefined);assert.deepEqual((await engine.collectOrphanFiles({retentionMs:100})).purgedIds,[]);assert.equal(engine.getSnapshot().runtime.orphanFiles.get('old').retirement.unreferencedAt,1100);}finally{await engine.close();} +}); diff --git a/tests/headless/recovery-encoding.test.mjs b/tests/headless/recovery-encoding.test.mjs index 74c0c87..fb6f08e 100644 --- a/tests/headless/recovery-encoding.test.mjs +++ b/tests/headless/recovery-encoding.test.mjs @@ -24,5 +24,5 @@ test('cached subtree height cannot bypass the depth bound and omitted object fie const cycle={};cycle.self=cycle;assert.throws(()=>encodeRecoveryValue(cycle),/invalid_recovery_payload/);assert.throws(()=>encodeRecoveryValue({big:1n})); }); test('only factory-owned images are recognized; immutable-looking impostors retain defensive-copy handling',()=>{ - const mutable={title:'before'},image=sealRecoveryImage(2,3,2,{mutable});assert.equal(isSealedRecoveryImage(image),true);assert.equal(isSealedRecoveryImage(structuredClone(image)),false);assert.equal(isSealedRecoveryImage(Object.freeze({...image})),false);mutable.title='after';assert.equal(image.payload.mutable.title,'before');assert.ok(Object.isFrozen(image.payload.mutable));assert.throws(()=>sealRecoveryImage(2,1,2,{}),/corrupt_recovery_image/);assert.equal(isSealedRecoveryImage(sealRecoveryImage(3,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(4,1,1,{})),true);assert.throws(()=>sealRecoveryImage(5,1,1,{}),/corrupt_recovery_image/); + const mutable={title:'before'},image=sealRecoveryImage(2,3,2,{mutable});assert.equal(isSealedRecoveryImage(image),true);assert.equal(isSealedRecoveryImage(structuredClone(image)),false);assert.equal(isSealedRecoveryImage(Object.freeze({...image})),false);mutable.title='after';assert.equal(image.payload.mutable.title,'before');assert.ok(Object.isFrozen(image.payload.mutable));assert.throws(()=>sealRecoveryImage(2,1,2,{}),/corrupt_recovery_image/);assert.equal(isSealedRecoveryImage(sealRecoveryImage(3,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(4,1,1,{})),true);assert.equal(isSealedRecoveryImage(sealRecoveryImage(5,1,1,{})),true);assert.throws(()=>sealRecoveryImage(6,1,1,{}),/corrupt_recovery_image/); });