Collect unreferenced local resource bytes atomically with recovery cuts

This commit is contained in:
2026-11-08 22:26:06 +08:00
parent 6387939510
commit bbe1e60496
11 changed files with 285 additions and 31 deletions
+69 -11
View File
@@ -1,3 +1,4 @@
import {resourceRetentionPins,prepareResourceRetention} from './resource-retention.ts';
import { encodeIndexedCopySnapshot, openIndexedCopySnapshot, type IndexedCopySnapshot } from './copy-snapshot-index.ts';
import { decodeCopySnapshot, copyRegistrySignature } from './copy-snapshot.ts';
import { createDuplicatePlanSource } from './duplicate-source.ts';
@@ -61,6 +62,8 @@ export async function openProjectEngine(options: EngineOptions) {
const network = createNetworkGate(2);
const runClock: Clock = options.clock ?? { now: () => Date.now(), setTimer: (fn, delay) => setTimeout(fn, delay), clearTimer: handle => clearTimeout(handle as ReturnType<typeof setTimeout>) };
const displays = new Set<() => void>();
const displayPins=new Map<Id,number>(),collectionReservations=new Set<Id>(),durablyPurged=new Set<Id>([...(recovered?.runtime.resources.values()??[])].filter(resource=>resource.retirement?.purgedAt!==undefined).map(resource=>resource.id));
let collection:Promise<{readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly complete:boolean;readonly checkedRootRevision:number;readonly rootRevision:number;readonly durableRootRevision:number}>|undefined;
const fileWrites = new Map<Id, Blob>(), digestTasks = new Map<Id, Promise<void>>();
let staged: readonly ResourceRecord[] = [];
let applying: RunApplyInput | undefined;
@@ -69,12 +72,14 @@ export async function openProjectEngine(options: EngineOptions) {
const copyPreparations=new Map<AbortController,Promise<Id>>();
const importRunners=new Map<Id,ReturnType<typeof createImportRunner>>(),importLoads=new Map<Id,Promise<void>>(),importHolds=new Map<symbol,Map<Id,()=>void>>();
let recovery: { controller: AbortController; release(): void; cancel():Promise<void> } | undefined;
const admissionPins=(snapshot:EngineSnapshot)=>copyPreparations.size?new Map([['copy-admission',[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt===undefined).map(resource=>resource.id)]]):new Map<Id,readonly Id[]>();
const assertWritable = () => { if(closing||closed)throw fault('engine_closing');if(recovery)throw fault('recovery_in_progress'); };
const initial: RuntimeState = recovered?.runtime ?? Object.freeze({ imports:readMap(new Map()), recoveries:readMap(new Map()), runs: readMap(new Map()), acknowledged: createDocument(options.document.projectId, [...options.document.canvases.values()], [...options.document.elements.values()], options.document.tombstones), versions: readMap(options.versions), ledger: createLedger().getSnapshot(), receipts: readMap(new Map()), conflicts: readMap(new Map()), resources: readMap(new Map()), resourcePins: readMap(new Map()) });
const editor = createEditor({ registry: options.registry, document: options.document, recovery: recovered, runtime: initial, createId: options.createId, historyLimit: options.historyLimit,queryBudget:options.queryBudget,
policy(command, document) { if (closing || closed || recovery) return { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] }; return options.policy?.(command, document) ?? { enabled: true }; },
traversalPolicy(entry, direction) { return closing || closed || recovery ? { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] } : options.traversalPolicy?.(entry, direction) ?? { enabled: true }; },
onListenerError: report,
prepareRuntime:(_previous,next)=>prepareResourceRetention(next,options.registry,displayPins,collectionReservations,admissionPins(next)),
prepareIntent: intent => applying && intent.origin==='user' ? {...intent,operationType:'run.apply',action:applying} : intent,
nextOperationSequence(runtime, fallback) { let next = fallback; for (const record of runtime.ledger.operations.values()) next = Math.max(next, record.intent.sequence + 1); return next + staged.length; },
reduceRuntime(previous, next, commit) {
@@ -113,9 +118,11 @@ export async function openProjectEngine(options: EngineOptions) {
while (pending.size) {
const [revision, snapshot] = pending.entries().next().value!;
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||[...(snapshot.runtime.imports?.values()??[])].some(job=>job.source?.fileId===id)).map(([id, bytes]) => ({ id, bytes }));
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files });
const deleteFiles=[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id);
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles });
if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt');
durableRootRevision = revision; pending.delete(revision);
for(const id of deleteFiles){durablyPurged.add(id);collectionReservations.delete(id);}
for (const file of files) fileWrites.delete(file.id);
if (editor.getSnapshot().runtime.ledger.durableGeneration < snapshot.runtime.ledger.generation) mutate(ledger => ledger.markDurable(snapshot.runtime.ledger.generation));
}
@@ -124,6 +131,39 @@ export async function openProjectEngine(options: EngineOptions) {
writing = task.catch(error => { persistenceFailed = true; throw error; }).finally(() => { writing = undefined; });
return writing;
}
function collectResources(settings:{retentionMs:number;maxFiles?:number}) {
assertWritable();
if(options.storage.fileDeletion!=='atomic-v1')throw fault('resource_collection_unavailable');
const retentionMs=settings.retentionMs,maxFiles=settings.maxFiles??64;
if(!Number.isSafeInteger(retentionMs)||retentionMs<0||!Number.isSafeInteger(maxFiles)||maxFiles<1||maxFiles>128)throw fault('invalid_resource_collection');
if(collection)return collection;
const task=(async()=>{
await persist();assertWritable();
const now=runClock.now();if(!Number.isSafeInteger(now)||now<0)throw fault('invalid_resource_collection_clock');
let root=editor.getSnapshot(),pins=resourceRetentionPins(root,options.registry,displayPins,admissionPins(root));
const markedIds:Id[]=[],purgedIds:Id[]=[];
const report=()=>Object.freeze({markedIds:Object.freeze(markedIds),purgedIds:Object.freeze(purgedIds),complete:pins.complete,checkedRootRevision:root.rootRevision,rootRevision:editor.getSnapshot().rootRevision,durableRootRevision});
if(!pins.complete)return report();
const fresh=[...root.runtime.resources.values()].filter(resource=>!resource.retirement&&!pins.pins.has(resource.id)&&!fileWrites.has(resource.id)&&!digestTasks.has(resource.id)).slice(0,maxFiles);
if(fresh.length){
const result=editor.updateRuntime(previous=>{const resources=new Map(previous.resources);for(const resource of fresh){resources.set(resource.id,freeze({...resource,retirement:{version:1 as const,unreferencedAt:now}}));markedIds.push(resource.id);}return Object.freeze({...previous,resources:readMap(resources)});},root.rootRevision);
if(result.status==='rejected')throw fault(result.code,result.targets);
await persist();assertWritable();
}
root=editor.getSnapshot();pins=resourceRetentionPins(root,options.registry,displayPins,admissionPins(root));
if(!pins.complete)return report();
const eligible=[...root.runtime.resources.values()].filter(resource=>resource.retirement&&resource.retirement.purgedAt===undefined&&now-resource.retirement.unreferencedAt>=retentionMs&&!pins.pins.has(resource.id)&&!fileWrites.has(resource.id)&&!digestTasks.has(resource.id)).slice(0,maxFiles);
if(eligible.length){
for(const resource of eligible)collectionReservations.add(resource.id);
const result=editor.updateRuntime(previous=>{const resources=new Map(previous.resources);for(const resource of eligible)resources.set(resource.id,freeze({...resource,retirement:{...resource.retirement!,purgedAt:now}}));return Object.freeze({...previous,resources:readMap(resources)});},root.rootRevision);
if(result.status==='rejected'){for(const resource of eligible)collectionReservations.delete(resource.id);throw fault(result.code,result.targets);}
// Persistence derives byte deletions from this immutable root, even if a listener already started it.
await persist();for(const resource of eligible)purgedIds.push(resource.id);
}
return report();
})();
collection=task.finally(()=>{collection=undefined;});return collection;
}
function mutate(reduce: (ledger: ReturnType<typeof createLedger>) => void) {
const result = editor.updateRuntime(previous => { const ledger = createLedger(previous.ledger); reduce(ledger); return Object.freeze({ ...previous, ledger: ledger.getSnapshot() }); });
if (result.status === 'rejected') throw fault(result.code, result.targets);
@@ -413,7 +453,12 @@ export async function openProjectEngine(options: EngineOptions) {
if(result.status==='rejected'){fileWrites.delete(fileId);throw fault(result.code,result.targets);}void loadImport(id);return id;
},
async duplicate(selection:readonly Target[],settings:{readonly worldOffset?:Point;readonly signal?:AbortSignal;readonly viewCanvasId?:Id}={}):Promise<Id>{
assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);const captured=editor.getSnapshot(),target=settings.viewCanvasId??(selection[0]?captured.document.elements.get(selection[0].id)?.ownerCanvasId:undefined),signal=settings.signal,controller=new AbortController(),abort=()=>controller.abort();
assertWritable();if(collectionReservations.size)throw fault('resource_collection_in_progress',[...collectionReservations]);
const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);
if([...editor.getSnapshot().runtime.resources.values()].some(resource=>resource.retirement&&resource.retirement.purgedAt===undefined)){
const reset=editor.updateRuntime(previous=>{const resources=new Map(previous.resources);for(const [id,resource]of resources)if(resource.retirement&&resource.retirement.purgedAt===undefined){const {retirement:_,...kept}=resource;resources.set(id,freeze(kept));}return Object.freeze({...previous,resources:readMap(resources)});});if(reset.status==='rejected')throw fault(reset.code,reset.targets);
}
const captured=editor.getSnapshot(),target=settings.viewCanvasId??(selection[0]?captured.document.elements.get(selection[0].id)?.ownerCanvasId:undefined),signal=settings.signal,controller=new AbortController(),abort=()=>controller.abort();
const targets=freeze(clone(selection)),offset=freeze(clone(settings.worldOffset??{x:32,y:32}));
for(const selected of targets){const element=captured.document.elements.get(selected.id);if(!element||element.incarnation!==selected.incarnation)throw fault('stale_incarnation',[selected.id]);}if(![offset.x,offset.y].every(Number.isFinite))throw fault('invalid_copy_offset');
signal?.addEventListener('abort',abort,{once:true});if(signal?.aborted)controller.abort();
@@ -437,7 +482,8 @@ export async function openProjectEngine(options: EngineOptions) {
scheduler.start(); prepareDigests(); runTracker.notify([...initial.runs.keys()]);
for(const job of initial.imports?.values()??[])if(['running','preparing'].includes(job.status))void loadImport(job.id);
return Object.freeze({
editor, sessions, imports,
editor, sessions, imports, collectResources,
getResourcePins:()=>resourceRetentionPins(editor.getSnapshot(),options.registry,displayPins,admissionPins(editor.getSnapshot())),
/** Read/preview under a scheduling hold. No request is rewritten or declared resolved here. */
async prepareRecovery(): Promise<RecoverySession> {
assertWritable();
@@ -587,13 +633,25 @@ export async function openProjectEngine(options: EngineOptions) {
},
async displayResource(id: Id) {
if (closing || closed) throw fault('engine_closing');
const bytes = fileWrites.get(id) ?? await options.storage.readFile(leader, id);
if (closing || closed) throw fault('engine_closing');
if (!bytes && !options.assets) throw fault('resource_missing', [id]);
const url = bytes ? URL.createObjectURL(bytes) : options.assets!.displayURL(id);
let released = false;
const release = () => { if (!released) { released = true; displays.delete(release); if (bytes) URL.revokeObjectURL(url); } };
displays.add(release); return Object.freeze({ url, release });
if(collectionReservations.has(id))throw fault('resource_collection_in_progress',[id]);
displayPins.set(id,(displayPins.get(id)??0)+1);
let released=false,url:string|undefined,local=false;
const release=()=>{if(released)return;released=true;displays.delete(release);const count=displayPins.get(id)??1;if(count<=1)displayPins.delete(id);else displayPins.set(id,count-1);if(local&&url)URL.revokeObjectURL(url);};
displays.add(release);
try{
const retained=editor.getSnapshot().runtime.resources.get(id);
if(retained?.retirement&&retained.retirement.purgedAt===undefined){
const result=editor.updateRuntime(previous=>{const resources=new Map(previous.resources),{retirement:_,...kept}=retained;resources.set(id,freeze(kept));return Object.freeze({...previous,resources:readMap(resources)});});
if(result.status==='rejected')throw fault(result.code,result.targets);
}
const bytes = fileWrites.get(id) ?? await options.storage.readFile(leader, id);
if (closing || closed) throw fault('engine_closing');
const resource=editor.getSnapshot().runtime.resources.get(id);
if(!bytes&&resource?.retirement?.purgedAt!==undefined&&!resource.stored)throw fault('resource_bytes_unavailable',[id]);
if (!bytes && !options.assets) throw fault('resource_missing', [id]);
local=!!bytes;url=bytes?URL.createObjectURL(bytes):options.assets!.displayURL(id);
return Object.freeze({url,release});
}catch(error){release();throw error;}
},
getSnapshot: editor.getSnapshot, persist,
cancelRecovery:async()=>{await recovery?.cancel();},
@@ -602,7 +660,7 @@ export async function openProjectEngine(options: EngineOptions) {
async settle() { await Promise.resolve();await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async close() {
if (closed) return;
closing = true;digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
closing = true;await collection?.catch(report);digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
await Promise.all([...importRunners.values()].map(runner=>runner.close()));while(importLoads.size)await Promise.all([...importLoads.values()]);while(digestTasks.size)await Promise.allSettled([...digestTasks.values()]);
recovery?.controller.abort();recovery?.release();
for (const release of [...displays]) release();