Retire terminal import sources with durable ownership and recovery v4

This commit is contained in:
2026-11-08 22:55:08 +08:00
parent bbe1e60496
commit 4b594f85eb
13 changed files with 252 additions and 25 deletions
+70 -10
View File
@@ -1,3 +1,4 @@
import {importFiles,terminalImport,markImportFiles,prepareImportFileRetention,type FileRetirement} from './import-file-retention.ts';
import {resourceRetentionPins,prepareResourceRetention} from './resource-retention.ts';
import { encodeIndexedCopySnapshot, openIndexedCopySnapshot, type IndexedCopySnapshot } from './copy-snapshot-index.ts';
import { decodeCopySnapshot, copyRegistrySignature } from './copy-snapshot.ts';
@@ -63,7 +64,10 @@ export async function openProjectEngine(options: EngineOptions) {
const runClock: Clock = options.clock ?? { now: () => Date.now(), setTimer: (fn, delay) => setTimeout(fn, delay), clearTimer: handle => clearTimeout(handle as ReturnType<typeof setTimeout>) };
const displays = new Set<() => void>();
const displayPins=new Map<Id,number>(),collectionReservations=new Set<Id>(),durablyPurged=new Set<Id>([...(recovered?.runtime.resources.values()??[])].filter(resource=>resource.retirement?.purgedAt!==undefined).map(resource=>resource.id));
let collection:Promise<{readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly complete:boolean;readonly checkedRootRevision:number;readonly rootRevision:number;readonly durableRootRevision:number}>|undefined;
type CollectionReport={readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly complete:boolean;readonly checkedRootRevision:number;readonly rootRevision:number;readonly durableRootRevision:number};
let sourceCollection:Promise<CollectionReport>|undefined;
for(const [id,entry]of importFiles(recovered?.runtime.imports))if(entry.file.retirement?.purgedAt!==undefined)durablyPurged.add(id);
let collection:Promise<CollectionReport>|undefined;
const fileWrites = new Map<Id, Blob>(), digestTasks = new Map<Id, Promise<void>>();
let staged: readonly ResourceRecord[] = [];
let applying: RunApplyInput | undefined;
@@ -79,7 +83,10 @@ export async function openProjectEngine(options: EngineOptions) {
policy(command, document) { if (closing || closed || recovery) return { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] }; return options.policy?.(command, document) ?? { enabled: true }; },
traversalPolicy(entry, direction) { return closing || closed || recovery ? { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] } : options.traversalPolicy?.(entry, direction) ?? { enabled: true }; },
onListenerError: report,
prepareRuntime:(_previous,next)=>prepareResourceRetention(next,options.registry,displayPins,collectionReservations,admissionPins(next)),
prepareRuntime:(previous,next)=>{
const runtime=prepareResourceRetention(next,options.registry,displayPins,collectionReservations,admissionPins(next));
return prepareImportFileRetention(previous.runtime,runtime,()=>resourceRetentionPins({...next,runtime},options.registry,displayPins,admissionPins({...next,runtime})),collectionReservations,previous.document!==next.document||previous.history!==next.history||previous.runtime.ledger!==runtime.ledger||previous.runtime.resourcePins!==runtime.resourcePins);
},
prepareIntent: intent => applying && intent.origin==='user' ? {...intent,operationType:'run.apply',action:applying} : intent,
nextOperationSequence(runtime, fallback) { let next = fallback; for (const record of runtime.ledger.operations.values()) next = Math.max(next, record.intent.sequence + 1); return next + staged.length; },
reduceRuntime(previous, next, commit) {
@@ -117,8 +124,9 @@ export async function openProjectEngine(options: EngineOptions) {
const task = (async () => {
while (pending.size) {
const [revision, snapshot] = pending.entries().next().value!;
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||[...(snapshot.runtime.imports?.values()??[])].some(job=>job.source?.fileId===id)).map(([id, bytes]) => ({ id, bytes }));
const deleteFiles=[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id);
const sourceFiles=importFiles(snapshot.runtime.imports);
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||sourceFiles.has(id)).map(([id, bytes]) => ({ id, bytes }));
const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)];
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles });
if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt');
durableRootRevision = revision; pending.delete(revision);
@@ -164,6 +172,52 @@ export async function openProjectEngine(options: EngineOptions) {
})();
collection=task.finally(()=>{collection=undefined;});return collection;
}
function collectImportSources(settings:{retentionMs:number;maxFiles?:number}):Promise<CollectionReport> {
assertWritable();if(options.storage.fileDeletion!=='atomic-v1')throw fault('resource_collection_unavailable');
const retentionMs=settings.retentionMs,maxFiles=settings.maxFiles??64;
if(!Number.isSafeInteger(retentionMs)||retentionMs<0||!Number.isSafeInteger(maxFiles)||maxFiles<1||maxFiles>128)throw fault('invalid_resource_collection');
if(sourceCollection)return sourceCollection;
const task=(async()=>{
await persist();assertWritable();
const now=runClock.now();if(!Number.isSafeInteger(now)||now<0)throw fault('invalid_resource_collection_clock');
let root=editor.getSnapshot(),pins=resourceRetentionPins(root,options.registry,displayPins,admissionPins(root));
const markedIds:Id[]=[],purgedIds:Id[]=[];
const report=()=>Object.freeze({markedIds:Object.freeze(markedIds),purgedIds:Object.freeze(purgedIds),complete:pins.complete,checkedRootRevision:root.rootRevision,rootRevision:editor.getSnapshot().rootRevision,durableRootRevision});
const candidates=()=>[...importFiles(root.runtime.imports)].filter(([id,entry])=>!pins.pins.has(id)&&!fileWrites.has(id)&&(!entry.current||terminalImport(root.runtime.imports!.get(entry.jobId)!.status)&&!importLoads.has(entry.jobId)));
if(!pins.complete)return report();
const fresh=candidates().filter(([,entry])=>!entry.file.retirement).slice(0,maxFiles);
if(fresh.length){
const markers=new Map(fresh.map(([id])=>[id,{version:1 as const,unreferencedAt:now}]));
const result=editor.updateRuntime(previous=>markImportFiles(previous,markers),root.rootRevision);if(result.status==='rejected')throw fault(result.code,result.targets);
markedIds.push(...markers.keys());await persist();assertWritable();
}
root=editor.getSnapshot();pins=resourceRetentionPins(root,options.registry,displayPins,admissionPins(root));if(!pins.complete)return report();
const eligible=candidates().filter(([,entry])=>entry.file.retirement&&entry.file.retirement.purgedAt===undefined&&now-entry.file.retirement.unreferencedAt>=retentionMs).slice(0,maxFiles);
if(eligible.length){
const markers=new Map<Id,FileRetirement>(eligible.map(([id,entry])=>[id,{...entry.file.retirement!,purgedAt:now}]));
for(const id of markers.keys())collectionReservations.add(id);
const result=editor.updateRuntime(previous=>markImportFiles(previous,markers),root.rootRevision);
if(result.status==='rejected'){for(const id of markers.keys())collectionReservations.delete(id);throw fault(result.code,result.targets);}
await persist();purgedIds.push(...markers.keys());
}
return report();
})();
sourceCollection=task.finally(()=>{sourceCollection=undefined;});return sourceCollection;
}
async function openImportSource(jobId:Id,fileId?:Id) {
if(closing||closed)throw fault('engine_closing');const job=getImport(jobId),file=fileId!==undefined?importFiles(new Map([[jobId,job]])).get(fileId)?.file:job.source;if(!file)throw fault('import_source_unavailable');
if(collectionReservations.has(file.fileId))throw fault('resource_collection_in_progress',[file.fileId]);
if(file.retirement?.purgedAt!==undefined)throw fault('import_source_retired',[file.fileId]);
displayPins.set(file.fileId,(displayPins.get(file.fileId)??0)+1);let released=false;
const release=()=>{if(released)return;released=true;displays.delete(release);const count=displayPins.get(file.fileId)??1;if(count<=1)displayPins.delete(file.fileId);else displayPins.set(file.fileId,count-1);};displays.add(release);
try{
if(file.retirement){const result=editor.updateRuntime(previous=>Object.freeze({...previous}));if(result.status==='rejected')throw fault(result.code,result.targets);}
const bytes=fileWrites.get(file.fileId)??await options.storage.readFile(leader,file.fileId);
if(closing||closed)throw fault('engine_closing');if(!bytes||bytes.size!==file.byteLength)throw fault('import_source_unavailable',[file.fileId]);
if(file.sha256&&await sourceDigest(bytes)!==file.sha256)throw fault('import_source_changed',[file.fileId]);
if(closing||closed)throw fault('engine_closing');return Object.freeze({bytes,release});
}catch(error){release();if(closing||closed)throw fault('engine_closing');throw error;}
}
function mutate(reduce: (ledger: ReturnType<typeof createLedger>) => void) {
const result = editor.updateRuntime(previous => { const ledger = createLedger(previous.ledger); reduce(ledger); return Object.freeze({ ...previous, ledger: ledger.getSnapshot() }); });
if (result.status === 'rejected') throw fault(result.code, result.targets);
@@ -398,7 +452,7 @@ export async function openProjectEngine(options: EngineOptions) {
const permission=options.policy?.({type:'changes.apply',changes:plan.changes},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);
const fileId=createId(),runtime=editor.getSnapshot().runtime;if(!fileId||fileWrites.has(fileId)||runtime.resources.has(fileId)||[...(runtime.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
const stableIds=Object.fromEntries([...Object.entries(plan.elementIds).map(([old,id])=>[`element:${old}`,id]),...Object.entries(plan.canvasIds).map(([old,id])=>[`canvas:${old}`,id])]);
const next={...job,generation:job.generation+1,status:'running' as const,stableIds,sourceFingerprint:`local-file:${fileId}:${encoded.size}`,source:{...job.source!,kind:'duplicate-ndjson' as const,fileId,byteLength:encoded.size,sha256,integrityVersion:1 as const,copy:{sourceRevision:job.source!.copy!.sourceRevision,roots:plan.roots}}};delete next.preparation;delete next.error;
const next={...job,generation:job.generation+1,status:'running' as const,stableIds,sourceFingerprint:`local-file:${fileId}:${encoded.size}`,source:{...job.source!,previousFiles:[...(job.source!.previousFiles??[]),{fileId:job.source!.fileId,byteLength:job.source!.byteLength,...(job.source!.sha256?{sha256:job.source!.sha256}:{})}],kind:'duplicate-ndjson' as const,fileId,byteLength:encoded.size,sha256,integrityVersion:1 as const,copy:{sourceRevision:job.source!.copy!.sourceRevision,roots:plan.roots}}};delete next.preparation;delete next.error;
fileWrites.set(fileId,encoded);try{publishImport(freeze(next),job.generation);}catch(error){fileWrites.delete(fileId);throw error;}
return{job:getImport(id),bytes:encoded,document:undefined,indexed:undefined};
}
@@ -443,9 +497,10 @@ export async function openProjectEngine(options: EngineOptions) {
}
const imports=Object.freeze({
getJobs:()=>readMap(editor.getSnapshot().runtime.imports??new Map()),
openSource:openImportSource,
startNdjson(bytes:Blob,targetCanvasId:Id):Id{
assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);const canvas=editor.getSnapshot().document.canvases.get(targetCanvasId);if(!canvas)throw fault('canvas_not_loaded',[targetCanvasId]);if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source');
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict');
const entry=canvas.parent?editor.getSnapshot().document.elements.get(canvas.parent.entryElementId)!:undefined;
const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'native-ndjson',name:((bytes as Blob&{name?:string}).name||'画布数据').slice(0,255),fileId,byteLength:bytes.size,targetCanvasId,...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'running' as const});
fileWrites.set(fileId,bytes);
@@ -467,7 +522,7 @@ export async function openProjectEngine(options: EngineOptions) {
const bytes=await encodeIndexedCopySnapshot(captured.document,controller.signal);assertWritable();if(controller.signal.aborted)throw fault('copy_aborted');
const canvas=captured.document.canvases.get(target);if(!canvas)throw fault('canvas_not_loaded',[target]);
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;
if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||importFiles(current.imports).has(fileId))throw fault('import_identity_conflict');
const entry=canvas.parent?captured.document.elements.get(canvas.parent.entryElementId)!:undefined;
const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'duplicate-snapshot',snapshotVersion:2,name:'复制选中内容',fileId,byteLength:bytes.size,targetCanvasId:target,copy:{sourceRevision:captured.revision,roots:[]},...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'preparing' as const,preparation:{version:4 as const,registrySignature:copyRegistrySignature(captured.document,options.registry),selection:targets,worldOffset:offset,allocationIds:[],steps:0}});
fileWrites.set(fileId,bytes);const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});});
@@ -482,7 +537,7 @@ export async function openProjectEngine(options: EngineOptions) {
scheduler.start(); prepareDigests(); runTracker.notify([...initial.runs.keys()]);
for(const job of initial.imports?.values()??[])if(['running','preparing'].includes(job.status))void loadImport(job.id);
return Object.freeze({
editor, sessions, imports, collectResources,
editor, sessions, imports, collectResources,collectImportSources,
getResourcePins:()=>resourceRetentionPins(editor.getSnapshot(),options.registry,displayPins,admissionPins(editor.getSnapshot())),
/** Read/preview under a scheduling hold. No request is rewritten or declared resolved here. */
async prepareRecovery(): Promise<RecoverySession> {
@@ -622,7 +677,7 @@ export async function openProjectEngine(options: EngineOptions) {
if (!(file.bytes instanceof Blob) || !file.bytes.size || file.bytes.size > maxResourceBytes || !file.bytes.type.startsWith('image/') || typeof file.name !== 'string' || !file.name) throw fault('invalid_staged_file');
return freeze({ id: createId(), operationId: createId(), name: file.name, mimeType: file.bytes.type, byteLength: file.bytes.size });
});
if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || fileWrites.has(resource.id) || [...(editor.getSnapshot().runtime.imports?.values()??[])].some(job=>job.source?.fileId===resource.id))) throw fault('resource_identity_conflict');
if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || fileWrites.has(resource.id) || importFiles(editor.getSnapshot().runtime.imports).has(resource.id))) throw fault('resource_identity_conflict');
staged = resources; resources.forEach((resource, index) => fileWrites.set(resource.id, files[index].bytes));
try {
const handles = freeze(resources.map(resource => ({ id: resource.id, src: options.assets!.sourceURL(resource.id) })));
@@ -644,6 +699,11 @@ export async function openProjectEngine(options: EngineOptions) {
const result=editor.updateRuntime(previous=>{const resources=new Map(previous.resources),{retirement:_,...kept}=retained;resources.set(id,freeze(kept));return Object.freeze({...previous,resources:readMap(resources)});});
if(result.status==='rejected')throw fault(result.code,result.targets);
}
if(!retained){
const source=importFiles(editor.getSnapshot().runtime.imports).get(id)?.file;
if(source?.retirement?.purgedAt!==undefined)throw fault('import_source_retired',[id]);
if(source?.retirement){const result=editor.updateRuntime(previous=>Object.freeze({...previous}));if(result.status==='rejected')throw fault(result.code,result.targets);}
}
const bytes = fileWrites.get(id) ?? await options.storage.readFile(leader, id);
if (closing || closed) throw fault('engine_closing');
const resource=editor.getSnapshot().runtime.resources.get(id);
@@ -660,7 +720,7 @@ export async function openProjectEngine(options: EngineOptions) {
async settle() { await Promise.resolve();await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async close() {
if (closed) return;
closing = true;await collection?.catch(report);digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
closing = true;await Promise.allSettled([collection,sourceCollection].filter(Boolean));digestLifecycle.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
await Promise.all([...importRunners.values()].map(runner=>runner.close()));while(importLoads.size)await Promise.all([...importLoads.values()]);while(digestTasks.size)await Promise.allSettled([...digestTasks.values()]);
recovery?.controller.abort();recovery?.release();
for (const release of [...displays]) release();