Separate request Root durability barriers from complete storage flushes
This commit is contained in:
@@ -85,7 +85,7 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
function getDurability(){const rootRevision=editor.getSnapshot().rootRevision;if(!durabilityCache||durabilityCache.rootRevision!==rootRevision||durabilityCache.durableRootRevision!==durableRootRevision||durabilityCache.closing!==closing||durabilityCache.closed!==closed||durabilityCache.error!==durabilityError)durabilityCache=Object.freeze({rootRevision,durableRootRevision,closing,closed,...(durabilityError?{error:durabilityError}:{})});return durabilityCache;}
|
||||
function notifyDurability(){for(const listener of [...durabilityListeners])try{listener();}catch(error){report(error);}}
|
||||
const createId = options.createId ?? (() => crypto.randomUUID());
|
||||
const network = createNetworkGate(2);
|
||||
const network = createNetworkGate(2,()=>{if(!leader.active)throw fault('storage_leader_lost');if(options.isCurrentIdentity&&!options.isCurrentIdentity())throw fault('identity_changed');});
|
||||
const depot=options.assetDepot??(options.storage.assetDepot?createAssetDepot({scope:options.scope,port:options.storage.assetDepot,current:()=>!closing&&!closed&&(!options.isCurrentIdentity||options.isCurrentIdentity()),digest:sourceDigest}):undefined);
|
||||
if(depot&&(depot.scope.accountId!==options.scope.accountId||depot.scope.workspaceId!==options.scope.workspaceId))throw fault('asset_depot_scope_mismatch');
|
||||
const adoptionTasks=new Map<AbortController,Promise<Result>>(),adoptedFiles=new Map<Id,{readonly bytes:Blob;readonly rootRevision:number}>();
|
||||
@@ -199,28 +199,47 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
const pending = new Map<number, EngineSnapshot>();
|
||||
if (!recovered) pending.set(editor.getSnapshot().rootRevision, editor.getSnapshot());
|
||||
let writing: Promise<void> | undefined, scheduled = false, persistenceFailed = false;
|
||||
function persist(): Promise<void> {
|
||||
// Receipt-derived publication obligation survives a failed metadata publication.
|
||||
let confirmedLedgerGeneration=editor.getSnapshot().runtime.ledger.durableGeneration;
|
||||
function writePrefix(throughRevision:number):Promise<void> {
|
||||
if (writing) return writing;
|
||||
const task = (async () => {
|
||||
while (pending.size) {
|
||||
const [revision, snapshot] = pending.entries().next().value!;
|
||||
const sourceFiles=importFiles(snapshot.runtime.imports);
|
||||
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||sourceFiles.has(id)).map(([id, bytes]) => ({ id, bytes }));
|
||||
const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id),...[...(snapshot.runtime.orphanFiles??[])].filter(([id,file])=>file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)];
|
||||
const depot=options.assets?.authority&&options.storage.assetDepot?await prepareDepotCut({snapshot,registry:options.registry,storage:options.storage,leader,authority:options.assets.authority.scope,files:fileWrites,adoptedFiles,displays:displayPins,temporary:admissionPins(snapshot)}):undefined;
|
||||
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles,...(depot?{depot}:{}) });
|
||||
if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt');
|
||||
durableRootRevision = revision;durableImportSeals=importSeals(snapshot.runtime.imports); pending.delete(revision);notifyDurability();
|
||||
for(const id of deleteFiles){durablyPurged.add(id);collectionReservations.delete(id);}
|
||||
for (const file of files) fileWrites.delete(file.id);
|
||||
for(const [id,entry]of adoptedFiles)if(entry.rootRevision<=snapshot.rootRevision&&depot?.writes.some(write=>write.ref.asset.id===id&&write.bytes===entry.bytes))adoptedFiles.delete(id);
|
||||
if (editor.getSnapshot().runtime.ledger.durableGeneration < snapshot.runtime.ledger.generation) mutate(ledger => ledger.markDurable(snapshot.runtime.ledger.generation));
|
||||
const task = Promise.resolve().then(async () => {
|
||||
const publishDurability=()=>{if(editor.getSnapshot().runtime.ledger.durableGeneration<confirmedLedgerGeneration)mutate(ledger=>ledger.markDurable(confirmedLedgerGeneration));};
|
||||
try {
|
||||
while (pending.size) {
|
||||
const [revision, snapshot] = pending.entries().next().value!;
|
||||
if(revision>throughRevision)break;
|
||||
const sourceFiles=importFiles(snapshot.runtime.imports);
|
||||
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||sourceFiles.has(id)).map(([id, bytes]) => ({ id, bytes }));
|
||||
const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id),...[...(snapshot.runtime.orphanFiles??[])].filter(([id,file])=>file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)];
|
||||
const depot=options.assets?.authority&&options.storage.assetDepot?await prepareDepotCut({snapshot,registry:options.registry,storage:options.storage,leader,authority:options.assets.authority.scope,files:fileWrites,adoptedFiles,displays:displayPins,temporary:admissionPins(snapshot)}):undefined;
|
||||
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles,...(depot?{depot}:{}) });
|
||||
if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt');
|
||||
confirmedLedgerGeneration=Math.max(confirmedLedgerGeneration,snapshot.runtime.ledger.generation);
|
||||
durableRootRevision = revision;durableImportSeals=importSeals(snapshot.runtime.imports); pending.delete(revision);notifyDurability();
|
||||
for(const id of deleteFiles){durablyPurged.add(id);collectionReservations.delete(id);}
|
||||
for (const file of files) fileWrites.delete(file.id);
|
||||
for(const [id,entry]of adoptedFiles)if(entry.rootRevision<=snapshot.rootRevision&&depot?.writes.some(write=>write.ref.asset.id===id&&write.bytes===entry.bytes))adoptedFiles.delete(id);
|
||||
}
|
||||
} catch(error) {
|
||||
// Preserve the known durable prefix even if a later write fails.
|
||||
publishDurability();throw error;
|
||||
}
|
||||
// One derived publication per captured prefix, not one per receipt.
|
||||
publishDurability();
|
||||
persistenceFailed = false;durabilityError=undefined;notifyDurability();
|
||||
})();
|
||||
});
|
||||
writing = task.catch(error => { persistenceFailed = true;const failure=error&&typeof error==='object'?error as {code?:unknown;message?:unknown}:{};durabilityError=typeof failure.code==='string'&&failure.code?failure.code:typeof failure.message==='string'&&failure.message?failure.message:'persistence_failed';notifyDurability(); throw error; }).finally(() => { writing = undefined;if(!persistenceFailed){if(loadingCompletionRevision!==undefined&&durableRootRevision>=loadingCompletionRevision&&!editor.getSnapshot().runtime.loading){const release=loadingRelease;loadingRelease=undefined;loadingCompletionRevision=undefined;release?.();}if(!closing&&!closed)scheduler.refreshAdmission();} });
|
||||
return writing;
|
||||
}
|
||||
function persistThrough(requiredRevision=editor.getSnapshot().rootRevision):Promise<void> {
|
||||
if(durableRootRevision>=requiredRevision)return Promise.resolve();
|
||||
return writePrefix(requiredRevision).then(()=>durableRootRevision<requiredRevision?persistThrough(requiredRevision):undefined);
|
||||
}
|
||||
async function persist():Promise<void> {
|
||||
do {await writePrefix(editor.getSnapshot().rootRevision);}
|
||||
while(pending.size||writing||confirmedLedgerGeneration>editor.getSnapshot().runtime.ledger.durableGeneration);
|
||||
}
|
||||
function collectResources(settings:{retentionMs:number;maxFiles?:number}) {
|
||||
assertWritable();
|
||||
if(options.storage.fileDeletion!=='atomic-v1')throw fault('resource_collection_unavailable');
|
||||
@@ -442,7 +461,7 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
report(error); return { blocked: true };
|
||||
}
|
||||
}
|
||||
const scheduler = createScheduler({ admit:intent=>{const root=editor.getSnapshot();if(!importsMaySend(root.runtime.imports,durableImportSeals))return false;if(root.runtime.loading&&root.runtime.loading.version<3)return true;if(closing||closed||loadingCompletionRevision!==undefined&&durableRootRevision<loadingCompletionRevision)return false;return (root.runtime.loading?.version===6&&intent.operationType==='asset.upload'||loadingCanSend(root.runtime,operationCanvases(intent,root.document),root.document))&&(!root.runtime.loading||durableRootRevision>=root.rootRevision);},getLedger: () => editor.getSnapshot().runtime.ledger, mutate, persist,
|
||||
const scheduler = createScheduler({ admit:intent=>{const root=editor.getSnapshot();if(!importsMaySend(root.runtime.imports,durableImportSeals))return false;if(root.runtime.loading&&root.runtime.loading.version<3)return true;if(closing||closed||loadingCompletionRevision!==undefined&&durableRootRevision<loadingCompletionRevision)return false;return (root.runtime.loading?.version===6&&intent.operationType==='asset.upload'||loadingCanSend(root.runtime,operationCanvases(intent,root.document),root.document))&&(!root.runtime.loading||durableRootRevision>=root.rootRevision);},getLedger: () => editor.getSnapshot().runtime.ledger, mutate, persist:persistThrough,
|
||||
createId: options.createId ?? (() => crypto.randomUUID()),
|
||||
participants: intent => { if(intent.historyEffect)validateCreationSizeEffect(intent.historyEffect);return operationCanvases(intent, editor.getSnapshot().document); },
|
||||
compile(intent, id) {
|
||||
@@ -527,7 +546,7 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
if(code!=='run_waiting_for_receipts')report(error);
|
||||
}
|
||||
}
|
||||
const runTracker=createRunTracker({clock:runClock,getRuns:()=>editor.getSnapshot().runtime.runs,persist,update:updateRun,accept:acceptRunPage,
|
||||
const runTracker=createRunTracker({clock:runClock,getRuns:()=>editor.getSnapshot().runtime.runs,persist:persistThrough,update:updateRun,accept:acceptRunPage,
|
||||
poll:(run,signal)=>options.runs ? network.run(signal,()=>options.runs!.poll(run,signal)) : Promise.reject(fault('run_interpreter_unavailable')),onError:report});
|
||||
const trackedRuns=new Set(initial.runs.keys());
|
||||
const sessions = createSessions(editor);
|
||||
@@ -1051,7 +1070,14 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
getDurability,
|
||||
subscribeDurability(listener:()=>void){if(closed)return()=>{};durabilityListeners.add(listener);return()=>{durabilityListeners.delete(listener);};},
|
||||
retry: async () => { if(recovery)await recovery.cancel();assertWritable(false);await persist();retryDefaultSizes();const runtime=editor.getSnapshot().runtime,ids=[...runtime.resources.values()].filter(resource=>!resource.digest&&runtime.ledger.operations.get(resource.operationId)?.blockers.some(code=>digestFailures.has(code))).map(resource=>resource.operationId);if(ids.length){mutate(ledger=>{for(const id of ids)ledger.block(id,['waiting_digest']);});prepareDigests();}if(options.images){editor.updateRuntime(previous=>{const resources=new Map(previous.resources);let changed=false;for(const [id,resource]of resources)if(resource.image?.version===1&&resource.image.status==='failed'){resources.set(id,freeze({...resource,image:{version:1 as const,status:'pending' as const}}));changed=true;}return changed?Object.freeze({...previous,resources:readMap(resources)}):previous;});prepareImages();}for(const job of editor.getSnapshot().runtime.imports?.values()??[])if(sourceNeedsSeal(job)){await importLoads.get(job.id);await resetImportReader(job.id);await loadImport(job.id);}await scheduler.resume();await runTracker.retry(); },
|
||||
async settle() { await Promise.resolve();await Promise.allSettled([...adoptionTasks.values()]);await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while(verificationTasks.size)await Promise.allSettled([...verificationTasks.values()].map(task=>task.promise));while (digestTasks.size) await Promise.all([...digestTasks.values()]);while(imageTasks.size)await Promise.all([...imageTasks.values()]);while(sizingTask)await sizingTask; await scheduler.settle(); await runTracker.settle(); await persist();await scheduler.settle();await persist(); },
|
||||
async settle() {
|
||||
let revision:number;
|
||||
do {
|
||||
revision=editor.getSnapshot().rootRevision;
|
||||
await Promise.resolve();await Promise.allSettled([...adoptionTasks.values()]);await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle()));while(verificationTasks.size)await Promise.allSettled([...verificationTasks.values()].map(task=>task.promise));while(digestTasks.size)await Promise.all([...digestTasks.values()]);while(imageTasks.size)await Promise.all([...imageTasks.values()]);while(sizingTask)await sizingTask;
|
||||
await scheduler.settle();await runTracker.settle();await persist();await scheduler.settle();await persist();
|
||||
} while(revision!==editor.getSnapshot().rootRevision||pending.size||writing||adoptionTasks.size||copyPreparations.size||importLoads.size||verificationTasks.size||digestTasks.size||imageTasks.size||sizingTask);
|
||||
},
|
||||
async close() {
|
||||
if (closed) return;
|
||||
closing = true;notifyDurability();loadingReadController?.abort();await loadingRead?.catch(()=>{});unsubscribeSizing();orphanController?.abort();await Promise.allSettled([collection,sourceCollection,orphanCollection].filter(Boolean));digestLifecycle.abort();for(const {controller}of verificationTasks.values())controller.abort();for(const controller of preparationControllers.values())controller.abort();for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]);for(const controller of adoptionTasks.keys())controller.abort();await Promise.allSettled([...adoptionTasks.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
|
||||
|
||||
@@ -2,7 +2,7 @@ import { fault } from '../canvas-core/values.ts';
|
||||
import { createDeque } from './structures.ts';
|
||||
|
||||
/** Shared bounded transport capacity for writes, uploads and run reads. */
|
||||
export function createNetworkGate(limit = 2) {
|
||||
export function createNetworkGate(limit = 2, beforeRun?:()=>void) {
|
||||
if(!Number.isSafeInteger(limit)||limit<1)throw fault('invalid_concurrency');
|
||||
let active = 0;
|
||||
const waiting = createDeque<{ signal: AbortSignal; grant(): void; cancelled: boolean }>();
|
||||
@@ -21,6 +21,6 @@ export function createNetworkGate(limit = 2) {
|
||||
}
|
||||
return Object.freeze({ async run<T>(signal: AbortSignal, callback: () => Promise<T>): Promise<T> {
|
||||
await enter(signal);
|
||||
try { signal.throwIfAborted();return await callback(); } finally { release(); }
|
||||
try { signal.throwIfAborted();beforeRun?.();return await callback(); } finally { release(); }
|
||||
} });
|
||||
}
|
||||
|
||||
@@ -210,6 +210,10 @@ export function createScheduler(options: SchedulerOptions) {
|
||||
own(options.getLedger().requests.get(requestId)!); launch(requestId, 'submit');
|
||||
} catch (error) { options.mutate(ledger => ledger.block(id, [(error as Error).message || 'compile_failed'])); report(error); }
|
||||
}
|
||||
} catch(error) {
|
||||
// A retired identity can reject both the attempt and its blocked-state write.
|
||||
// Report the failure without rejecting an unobserved scheduled pump promise.
|
||||
report(error);
|
||||
} finally {
|
||||
pumping = false; armTimer();
|
||||
if (!stopped && !holds.size && ready.size && active.size < concurrency && pumpTimer === undefined) pumpTimer = clock.setTimer(() => { pumpTimer = undefined; pump(); }, 0);
|
||||
|
||||
Reference in New Issue
Block a user