Bound optimistic imports with persistence backpressure
This commit is contained in:
@@ -99,7 +99,7 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
});
|
||||
const pending = new Map<number, EngineSnapshot>();
|
||||
if (!recovered) pending.set(editor.getSnapshot().rootRevision, editor.getSnapshot());
|
||||
let writing: Promise<void> | undefined, scheduled = false;
|
||||
let writing: Promise<void> | undefined, scheduled = false, persistenceFailed = false;
|
||||
function persist(): Promise<void> {
|
||||
if (writing) return writing;
|
||||
const task = (async () => {
|
||||
@@ -112,8 +112,9 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
for (const file of files) fileWrites.delete(file.id);
|
||||
if (editor.getSnapshot().runtime.ledger.durableGeneration < snapshot.runtime.ledger.generation) mutate(ledger => ledger.markDurable(snapshot.runtime.ledger.generation));
|
||||
}
|
||||
persistenceFailed = false;
|
||||
})();
|
||||
writing = task.finally(() => { writing = undefined; });
|
||||
writing = task.catch(error => { persistenceFailed = true; throw error; }).finally(() => { writing = undefined; });
|
||||
return writing;
|
||||
}
|
||||
function mutate(reduce: (ledger: ReturnType<typeof createLedger>) => void) {
|
||||
@@ -354,7 +355,7 @@ export async function openProjectEngine(options: EngineOptions) {
|
||||
const latest=getImport(id);if(latest.sourceFingerprint!==job.sourceFingerprint||latest.source?.fileId!==source.fileId||['completed','partial','cancelled'].includes(latest.status))return;
|
||||
if(!bytes)throw fault('import_source_unavailable');if(bytes.size!==source.byteLength)throw fault('import_source_changed');
|
||||
if(source.kind==='duplicate-snapshot'){if(preparationController.signal.aborted||importHolds.size||latest.status!=='preparing')return;const materialized=await materializeCopy(id,bytes,preparationController);job=materialized.job;source=job.source!;bytes=materialized.bytes;expectedFingerprint=job.sourceFingerprint;}
|
||||
const runner=createImportRunner({optimisticLocal:true,source:createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint}),getJob:()=>getImport(id),createId,
|
||||
const runner=createImportRunner({optimisticLocal:true,awaitDurabilityBudget:()=>persistenceFailed||pending.size>=32?persist():undefined,source:createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint}),getJob:()=>getImport(id),createId,
|
||||
publish:publishImport,prepare:(line,job,reserve)=>job.source?.kind==='duplicate-ndjson'?prepareDuplicateImport(line,job,reserve,options.registry):prepareNativeImport(line,job,reserve,editor.getSnapshot().document,options.registry),persist,onError:report,
|
||||
commit(units,next,expected){
|
||||
assertWritable();if(importing)throw fault('reentrant_dispatch');const current=getImport(id);if(current.generation!==expected)throw fault('stale_import_job');
|
||||
|
||||
@@ -86,6 +86,8 @@ export type ImportRunnerPorts={
|
||||
persist():Promise<void>;onError(error:unknown):void;
|
||||
/** Host must durably gate every network write on the same document/job cut. */
|
||||
optimisticLocal?:boolean;
|
||||
/** Host backpressure for speculative local cuts. Network remains durably gated by persist(). */
|
||||
awaitDurabilityBudget?():void|Promise<void>;
|
||||
now?:()=>number;yieldTask?:()=>Promise<void>;frameBudgetMs?:number;maxPending?:number;maxPendingBytes?:number;maxBatch?:number;
|
||||
};
|
||||
|
||||
@@ -110,6 +112,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
|
||||
if(new TextEncoder().encode(JSON.stringify(unit)).byteLength>maxBytes)throw fault('import_unit_capacity');
|
||||
return{unit:freeze(clone(unit)),ids:{...ids}};
|
||||
}
|
||||
const checkpoint=async()=>{if(ports.optimisticLocal&&ports.awaitDurabilityBudget)await ports.awaitDurabilityBudget();else await ports.persist();};
|
||||
const digest=async(unit:ImportUnit)=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(unit))))].map(value=>value.toString(16).padStart(2,'0')).join('');
|
||||
function completed(job:ImportJob,units:readonly ImportUnit[],pending:readonly ImportLocator[]){
|
||||
const bindings:Record<Id,ImportBinding>=Object.assign(Object.create(null),job.bindings??{});
|
||||
@@ -166,15 +169,17 @@ export function createImportRunner(ports:ImportRunnerPorts){
|
||||
proposed=completed(job,members,job.pending.filter(value=>!internal.has(value.key)));
|
||||
result=commit(members,job,proposed);
|
||||
if(result.status==='rejected'){const code=result.code;publish(job,{pending:job.pending.filter(value=>!internal.has(value.key)),failures:[...job.failures,...group.map(row=>({key:row.key,cursor:row.cursor,code}))]});}
|
||||
await ports.persist();if(closed||signal.aborted||current().status!=='running')return;
|
||||
await checkpoint();if(closed||signal.aborted||current().status!=='running')return;
|
||||
if(now()-frame>=budget){await(ports.yieldTask??yieldTask)();frame=now();}
|
||||
}
|
||||
}else await ports.persist();
|
||||
}else await checkpoint();
|
||||
for(const unit of units)forget(unit.key);
|
||||
continue;
|
||||
}
|
||||
if(job.eof){
|
||||
const unresolved=job.pending.map(unit=>({key:unit.key,cursor:unit.cursor,code:'unresolved_dependency',dependencies:unit.dependencies.filter(key=>!committed.has(key))}));
|
||||
// Completion must never overtake the final document/source durability barrier.
|
||||
await ports.persist();if(closed||signal.aborted||current().generation!==job.generation)return;
|
||||
const unresolved=job.pending.filter(unit=>!job.failures.some(failure=>failure.key===unit.key&&failure.code==='unresolved_dependency')).map(unit=>({key:unit.key,cursor:unit.cursor,code:'unresolved_dependency',dependencies:unit.dependencies.filter(key=>!committed.has(key))}));
|
||||
publish(job,{status:job.failures.length||unresolved.length?'partial':'completed',failures:[...job.failures,...unresolved]});await ports.persist();return;
|
||||
}
|
||||
if(job.pending.length>=maxPending||new TextEncoder().encode(JSON.stringify(job.pending)).byteLength>=maxBytes){publish(job,{status:'paused',error:'dependency_capacity'});await ports.persist();return;}
|
||||
@@ -208,7 +213,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
|
||||
// First ready record displays immediately; subsequent slices amortize publication.
|
||||
if(costPerUnit===undefined&&staged.pending.some(unit=>unit.dependencies.every(key=>committed.has(key)))||now()-frame>=budget)break;
|
||||
}
|
||||
if(staged!==base){const{generation,...patch}=staged;publish(base,patch);if(!ports.optimisticLocal||!staged.pending.some(unit=>unit.dependencies.every(key=>committed.has(key))))await ports.persist();}
|
||||
if(staged!==base){const{generation,...patch}=staged;publish(base,patch);if(!ports.optimisticLocal||!staged.pending.some(unit=>unit.dependencies.every(key=>committed.has(key))))await checkpoint();}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -217,7 +222,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
|
||||
restart=false;controller=new AbortController();
|
||||
active=pump(controller.signal).catch(async error=>{
|
||||
if(closed||controller?.signal.aborted)return;
|
||||
try{const job=current();if(job.status==='running'){publish(job,{status:'paused',error:(error as {code?:string}).code??'import_failed'});try{await ports.persist();}catch{/* Storage may still be unavailable. */}}}catch{/* A replaced identity cannot receive an old error. */}
|
||||
try{const job=current();if(['running','completed','partial'].includes(job.status)){publish(job,{status:'paused',error:(error as {code?:string}).code??'import_failed'});try{await ports.persist();}catch{/* Storage may still be unavailable. */}}}catch{/* A replaced identity cannot receive an old error. */}
|
||||
report(error);
|
||||
}).finally(()=>{active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}});
|
||||
return active;
|
||||
|
||||
Reference in New Issue
Block a user