Allow import feedback to probe larger batches within the existing time budget

This commit is contained in:
2026-11-10 18:25:09 +08:00
parent 2452fd6ae9
commit b051509dee
3 changed files with 19 additions and 1 deletions
+10
View File
@@ -947,3 +947,13 @@ validation step remain indivisible. Copy cancellation still reports
`copy_aborted`; host clock/yield errors remain observable. Snapshot formats,
byte locators, size limits, source digests, lazy payload reads and admission/seal
ordering are unchanged. No partially encoded snapshot is admitted or returned.
ProjectEngine allows the existing import feedback controller to probe batches up
to 128 ordinary units; the low-level runner's default remains 32 unless the host overrides
it. The ceiling is not a fixed batch size or a higher time budget. Initial batches
start at one; measured commits still grow/back off against the same 6ms soft
budget. Parser time checks, the pending/decoded byte limits, SCC isolation,
resource authorization limits, durable send barriers and source ordering remain
in force. Hosts that set a lower runner ceiling retain that ordinary-batch bound. An
indivisible SCC can exceed the soft unit limit under its existing hard byte and
change-count limits.
+2 -1
View File
@@ -769,7 +769,8 @@ export async function openProjectEngine(options: EngineOptions) {
if(closing||closed||preparationController.signal.aborted||getImport(id).generation!==job.generation)return;
if(importHolds.size){deferredImportLoads.add(id);return;}
const authorizedImportAssets=new Map<Id,{ref:SharedAsset;bytes:Blob}>(),assetPreparationSignals=new WeakSet<AbortSignal>();
const runner=createImportRunner({optimisticLocal:true,checkpointSource:source.integrityVersion===2?checkpointImportSource:undefined,awaitDurabilityBudget:stage=>pending.size>=32||stage!=='parsed'&&persistenceFailed?persist():undefined,source:importSource,getJob:()=>getImport(id),createId,
// Ordinary-batch ceiling only; commit timing and indivisible SCC limits still apply.
const runner=createImportRunner({maxBatch:128,optimisticLocal:true,checkpointSource:source.integrityVersion===2?checkpointImportSource:undefined,awaitDurabilityBudget:stage=>pending.size>=32||stage!=='parsed'&&persistenceFailed?persist():undefined,source:importSource,getJob:()=>getImport(id),createId,
publish:publishImport,prepare:(line,job,reserve)=>['duplicate-ndjson','duplicate-plan'].includes(job.source?.kind??'')?prepareDuplicateImport(line,job,reserve,options.registry):prepareNativeImport(line,job,reserve,editor.getSnapshot().document,options.registry,depot&&options.assets?.authority?options.assets.sourceURL:undefined),persist,onError:report,
async beforeCommit(units,job,signal){
authorizedImportAssets.clear();if(job.source?.kind!=='native-document')return{enabled:true};
+7
View File
@@ -155,3 +155,10 @@ test('identity allocation keeps the caller port method receiver',async()=>{
test('fixed-cost commits amortize ready records while preserving the immediate first record and yielding after expensive work',async()=>{
let clock=0;const f=fixture(Array.from({length:80},(_,i)=>({id:`fixed-${i}`})),{ports:{now:()=>clock,frameBudgetMs:6,maxBatch:16,optimisticLocal:true}}),commit=f.ports.commit;f.ports.commit=(units,...rest)=>{clock+=4+units.length*.05;return commit(units,...rest);};await f.runner.start();assert.equal(f.root.job.status,'completed');assert.deepEqual(f.calls[0],['fixed-0']);assert.ok(f.calls.some(group=>group.length>=8));assert.ok(f.calls.length<20);assert.ok(f.yields>0);assert.equal(f.errors.length,0);validateImportJob(f.root.job);
});
test('a cheap runner can use the engine-sized ceiling without dropping records or changing FIFO order',async()=>{
const records=Array.from({length:700},(_,index)=>({id:`wide-${index}`})),f=fixture(records,{ports:{maxBatch:128,now:()=>0}});await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.document.elements.size,700);assert.ok(f.calls.some(batch=>batch.length>32));assert.ok(f.calls.every(batch=>batch.length<=128));assert.deepEqual(f.calls.flat(),records.map(row=>row.id));assert.equal(f.errors.length,0);await f.runner.close();
});
test('the larger ceiling still backs off when a measured commit exceeds the unchanged time budget',async()=>{
let clock=0;const records=Array.from({length:500},(_,index)=>({id:`cost-${index}`})),f=fixture(records,{ports:{maxBatch:128,now:()=>clock}}),commit=f.ports.commit;f.ports.commit=(units,...args)=>{clock+=units.length>40?12:1;return commit(units,...args);};await f.runner.start();const sizes=f.calls.map(batch=>batch.length),wide=sizes.findIndex(size=>size>40);assert.ok(wide>=0);assert.ok(sizes[wide+1]<sizes[wide]);assert.equal(f.root.job.status,'completed');assert.equal(f.root.document.elements.size,500);assert.deepEqual(f.calls.flat(),records.map(row=>row.id));assert.equal(f.errors.length,0);await f.runner.close();
});