Add quiescent canonical recovery sessions without rewriting requests

tests/headless/browser-storage-cases.mjs

@@ -5,6 +5,29 @@ import { createFoundryRuns } from '../../src/lib/canvas-service/runs.ts';
 import { createFoundryAssets } from '../../src/lib/canvas-service/assets.ts';
 import { createFoundryPersistence } from '../../src/lib/canvas-service/foundry.ts';
 
+async function canonicalRefreshCase(name,assert){
+  const root=crypto.randomUUID(),workspace=crypto.randomUUID(),account=crypto.randomUUID(),registry=createRegistry(builtinComponents());
+  const target={id:'a',incarnation:1},element={...target,structure:'node',componentId:'text',dataVersion:1,ownerCanvasId:root,geometry:{x:0,y:0,width:300,height:200,rotation:0},data:{title:'a',text:'original'}};
+  const original=createDocument(root,[{id:root,parent:null}],[element]),storage=createBrowserStorage({name:`${name}-canonical-refresh`}),calls=[];let server=original,version=1,engine;
+  const model=complete=>({documentFormatVersion:1,projectId:root,complete,removedCanvasIds:[],canvases:[...server.canvases.values()],elements:[...server.elements.values()],tombstones:{},versions:{[root]:version}});
+  const transport=async request=>{calls.push(request);if(request.method==='GET')return{status:200,body:{model:model(true)}};const body=JSON.parse(request.body);for(const operation of body.operations)server=applyChanges(server,operation.changes);version++;return{status:200,body:{protocol_version:2,request_id:request.requestId,status:'applied',operation_ids:body.operations.map(operation=>operation.id),model:model(false)}};};
+  const options={scope:{accountId:account,workspaceId:workspace,projectId:root},registry,document:original,versions:new Map([[root,1]]),storage,persistence:createFoundryPersistence({workspaceId:workspace,rootId:root,registry,transport})};
+  try{
+    engine=await openProjectEngine(options);engine.editor.dispatch({type:'element.data',target,patch:{title:'local'}});await engine.settle();
+    const historyId=engine.getSnapshot().history.undo[0],request=[...engine.getSnapshot().runtime.ledger.requests.values()][0];
+    server=applyChanges(server,[{kind:'field',target,field:'geometry',before:element.geometry,after:{...element.geometry,x:400}}]);version++;
+    const session=await engine.prepareRecovery();assert(session.conflict===undefined,'clean canonical refresh reported a conflict');
+    assert(engine.getSnapshot().document.elements.get('a').geometry.x===0,'preview mutated the working document');
+    assert(engine.editor.history.undo().code==='recovery_in_progress','history bypassed the recovery fence');
+    assert((await session.apply()).status==='applied','fresh canonical state did not publish');await engine.close();
+    engine=await openProjectEngine(options);const restored=engine.getSnapshot();
+    assert(restored.document.elements.get('a').geometry.x===400&&restored.document.elements.get('a').data.title==='local','canonical refresh did not survive IndexedDB recovery');
+    assert(restored.runtime.versions.get(root)===3&&restored.history.undo[0]===historyId,'refresh lost its version/history cut');
+    assert(restored.runtime.ledger.requests.size===1&&restored.runtime.ledger.requests.get(request.id).body===request.body,'refresh changed an original request');
+    assert(calls.filter(call=>call.method==='POST').length===1,'refresh or reopen resent an applied write');
+  }finally{await engine?.close();storage.close();indexedDB.deleteDatabase(`${name}-canonical-refresh`);}
+}
+
 async function runRecoveryCase(name,assert){
   const root=crypto.randomUUID(),workspace=crypto.randomUUID(),account=crypto.randomUUID(),runId=crypto.randomUUID();
   const registry=createRegistry(builtinComponents()),element={id:'a',incarnation:1,structure:'node',componentId:'text',dataVersion:1,ownerCanvasId:root,geometry:{x:0,y:0,width:300,height:200,rotation:0},data:{title:'a',text:'original'}},original=createDocument(root,[{id:root,parent:null}],[element]);
@@ -205,6 +228,8 @@ export async function run() {
     results.push('protocol 2 service adapter recovers exact frozen bytes and reconciles a lost ACK through real IndexedDB');
     await runRecoveryCase(name,assert);
     results.push('AI admission and RunRecord recover through real IndexedDB; cursor and generated content share a durable cut without a second invocation');
+    await canonicalRefreshCase(name,assert);
+    results.push('quiescent canonical refresh preserves original requests and history; the new base/document/version cut survives IndexedDB reopening');
     return results;
   } finally { await leader?.release(); other?.close(); storage.close(); indexedDB.deleteDatabase(name); }
 }

tests/headless/engine.test.mjs

@@ -140,3 +140,67 @@ test('a fresh canonical read replaces a clean recovered base, keeps history and
   assert.equal(reopened.engine.getSnapshot().document.elements.get('a').geometry.x, 400);
   await reopened.engine.close();
 });
+
+test('recovery waits for admitted writes, reads a complete base, fences edits and publishes without changing history', async()=>{
+  const storage=storageFixture(),port=persistenceFixture(),response=deferred(),realSubmit=port.submit,readGate=deferred();
+  let reads=0;port.submit=async request=>{await response.promise;return realSubmit(request);};
+  port.read=async()=>{reads++;await readGate.promise;return{document:applyChanges(original,[{kind:'field',target,field:'data.title',before:'a',after:'local',beforePresent:true,afterPresent:true},{kind:'field',target,field:'geometry',before:original.elements.get('a').geometry,after:{...original.elements.get('a').geometry,x:300}}]),versions:new Map([['canvas',3]])};};
+  const{engine}=await open(storage,port);engine.editor.dispatch({type:'element.data',target,patch:{title:'local'}});await tick();
+  const preparing=engine.prepareRecovery();await tick();assert.equal(reads,0);
+  assert.equal(engine.editor.dispatch({type:'element.data',target,patch:{title:'blocked'}}).code,'recovery_in_progress');
+  assert.equal(engine.startRun({}).code,'recovery_in_progress');assert.equal(engine.refresh(original,new Map([['canvas',1]])).code,'recovery_in_progress');
+  await assert.rejects(engine.prepareRecovery(),/recovery_in_progress/);
+  response.resolve();await tick();assert.equal(reads,1);const history=engine.getSnapshot().history;
+  readGate.resolve();const session=await preparing;assert.equal(session.conflict,undefined);assert.equal(session.canonical.document.elements.get('a').geometry.x,300);
+  assert.equal(engine.getSnapshot().document.elements.get('a').geometry.x,0);assert.equal(engine.editor.history.undo().code,'recovery_in_progress');
+  assert.equal((await session.apply()).status,'applied');assert.equal(engine.getSnapshot().document.elements.get('a').geometry.x,300);assert.equal(engine.getSnapshot().history,history);
+  assert.equal(engine.getDurability().durableRootRevision,engine.getSnapshot().rootRevision);await assert.rejects(session.apply(),/recovery_session_closed/);
+  await engine.close();
+});
+
+test('unknown receipts prevent a recovery read and preserve the frozen request identity',async()=>{
+  const storage=storageFixture();let reads=0;const port=persistenceFixture({async submit(){return{outcome:'unresolved'};},async read(){reads++;return{document:original,versions:new Map([['canvas',1]])};}});
+  const{engine}=await open(storage,port);engine.editor.dispatch({type:'element.data',target,patch:{title:'local'}});await engine.settle();
+  const before=engine.getSnapshot(),request=[...before.runtime.ledger.requests.values()][0];
+  await assert.rejects(engine.prepareRecovery(),/recovery_waiting_for_receipts/);assert.equal(reads,0);
+  const after=engine.getSnapshot();assert.equal(after.document,before.document);assert.equal(after.history,before.history);assert.equal(after.runtime.ledger.requests.get(request.id).body,request.body);
+  assert.equal(engine.editor.can({type:'element.data',target,patch:{title:'allowed'}}).enabled,true);await engine.close();
+});
+
+test('recovery conflict previews preserve local data/history and cancel releases the edit fence',async()=>{
+  const remote=applyChanges(original,[{kind:'field',target,field:'data.title',before:'a',after:'remote',beforePresent:true,afterPresent:true}]);
+  const storage=storageFixture(),port=persistenceFixture({async submit(){return{outcome:'not-applied',error:'version_conflict'};},async read(){return{document:remote,versions:new Map([['canvas',2]])};}});
+  const{engine}=await open(storage,port);engine.editor.dispatch({type:'element.data',target,patch:{title:'local'}});await engine.settle();
+  const before=engine.getSnapshot(),session=await engine.prepareRecovery();assert.ok(session.conflict);assert.equal(session.pendingOperationIds.length,1);
+  assert.equal((await session.apply()).status,'rejected');assert.equal(engine.getSnapshot(),before);assert.equal(engine.getSnapshot().document.elements.get('a').data.title,'local');
+  await session.cancel();await session.cancel();assert.equal(engine.editor.can({type:'element.data',target,patch:{title:'more'}}).enabled,true);await engine.close();
+});
+
+test('failed recovery durability retains its fence and retries the same published root',async()=>{
+  const storage=storageFixture(),port=persistenceFixture({async read(){return{document:original,versions:new Map([['canvas',1]])};}}),{engine}=await open(storage,port);
+  const session=await engine.prepareRecovery();storage.broken=true;await assert.rejects(session.apply(),/quota/);const revision=engine.getSnapshot().rootRevision;
+  assert.equal(engine.editor.can({type:'element.data',target,patch:{title:'blocked'}}).code,'recovery_in_progress');await assert.rejects(session.cancel(),/quota/);
+  storage.broken=false;assert.equal((await session.apply()).status,'applied');assert.equal(engine.getSnapshot().rootRevision,revision);assert.equal(engine.getDurability().durableRootRevision,revision);
+  await engine.close();
+});
+
+test('closing the engine aborts a recovery read and rejects a late canonical response',async()=>{
+  const storage=storageFixture(),response=deferred();let signal;
+  const port=persistenceFixture({async read(input){signal=input;return response.promise;}}),{engine}=await open(storage,port);
+  const preparing=engine.prepareRecovery();await tick();assert.ok(signal);await engine.close();assert.equal(signal.aborted,true);
+  response.resolve({document:original,versions:new Map([['canvas',1]])});await assert.rejects(preparing,/engine_closing/);assert.equal(storage.owner(),null);
+});
+
+test('complete recovery reads reject same-version rewrites and lifecycle regression before creating a session',async()=>{
+  const remote=applyChanges(original,[{kind:'field',target,field:'data.title',before:'a',after:'remote',beforePresent:true,afterPresent:true}]);
+  const storage=storageFixture(),port=persistenceFixture({async read(){return{document:remote,versions:new Map([['canvas',1]])};}}),{engine}=await open(storage,port);
+  await assert.rejects(engine.prepareRecovery(),/canonical_version_mismatch/);
+  assert.equal(engine.getSnapshot().document.elements.get('a').data.title,'a');
+  assert.equal(engine.editor.can({type:'element.data',target,patch:{title:'allowed'}}).enabled,true);
+  assert.equal(engine.refresh(remote,new Map([['canvas',1]])).code,'canonical_version_mismatch');
+  const {validateCanonicalRead}=await import('../../src/lib/canvas-runtime/reconcile.ts');
+  const withTombstone=createDocument(original.projectId,[...original.canvases.values()],[...original.elements.values()],new Map([['deleted',2]]));
+  assert.throws(()=>validateCanonicalRead(withTombstone,new Map([['canvas',1]]),original,new Map([['canvas',2]]),registry),/stale_canonical_lifecycle/);
+  assert.throws(()=>validateCanonicalRead(original,new Map([['canvas',1]]),withTombstone,new Map([['canvas',1]]),registry),/canonical_version_mismatch/);
+  await engine.close();
+});

tests/headless/run-tracker.test.mjs

@@ -0,0 +1,27 @@
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { createRunTracker } from '../../src/lib/canvas-runtime/runs.ts';
+
+const tick = async () => { for (let index=0;index<30;index++)await Promise.resolve(); };
+const deferred = () => { let resolve;const promise=new Promise(yes=>{resolve=yes;});return{promise,resolve}; };
+function fixture() {
+  let now=0,next=0;const timers=new Map(),runs=new Map([['run',{id:'run',status:'running',cursor:0,hasMore:true,attempt:0}]]),calls=[],saved=[],response=deferred();
+  const tracker=createRunTracker({clock:{now:()=>now,setTimer(callback,delay){const id=++next;timers.set(id,{callback,due:now+delay});return id;},clearTimer:id=>timers.delete(id)},getRuns:()=>runs,async persist(){saved.push(structuredClone([...runs]));},async poll(run){calls.push(run);return calls.length===1?response.promise:{status:'succeeded',events:[],next:2,hasMore:false,outputs:[],steps:[]};},accept(run,page){runs.set(run.id,{...runs.get(run.id),status:page.status,cursor:page.next,hasMore:page.hasMore});},update(id,reduce){runs.set(id,reduce(runs.get(id)));},onError(error){throw error;}});
+  return{tracker,runs,calls,saved,response,timers,async advance(ms){now+=ms;for(const[id,timer]of[...timers])if(timer.due<=now){timers.delete(id);timer.callback();}await tick();}};
+}
+
+test('run holds preserve a completed cursor and notifications without launching another poll',async()=>{
+  const f=fixture();f.tracker.notify(['run']);await f.advance(0);assert.equal(f.calls.length,1);
+  const releaseA=f.tracker.hold(),releaseB=f.tracker.hold();
+  f.response.resolve({status:'running',events:[],next:1,hasMore:true,outputs:[],steps:[]});await f.tracker.settle();
+  assert.equal(f.runs.get('run').cursor,1);assert.equal(f.saved.at(-1)[0][1].cursor,1);assert.equal(f.timers.size,0);
+  await f.advance(10000);await f.tracker.retry();assert.equal(f.calls.length,1);assert.equal(f.timers.size,0);
+  releaseA();releaseA();await f.advance(0);assert.equal(f.calls.length,1);
+  releaseB();await f.advance(0);await f.tracker.settle();assert.equal(f.calls.length,2);assert.equal(f.calls[1].cursor,1);assert.equal(f.runs.get('run').cursor,2);
+  f.tracker.stop();
+});
+
+test('releasing a run hold after stop cannot restart a poll',async()=>{
+  const f=fixture(),release=f.tracker.hold();f.tracker.notify(['run']);f.tracker.stop();release();await f.advance(10000);
+  assert.equal(f.calls.length,0);assert.equal(f.timers.size,0);assert.throws(()=>f.tracker.hold(),/run_tracker_closed/);
+});

tests/headless/scheduler.test.mjs

@@ -143,3 +143,26 @@ test('a capacity-blocked upload retry does not spin or hide a later document ret
   gate.resolve(); await state.scheduler.settle();
   assert.ok(calls.some(([action, id]) => action === 'query' && id === 'asset-1')); state.scheduler.stop();
 });
+
+test('nested holds finish admitted receipts but suppress dependent dispatch and due retries until the final release', async () => {
+  const response = deferred(), calls = [];
+  const state = setup({ concurrency: 1, submit: async request => { calls.push(['submit', request]); return request.operationIds[0] === 'first' ? response.promise : { outcome: 'applied' }; }, query: async request => { calls.push(['query', request]); return { outcome: 'applied' }; } });
+  state.add('first'); state.add('dependent', ['canvas'], ['first']); state.scheduler.start(); await tick();
+  const releaseA = state.scheduler.hold(), releaseB = state.scheduler.hold();
+  state.add('later', ['other']); response.resolve({ outcome: 'unresolved' }); await state.scheduler.settle();
+  const frozen = state.ledger().getSnapshot().requests.get('request-1');
+  assert.equal(frozen.outcome, 'unresolved'); assert.equal(state.scheduler.getClaims().get('canvas'), frozen.id);
+  await state.advance(1000); assert.equal(calls.length, 1); assert.equal(state.timerCount(), 0);
+  releaseA(); releaseA(); await tick(); assert.equal(calls.length, 1);
+  releaseB(); await state.scheduler.settle();
+  assert.deepEqual(calls.map(([action, request]) => [action, request.operationIds[0]]), [['submit', 'first'], ['query', 'first'], ['submit', 'later'], ['submit', 'dependent']]);
+  assert.equal(calls[1][1].id, frozen.id); assert.equal(calls[1][1].body, frozen.body); assert.equal(state.scheduler.getClaims().size, 0);
+  state.scheduler.stop();
+});
+
+test('holds acquired before start retain notified work; stop is terminal even when a hold is later released', async () => {
+  const state = setup(), release = state.scheduler.hold(); state.add('first'); state.scheduler.start(); await tick();
+  assert.equal(state.calls.length, 0); release(); await state.scheduler.settle(); assert.equal(state.calls.length, 1);
+  const finalRelease = state.scheduler.hold(); state.add('never'); state.scheduler.stop(); finalRelease(); await tick();
+  assert.equal(state.calls.length, 1); assert.throws(() => state.scheduler.hold(), /scheduler_closed/);
+});

tests/headless/service.test.mjs

@@ -98,3 +98,16 @@ test('shared service transport binds the immutable AI admission version header',
   for(const value of [0,-1,1.5,NaN,Number.MAX_SAFE_INTEGER+1])await assert.rejects(()=>transport({method:'POST',path:'fixture',expectedVersion:value}),/invalid_server_version/);
   assert.equal(calls.length,1);
 });
+
+test('recovery reader requires a complete canonical model for its bound project and forwards cancellation',async()=>{
+  const controller=new AbortController(),calls=[];
+  const{port}=setup(async request=>{calls.push(request);return{status:200,body:{model:chunk()}};});
+  const loaded=await port.read(controller.signal);assert.equal(loaded.document.projectId,root);assert.equal(loaded.versions.get(child),7);
+  assert.equal(calls[0].method,'GET');assert.equal(calls[0].signal,controller.signal);assert.equal(calls[0].body,undefined);assert.equal(calls[0].requestId,undefined);
+  for(const[response,error]of[
+    [{status:200,body:{model:chunk(false)}},/incomplete_initial_model/],
+    [{status:409,body:{error:'canonical_migration_required',document_format_version:1,root_id:root,source_versions:{[root]:1}}},/canonical_migration_required/],
+    [{status:200,body:{model:{documentFormatVersion:1,projectId:child,canvases:[{id:child,parent:null}],elements:[],tombstones:{},versions:{[child]:1},complete:true,removedCanvasIds:[]}}},/project_mismatch/],
+    [{status:403,body:{error:'forbidden'}},/forbidden/],
+  ]){const{port:invalid}=setup(async()=>response);await assert.rejects(invalid.read(controller.signal),error);}
+});