test(canvas): verify retry deadlines across wall-clock shifts and recovery

headless/SERVICE_ACCEPTANCE.md

@@ -1,21 +1,27 @@
 # Foundry service integration evidence
 
-Status: step 5 service scope audit completed; overall progress is 5/7.
-This does not complete migration/delivery,
-the complete Draft acceptance matrix, main integration or production deployment.
+Status: steps 5 (service) and 6 (migration/delivery) completed; overall progress is 6/7.
+The complete Draft acceptance matrix, main integration and production deployment
+are not complete.
 
 ## Verified revisions and environment
 
-- Endoleap consumer: `26da326c18bdf5363c646dff87756590b29433ea`.
-- Foundry: `3268c57bc64cb85f9de80d5e04107d963ede963a`.
+- Current delivered Endoleap consumer: `24c45920602378d52a42cda7e8ec2840864c8dbd`.
+- Current Foundry: `5db2bf3c98f2627139cbb6d47afefa686b4c6547`.
+- [Foundry CI #1182](https://gitea.entroia.com/Zayden-Jung/foundry-core/actions/runs/1182):
+  job 1279 completed successfully, all seven steps successful, pinned to the
+  current delivered consumer. Its report has all document/resource/AI and
+  migration flags true, `aiExecutionCount: 3`, and 75 transport calls.
+  Same-commit PR CI #1183/job 1280 was still running at this audit.
 - [Foundry CI #1180](https://gitea.entroia.com/Zayden-Jung/foundry-core/actions/runs/1180)
   and [PR CI #1181](https://gitea.entroia.com/Zayden-Jung/foundry-core/actions/runs/1181):
-  completed successfully; jobs 1277/1278, all seven steps successful. The original
-  service scope audit also passed CI #1178 and #1179.
+  completed successfully for consumer `26da326` and Foundry `3268c57`;
+  jobs 1277/1278, all seven steps successful. The original service scope audit
+  also passed CI #1178 and #1179.
 - Existing `ces-control` runner executes Go checks/build, schema migration and
   isolated PostgreSQL integration, then the pinned Node 24 consumer and HTTP smoke
   checks. No local backend build or Docker execution was used.
-- Local frontend: 800 Node tests and full TypeScript checking passed. Tests use
+- Local frontend: 802 Node tests and full TypeScript checking passed. Tests use
   the source modules; packaged SDK and application delivery remain separate gates.
 
 ## What the checks actually prove
@@ -92,8 +98,13 @@ webpack and Turbopack application regressions each passed 51 checks; the migrati
 entry adds five scenarios and three lock-lifetime unit tests. Unknown
 receipt/error states remain read-only with an export action. Unsupported Web
 Locks prevents upgrading, while existing legacy editing remains available when
-there is no pending migration. Stage 6 and the final requirement matrix still
-require their delivery audit; passing these checks alone does not complete them.
+there is no pending migration. The stage 6 audit also inspected source/package
+exports, peer dependencies and documentation targets, then passed the actual
+267-file unpacked package's Node/types, framework-peer, compiled browser storage
+and Worker checks. Together with CI #1182 and both browser bundlers this closes
+stage 6. The full requirement matrix remains stage 7; the two new scheduler
+clock-shift checks cover backward/forward wall time, dependency/unknown claims,
+stale timer callbacks and remaining wait calculation after recovery.
 
 `canvas_headless_worker_integration_test.go` supplies deterministic generation
 output over the actual worker HTTP protocol. Admission, claim, events, canonical
@@ -110,6 +121,7 @@ including subtests; that count is not 154 independent end-to-end scenarios.
 The service audit covers the original stage requirements: target incremental and
 idempotent protocols, frontend/runtime/resource adapters, explicit legacy
 compatibility, identity mapping, source-version/permission/run admission barriers,
-and rollback after writes have begun. Migration/delivery and the final requirement
-matrix remain. A green feature-branch run does not merge or deploy
+and rollback after writes have begun. Migration/delivery now has the separate
+browser/package evidence above; the final requirement matrix remains. A green
+feature-branch run does not merge or deploy
 either repository. Foundry PR #19 remains open and Draft.

tests/headless/scheduler.test.mjs

@@ -225,3 +225,48 @@ test('bounded batches drain through settle without waiting for fake timers and h
 test('a fenced pump reports failed state publication without throwing or sending through the old identity',async()=>{
  const state=setup({mutate(){throw Error('storage_leader_lost');}});state.scheduler.start();state.add('fenced');await tick();await state.scheduler.settle();assert.equal(state.calls.length,0);assert.equal(state.ledger().getSnapshot().requests.size,0);assert.equal(state.ledger().getSnapshot().operations.get('fenced').phase,'pending');assert.ok(state.errors.some(error=>error.message==='storage_leader_lost'));state.scheduler.stop();
 });
+
+// Wall time can change independently of the elapsed time used by browser timers.
+function adjustableClock(initial=1000){
+  let wall=initial,elapsed=0,next=0;const timers=new Map();
+  return{clock:{now:()=>wall,setTimer(callback,delay){const id=++next;timers.set(id,{callback,delay,at:elapsed+delay});return id;},clearTimer:id=>timers.delete(id)},timers,
+    setWall(value){wall=value;},fireAfter(ms){elapsed+=ms;const due=[...timers].filter(([,timer])=>timer.at<=elapsed);for(const[id,timer]of due){timers.delete(id);timer.callback();}return due.map(([,timer])=>timer.callback);}};
+}
+
+test('wall-clock jumps recheck retry deadlines without releasing unknown claims or bypassing dependencies',async()=>{
+  const clock=adjustableClock(),receipt=deferred(),calls=[];
+  const state=setup({clock:clock.clock,
+    async submit(request){calls.push(['submit',request]);return{outcome:request.operationIds[0]==='first'?'unresolved':'applied'};},
+    async query(request){calls.push(['query',request]);return receipt.promise;},
+  });
+  state.add('first');state.add('dependent',['canvas'],['first']);state.scheduler.start();await state.scheduler.settle();
+  const frozen=state.ledger().getSnapshot().requests.get('request-1');assert.equal(frozen.retryAt,1100);
+  clock.setWall(800);clock.fireAfter(100);await state.scheduler.settle();
+  assert.equal(calls.length,1);assert.equal(state.scheduler.getClaims().get('canvas'),frozen.id);
+  assert.equal(clock.timers.size,1);assert.equal([...clock.timers.values()][0].delay,300);
+  state.add('independent',['other']);await state.scheduler.settle();
+  assert.deepEqual(calls.map(([action,request])=>[action,request.operationIds[0]]),[['submit','first'],['submit','independent']]);
+  clock.setWall(5000);const staleCallbacks=clock.fireAfter(300);await tick();
+  assert.equal(calls.at(-1)[0],'query');assert.equal(calls.at(-1)[1].body,frozen.body);assert.equal(calls.at(-1)[1].id,frozen.id);
+  assert.equal(state.scheduler.getClaims().get('canvas'),frozen.id);assert.equal(state.ledger().getSnapshot().operations.get('dependent').phase,'pending');
+  for(const callback of staleCallbacks)callback();await tick();assert.equal(calls.length,3);
+  receipt.resolve({outcome:'applied'});await state.scheduler.settle();
+  assert.deepEqual(calls.map(([action,request])=>[action,request.operationIds[0]]),[['submit','first'],['submit','independent'],['query','first'],['submit','dependent']]);
+  for(const callback of staleCallbacks)callback();await state.scheduler.settle();assert.equal(calls.length,4);assert.equal(clock.timers.size,0);assert.equal(state.scheduler.getClaims().size,0);state.scheduler.stop();
+});
+
+test('recovered retries recompute remaining wall time and wake the earliest valid request after either clock shift',async()=>{
+  const original=setup({async submit(request){return{outcome:'unresolved',retryAfterMs:request.operationIds[0]==='long'?500:100};}});
+  original.add('long',['a']);original.add('short',['b']);original.scheduler.start();await original.scheduler.settle();const saved=original.ledger().getSnapshot();original.scheduler.stop();
+  for(const now of[800,1200]){
+    const clock=adjustableClock(now),calls=[],restored=setup({clock:clock.clock,async query(request){calls.push(request);return{outcome:'applied'};}},saved);
+    restored.scheduler.start();await restored.scheduler.settle();
+    assert.deepEqual(calls.map(request=>request.operationIds[0]),now===800?[]:['short']);
+    assert.equal(clock.timers.size,1);assert.equal([...clock.timers.values()][0].delay,300);
+    if(now===800){clock.setWall(1100);clock.fireAfter(300);await restored.scheduler.settle();assert.deepEqual(calls.map(request=>request.operationIds[0]),['short']);assert.equal([...clock.timers.values()][0].delay,400);}
+    clock.setWall(1500);clock.fireAfter(now===800?400:300);await restored.scheduler.settle();
+    assert.deepEqual(calls.map(request=>request.operationIds[0]),['short','long']);
+    for(const request of calls){const before=saved.requests.get(request.id);assert.equal(request.body,before.body);assert.deepEqual(request.operationIds,before.operationIds);}
+    assert.equal(clock.timers.size,0);assert.equal(restored.scheduler.getClaims().size,0);restored.scheduler.stop();
+  }
+});