Add quiescent canonical recovery sessions without rewriting requests

src/lib/canvas-runtime/contracts.ts

@@ -17,6 +17,8 @@ export type RuntimeState = {
 export type EngineSnapshot = Snapshot<RuntimeState>;
 export type CanonicalProposal = { readonly document: Document; readonly versions: ReadonlyMap<Id, number> };
 export type PersistencePort = {
+  /** Complete canonical read for a quiescent recovery session; never a legacy fallback. */
+  read?: (signal: AbortSignal) => Promise<CanonicalProposal>;
   supportsRequest?: (request: FrozenRequest) => boolean;
   compile(intent: OperationIntent, requestId: Id, snapshot: EngineSnapshot): CompiledRequest;
   submit(request: FrozenRequest, signal: AbortSignal): Promise<OperationStatus>;

src/lib/canvas-runtime/engine.ts

@@ -3,11 +3,11 @@ import { createEditor, type EditorOptions, type Transaction } from '../canvas-co
 import { createDocument } from '../canvas-core/document.ts';
 import { createSessions } from '../canvas-core/gestures.ts';
 import { equal, fault, freeze, readMap, reject } from '../canvas-core/values.ts';
-import type { EngineSnapshot, PersistencePort, RuntimeState } from './contracts.ts';
+import type { CanonicalProposal, EngineSnapshot, PersistencePort, RuntimeState } from './contracts.ts';
 import { createRunTracker, validateRunInput, runContainsTarget, type RunPort, type RunRecord, type RunPage } from './runs.ts';
 import { createNetworkGate } from './network.ts';
 import { createLedger, type FrozenRequest } from './ledger.ts';
-import { operationCanvases, replayPending } from './reconcile.ts';
+import { operationCanvases, replayPending, validateCanonicalRead } from './reconcile.ts';
 import { decodeRecovery, encodeRecovery } from './recovery.ts';
 import { createScheduler, type Clock, type OperationStatus } from './scheduler.ts';
 import { conflictsWith, maxResourceBytes, operationFootprint, referencedResources, type AssetPort, type ResourceRecord, type ResourceHandle, type StagedFile } from './resources.ts';
@@ -19,6 +19,15 @@ export type EngineOptions = {
   historyLimit?: number; policy?: EditorOptions['policy']; traversalPolicy?: EditorOptions['traversalPolicy']; clock?: Clock;
   onError?: (error: unknown) => void;
 };
+export type RecoverySession = {
+  readonly baseRootRevision: number;
+  readonly canonical: CanonicalProposal;
+  readonly pendingOperationIds: readonly Id[];
+  /** A conflict is diagnostic only: the existing document/history/requests remain intact. */
+  readonly conflict?: { readonly code: string; readonly targets: readonly Id[] };
+  apply(): Promise<Result>;
+  cancel(): Promise<void>;
+};
 
 /** One project owner. Detaching a view does not close this engine or abort persistent work. */
 export async function openProjectEngine(options: EngineOptions) {
@@ -37,10 +46,12 @@ export async function openProjectEngine(options: EngineOptions) {
     const fileWrites = new Map<Id, Blob>(), digestTasks = new Map<Id, Promise<void>>();
     let staged: readonly ResourceRecord[] = [];
     let applying: RunApplyInput | undefined;
+    let recovery: { controller: AbortController; release(): void } | undefined;
+    const assertWritable = () => { if(closing||closed)throw fault('engine_closing');if(recovery)throw fault('recovery_in_progress'); };
     const initial: RuntimeState = recovered?.runtime ?? Object.freeze({ runs: readMap(new Map()), acknowledged: createDocument(options.document.projectId, [...options.document.canvases.values()], [...options.document.elements.values()], options.document.tombstones), versions: readMap(options.versions), ledger: createLedger().getSnapshot(), receipts: readMap(new Map()), conflicts: readMap(new Map()), resources: readMap(new Map()), resourcePins: readMap(new Map()) });
     const editor = createEditor({ registry: options.registry, document: options.document, recovery: recovered, runtime: initial, createId: options.createId, historyLimit: options.historyLimit,
-      policy(command, document) { if (closing || closed) return { enabled: false, code: 'engine_closing', targets: [] }; return options.policy?.(command, document) ?? { enabled: true }; },
-      traversalPolicy(entry, direction) { return closing || closed ? { enabled: false, code: 'engine_closing', targets: [] } : options.traversalPolicy?.(entry, direction) ?? { enabled: true }; },
+      policy(command, document) { if (closing || closed || recovery) return { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] }; return options.policy?.(command, document) ?? { enabled: true }; },
+      traversalPolicy(entry, direction) { return closing || closed || recovery ? { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] } : options.traversalPolicy?.(entry, direction) ?? { enabled: true }; },
       onListenerError: report,
       prepareIntent: intent => applying && intent.origin==='user' ? {...intent,operationType:'run.apply',action:applying} : intent,
       nextOperationSequence(runtime, fallback) { let next = fallback; for (const record of runtime.ledger.operations.values()) next = Math.max(next, record.intent.sequence + 1); return next + staged.length; },
@@ -283,6 +294,51 @@ export async function openProjectEngine(options: EngineOptions) {
     scheduler.start(); prepareDigests(); runTracker.notify([...initial.runs.keys()]);
     return Object.freeze({
       editor, sessions,
+      /** Read/preview under a scheduling hold. No request is rewritten or declared resolved here. */
+      async prepareRecovery(): Promise<RecoverySession> {
+        assertWritable();
+        if(!options.persistence.read)throw fault('canonical_reader_unavailable');
+        const releaseScheduler=scheduler.hold(),releaseRuns=runTracker.hold(),controller=new AbortController();
+        const active={controller,release(){if(recovery!==active)return;recovery=undefined;releaseRuns();releaseScheduler();}};
+        recovery=active;
+        sessions.cancel('recovery');
+        const assertCurrent=()=>{if(closing||closed)throw fault('engine_closing');if(recovery!==active||controller.signal.aborted)throw fault('recovery_session_closed');};
+        try {
+          // Both holds are acquired before awaiting either lane; already admitted callbacks finish normally.
+          await Promise.all([scheduler.settle(),runTracker.settle()]);
+          while(digestTasks.size)await Promise.all([...digestTasks.values()]);
+          await persist();assertCurrent();
+          const root=editor.getSnapshot(),previous=root.runtime;
+          const unresolved=[...previous.ledger.requests.values()].filter(request=>request.phase!=='resolved');
+          if(unresolved.length)throw fault('recovery_waiting_for_receipts',unresolved.map(request=>request.id));
+          const loaded=await network.run(controller.signal,()=>options.persistence.read!(controller.signal));assertCurrent();
+          if(editor.getSnapshot().rootRevision!==root.rootRevision)throw fault('stale_plan');
+          validateCanonicalRead(previous.acknowledged,previous.versions,loaded.document,loaded.versions,options.registry);
+          const acknowledged=createDocument(loaded.document.projectId,[...loaded.document.canvases.values()],[...loaded.document.elements.values()],loaded.document.tombstones);
+          const canonical=Object.freeze({document:acknowledged,versions:readMap(loaded.versions)});
+          let working:Document|undefined,conflict:RecoverySession['conflict'];
+          try{working=replayPending(acknowledged,previous.ledger,options.registry);}catch(error){const rejected=reject(error);conflict=freeze({code:rejected.code,targets:rejected.targets});}
+          let committed=false;
+          return Object.freeze({baseRootRevision:root.rootRevision,canonical,
+            pendingOperationIds:Object.freeze([...previous.ledger.operations.values()].filter(record=>record.phase!=='resolved').map(record=>record.intent.id)),
+            ...(conflict?{conflict}:{}),
+            async apply():Promise<Result>{
+              assertCurrent();
+              if(conflict)return{status:'rejected',...conflict};
+              if(!committed){
+                const conflicts=new Map(previous.conflicts);conflicts.delete('refresh');
+                const result=editor.reconcile(working!,Object.freeze({...previous,acknowledged,versions:canonical.versions,conflicts:readMap(conflicts)}),root.rootRevision);
+                if(result.status==='rejected')return result;
+                committed=true;
+              }
+              // Persistence failure retains the hold; apply/cancel can retry this barrier without republishing.
+              await persist();assertCurrent();active.release();
+              return{status:'applied',revision:editor.getSnapshot().revision,operationIds:[]};
+            },
+            async cancel(){if(recovery!==active)return;if(committed)await persist();active.release();},
+          });
+        }catch(error){active.release();throw error;}
+      },
       models(boardId: Id, signal: AbortSignal) {
         if(closing||closed||!options.runs)return Promise.reject(fault('run_interpreter_unavailable'));
         if(!editor.getSnapshot().document.canvases.has(boardId))return Promise.reject(fault('canvas_missing'));
@@ -290,7 +346,7 @@ export async function openProjectEngine(options: EngineOptions) {
       },
       cancelRun(runId:Id):Result {
         try{
-          if(closing||closed)throw fault('engine_closing');if(!options.runs)throw fault('run_interpreter_unavailable');
+          assertWritable();if(!options.runs)throw fault('run_interpreter_unavailable');
           const root=editor.getSnapshot(),run=root.runtime.runs.get(runId);if(!run)throw fault('run_missing');
           const allowed=options.policy?.({type:'element.data',target:run.target,patch:{}},root.document);if(allowed&&!allowed.enabled)throw fault(allowed.code,allowed.targets);
           if([...root.runtime.ledger.operations.values()].some(record=>record.phase!=='resolved'&&record.intent.operationType==='run.cancel'&&record.intent.action.runId===runId))throw fault('run_action_pending');
@@ -299,7 +355,7 @@ export async function openProjectEngine(options: EngineOptions) {
       },
       applyRunOutput(runId:Id,target:RunApplyInput['target'],outputId:Id):Result {
         try{
-          if(closing||closed)throw fault('engine_closing');if(!options.runs)throw fault('run_interpreter_unavailable');if(applying)throw fault('reentrant_dispatch');
+          assertWritable();if(!options.runs)throw fault('run_interpreter_unavailable');if(applying)throw fault('reentrant_dispatch');
           const root=editor.getSnapshot(),run=root.runtime.runs.get(runId),element=root.document.elements.get(target.id);
           if(!run||run.blocked||!runContainsTarget(run,target)||!element||element.incarnation!==target.incarnation)throw fault('target_changed');
           if((run.appliedOutputIds??[]).includes(outputId))throw fault('output_already_applied');
@@ -315,7 +371,7 @@ export async function openProjectEngine(options: EngineOptions) {
       },
       startRun(input: RunInput): Result {
         try {
-          if(closing||closed)throw fault('engine_closing');
+          assertWritable();
           if(!options.runs)throw fault('run_interpreter_unavailable');
           validateRunInput(input);
           const root=editor.getSnapshot(),element=root.document.elements.get(input.target.id);
@@ -335,11 +391,9 @@ export async function openProjectEngine(options: EngineOptions) {
       refresh(document: Document, versions: ReadonlyMap<Id, number>): Result {
         const root = editor.getSnapshot(), previous = root.runtime;
         try {
-          if (closing || closed) throw fault('engine_closing');
-          if (document.projectId !== root.document.projectId) throw fault('project_mismatch');
+          assertWritable();
           if ([...previous.ledger.requests.values()].some(request => request.phase !== 'resolved')) throw fault('refresh_waiting_for_receipts');
-          for (const canvas of document.canvases.keys()) if (!Number.isSafeInteger(versions.get(canvas)) || versions.get(canvas)! < 1) throw fault('missing_server_version', [canvas]);
-          for (const [id, version] of previous.versions) if ((versions.get(id) ?? -1) < version) throw fault('stale_canonical_version', [id]);
+          validateCanonicalRead(previous.acknowledged,previous.versions,document,versions,options.registry);
           if (equal([...previous.versions], [...versions]) && equal([...previous.acknowledged.canvases], [...document.canvases]) && equal([...previous.acknowledged.elements], [...document.elements]) && equal([...previous.acknowledged.tombstones], [...document.tombstones])) return { status: 'applied', revision: root.revision, operationIds: [] };
           const acknowledged = createDocument(document.projectId, [...document.canvases.values()], [...document.elements.values()], document.tombstones);
           const working = replayPending(acknowledged, previous.ledger, options.registry);
@@ -347,12 +401,12 @@ export async function openProjectEngine(options: EngineOptions) {
           return editor.reconcile(working, Object.freeze({ ...previous, acknowledged, versions: readMap(versions), conflicts: readMap(conflicts) }), root.rootRevision);
         } catch (error) {
           const result = reject(error);
-          if (result.code !== 'refresh_waiting_for_receipts' && result.code !== 'engine_closing') editor.updateRuntime(previous => { const conflicts = new Map(previous.conflicts); conflicts.set('refresh', result.code); return Object.freeze({ ...previous, conflicts: readMap(conflicts) }); });
+          if (!['refresh_waiting_for_receipts','engine_closing','recovery_in_progress'].includes(result.code)) editor.updateRuntime(previous => { const conflicts = new Map(previous.conflicts); conflicts.set('refresh', result.code); return Object.freeze({ ...previous, conflicts: readMap(conflicts) }); });
           return result;
         }
       },
       stageFiles(files: readonly StagedFile[], callback: (resources: readonly ResourceHandle[], transaction: Transaction) => unknown) {
-        if (closing || closed) throw fault('engine_closing');
+        assertWritable();
         if (staged.length) throw fault('reentrant_dispatch');
         if (!options.assets) throw fault('asset_interpreter_unavailable');
         const resources = files.map(file => {
@@ -380,11 +434,12 @@ export async function openProjectEngine(options: EngineOptions) {
       },
       getSnapshot: editor.getSnapshot, persist,
       getDurability: () => Object.freeze({ rootRevision: editor.getSnapshot().rootRevision, durableRootRevision }),
-      retry: async () => { await scheduler.resume();await runTracker.retry(); },
+      retry: async () => { assertWritable();await scheduler.resume();await runTracker.retry(); },
       async settle() { await Promise.resolve(); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
       async close() {
         if (closed) return;
         closing = true; sessions.dispose(); scheduler.stop(); runTracker.stop();
+        recovery?.controller.abort();recovery?.release();
         for (const release of [...displays]) release();
         // Failure keeps the leader/recovery material available for an explicit retry of close.
         await persist(); unsubscribe(); editor.dispose(); await leader.release(); closed = true;

src/lib/canvas-runtime/reconcile.ts

@@ -3,6 +3,26 @@ import { applyChanges, validateDocument } from '../canvas-core/document.ts';
 import { equal, fault } from '../canvas-core/values.ts';
 import type { LedgerSnapshot } from './ledger.ts';
 
+/** A complete read cannot rewind known scopes or change an unchanged-version canvas. */
+export function validateCanonicalRead(previous: Document, previousVersions: ReadonlyMap<Id,number>, next: Document, versions: ReadonlyMap<Id,number>, registry: Registry) {
+  if(next.projectId!==previous.projectId)throw fault('project_mismatch');
+  validateDocument(next,registry);
+  for(const [id,version]of versions)if(!Number.isSafeInteger(version)||version<1)throw fault('invalid_canonical_version',[id]);
+  for(const canvas of next.canvases.keys())if(!versions.has(canvas))throw fault('missing_server_version',[canvas]);
+  const unchanged=new Set<Id>();
+  for(const [id,version]of previousVersions){
+    if((versions.get(id)??-1)<version)throw fault('stale_canonical_version',[id]);
+    if(versions.get(id)!==version)continue;
+    if(!equal(previous.canvases.get(id),next.canvases.get(id)))throw fault('canonical_version_mismatch',[id]);
+    unchanged.add(id);
+  }
+  for(const [id,element]of previous.elements)if(unchanged.has(element.ownerCanvasId)&&!equal(element,next.elements.get(id)))throw fault('canonical_version_mismatch',[element.ownerCanvasId]);
+  for(const [id,element]of next.elements)if(unchanged.has(element.ownerCanvasId)&&!equal(element,previous.elements.get(id)))throw fault('canonical_version_mismatch',[element.ownerCanvasId]);
+  for(const [id,incarnation]of previous.tombstones)if((next.tombstones.get(id)??0)<incarnation)throw fault('stale_canonical_lifecycle',[id]);
+  if(versions.size===previousVersions.size&&[...versions].every(([id,version])=>previousVersions.get(id)===version)&&
+    (previous.tombstones.size!==next.tombstones.size||[...previous.tombstones].some(([id,incarnation])=>next.tombstones.get(id)!==incarnation)))throw fault('canonical_version_mismatch');
+}
+
 /** The footprint follows ownership; geometry does not decide which canvas owns an object. */
 export function operationCanvases(intent: OperationIntent, document: Document): readonly Id[] {
   if(intent.operationType==='asset.upload'||intent.operationType==='run.cancel')return[];

src/lib/canvas-runtime/runs.ts

@@ -49,13 +49,14 @@ export function createRunTracker(options: {
 }) {
   const heap = createMinHeap<{ id: Id; due: number; token: number; sequence: number }>((a,b) => a.due-b.due || a.sequence-b.sequence);
   const tokens = new Map<Id,number>();
+  const holds = new Set<symbol>();
   let sequence = 0, stopped = false, timer: unknown, active: { controller: AbortController; promise: Promise<void> } | undefined;
   const report = (error: unknown) => { try { options.onError(error); } catch { /* Reporting cannot alter cursor durability. */ } };
   const eligible = (run: RunRecord) => !run.blocked && (run.hasMore || !runTerminal(run.status));
   function arm() {
     if (timer !== undefined) options.clock.clearTimer(timer); timer = undefined;
     while (heap.peek() && tokens.get(heap.peek()!.id) !== heap.peek()!.token) heap.pop();
-    if (stopped || active || !heap.peek()) return;
+    if (stopped || holds.size || active || !heap.peek()) return;
     timer = options.clock.setTimer(() => { timer = undefined; pump(); }, Math.max(0,heap.peek()!.due-options.clock.now()));
   }
   function notify(ids: readonly Id[]) {
@@ -68,7 +69,7 @@ export function createRunTracker(options: {
     arm();
   }
   function pump() {
-    if (stopped || active) return;
+    if (stopped || holds.size || active) return;
     let item = heap.pop();
     while (item && tokens.get(item.id)!==item.token) item=heap.pop();
     if (!item) return;
@@ -100,6 +101,13 @@ export function createRunTracker(options: {
     active={controller,promise};
   }
   return Object.freeze({ notify,
+    /** Holds compose; completing a poll still commits its cursor before settle() returns. */
+    hold() {
+      if(stopped)throw fault('run_tracker_closed');
+      const token=Symbol();holds.add(token);
+      if(timer!==undefined)options.clock.clearTimer(timer);timer=undefined;
+      return()=>{if(holds.delete(token)&&!holds.size&&!stopped)arm();};
+    },
     async retry() { await options.persist();for(const id of options.getRuns().keys())options.update(id,run=>({...run,blocked:false,error:undefined,nextPollAt:options.clock.now()}));notify([...options.getRuns().keys()]); },
     async settle() { while(active)await active.promise; },
     stop() { stopped=true;if(timer!==undefined)options.clock.clearTimer(timer);timer=undefined;active?.controller.abort();heap.clear(); },

src/lib/canvas-runtime/scheduler.ts

@@ -34,6 +34,7 @@ export function createScheduler(options: SchedulerOptions) {
   const waiters = new Map<Id, Set<Id>>(), dependents = new Map<Id, Set<Id>>(), indexed = new Set<Id>();
   const retry = createMinHeap<{ id: Id; dueAt: number; sequence: number; token: number }>((a, b) => a.dueAt - b.dueAt || a.sequence - b.sequence);
   const tokens = new Map<Id, number>(), active = new Map<Id, { controller: AbortController; promise: Promise<void>; kind: FrozenRequest['kind'] }>();
+  const holds = new Set<symbol>();
   const laneWaiters = new Map<NonNullable<FrozenRequest['kind']>, Set<Id>>(), deferredRetries = new Map<Id, { id: Id; dueAt: number; sequence: number; token: number }>();
   let timer: unknown, pumpTimer: unknown, enqueueSequence = 0, stopped = true, closed = false, pumping = false;
   const delay = (attempt: number) => Math.max(1, options.retryDelay?.(attempt) ?? Math.min(30_000, 500 * 2 ** Math.min(attempt, 6)));
@@ -84,11 +85,11 @@ export function createScheduler(options: SchedulerOptions) {
   function armTimer() {
     if (timer !== undefined) { clock.clearTimer(timer); timer = undefined; }
     while (retry.peek() && !validRetry(retry.peek()!)) retry.pop();
-    if (stopped || active.size >= concurrency || !retry.peek()) return;
+    if (stopped || holds.size || active.size >= concurrency || !retry.peek()) return;
     timer = clock.setTimer(() => { timer = undefined; pump(); }, Math.max(0, retry.peek()!.dueAt - clock.now()));
   }
   function launch(id: Id, action: 'submit' | 'query') {
-    if (active.has(id) || stopped) return;
+    if (active.has(id) || stopped || holds.size) return;
     const controller = new AbortController();
     // Start in a microtask so the handle exists before any synchronous adapter error.
     const promise = Promise.resolve().then(async () => {
@@ -126,7 +127,7 @@ export function createScheduler(options: SchedulerOptions) {
     active.set(id, { controller, promise, kind: options.getLedger().requests.get(id)?.kind });
   }
   function pump() {
-    if (stopped || pumping) return;
+    if (stopped || holds.size || pumping) return;
     pumping = true;
     try {
       while (active.size < concurrency && retry.peek() && retry.peek()!.dueAt <= clock.now()) {
@@ -163,7 +164,7 @@ export function createScheduler(options: SchedulerOptions) {
       }
     } finally {
       pumping = false; armTimer();
-      if (!stopped && ready.size && active.size < concurrency && pumpTimer === undefined) pumpTimer = clock.setTimer(() => { pumpTimer = undefined; pump(); }, 0);
+      if (!stopped && !holds.size && ready.size && active.size < concurrency && pumpTimer === undefined) pumpTimer = clock.setTimer(() => { pumpTimer = undefined; pump(); }, 0);
     }
   }
   return Object.freeze({
@@ -182,6 +183,18 @@ export function createScheduler(options: SchedulerOptions) {
       pump();
     },
     notify(operationIds: readonly Id[]) { for (const id of operationIds) index(id); pump(); },
+    /** Acquire synchronously, then settle(): already admitted work completes with its original identity. */
+    hold() {
+      if (closed) throw fault('scheduler_closed');
+      const token = Symbol(); holds.add(token);
+      if (timer !== undefined) clock.clearTimer(timer);
+      if (pumpTimer !== undefined) clock.clearTimer(pumpTimer);
+      timer = pumpTimer = undefined;
+      return () => {
+        if (!holds.delete(token) || holds.size || stopped) return;
+        pump();
+      };
+    },
     /** Explicit retry after storage/auth/permission recovery; preserves frozen bodies. */
     async resume() {
       if (stopped) return;

src/lib/canvas-service/foundry.ts

@@ -114,6 +114,12 @@ export function createFoundryPersistence(options: { workspaceId: Id; rootId: Id;
   }
   return Object.freeze({
     supportsRequest: (request: FrozenRequest) => request.protocolVersion === 2,
+    async read(signal: AbortSignal) {
+      const loaded=await loadFoundryModel(options.transport,options.workspaceId,options.rootId,options.registry,signal);
+      if(loaded.status!=='ready')throw fault('canonical_migration_required');
+      if(loaded.document.projectId!==options.rootId)throw fault('project_mismatch');
+      return loaded;
+    },
     compile(intent: OperationIntent, requestId: Id, snapshot: EngineSnapshot) {
       if (snapshot.document.projectId !== options.rootId || !uuid.test(intent.id) || !uuid.test(requestId) || intent.operationType !== 'document.change' || intent.operationSchemaVersion !== 1) throw fault('operation_interpreter_unavailable');
       const canvasIds = [...new Set(intent.canvasIds)].sort(), expectedVersions: Record<Id, number> = {};