Reuse preallocated import identities and stable recovery document views

headless/README.md

@@ -751,3 +751,28 @@ The 10000-child copy probe still exceeded its 240-second completion window after
 this change. The exact hashed-input browser test proves removal of whole-image
 hashing for ordinary adjacent delta writes; this is not a passed end-to-end copy
 budget or a claim that encoding, rendering and dispatch costs are all resolved.
+
+
+Import identity capture now reuses a frozen, own-data string dictionary when no
+new ID is reserved. One active runner retains its last ID-table reference and a
+reverse Set; only the first actual allocation copies both. Mutable dictionaries
+and frozen getters are captured defensively, and a late reserve callback cannot
+mutate an already published result. The index is cleared when the pump ends,
+including pause/cancel/close/restart; the durable `stableIds` remains on ImportJob.
+Allocation keeps the caller port method receiver. No task or recovery version
+changes, and first-item publication/durability barriers remain in place.
+
+Recovery also reuses frozen plain document views across metadata-only Roots.
+`isFactoryDocument` identifies an envelope with fixed table membership produced
+by core constructors; it is not a content-validation or deep-immutability test.
+A WeakMap owns only a derived plain view. Its array/tuple containers are frozen
+without freezing caller-owned records. The codec still independently checks deep
+immutability, so mutable values inherited through `applyChanges` are reevaluated.
+Unowned envelopes reread mutable map membership. A test prohibits document-record
+enumeration during a metadata-only encode after the initial view is built.
+
+The 1000-child copy probe completes with these paths, but the 10000-child copy
+still exceeds 240 seconds. Removing ID-table copies and metadata-only document
+scans is directly verified; a general throughput improvement at 10000 nodes is
+not established. Per-record commit, ACK and remaining table traversal costs remain
+part of the larger implementation goal.

src/lib/canvas-core/document.ts

@@ -1,11 +1,17 @@
 import type { CanvasRecord, Change, Document, ElementRecord, Geometry, Id, Registry, Target } from './types.ts';
 import { clone, equal, fault, freeze, readMap, validJson } from './values.ts';
 
+// This owns only the envelope and fixed table membership, not validation or
+// immutability of values inherited by applyChanges from an external document.
+const factoryDocuments=new WeakSet<object>();
+function documentEnvelope(document:Document):Document{const value=Object.freeze(document);factoryDocuments.add(value);return value;}
+export const isFactoryDocument=(document:Document)=>factoryDocuments.has(document);
+
 export function createDocument(projectId: Id, canvases: readonly CanvasRecord[], elements: readonly ElementRecord[] = [], tombstones: ReadonlyMap<Id, number> = new Map()): Document {
   for (const element of elements) { const id = element.id; if (!validJson(element)) throw fault('invalid_element', [id]); }
   for (const canvas of canvases) if (!validJson(canvas)) throw fault('invalid_canvas');
   if (!projectId || new Set(canvases.map(item => item.id)).size !== canvases.length || new Set(elements.map(item => item.id)).size !== elements.length) throw fault('duplicate_identity');
-  return Object.freeze({ projectId, canvases: readMap(new Map(canvases.map(item => [item.id, freeze(clone(item))]))), elements: readMap(new Map(elements.map(item => [item.id, freeze(clone(item))]))), tombstones: readMap(tombstones) });
+  return documentEnvelope({ projectId, canvases: readMap(new Map(canvases.map(item => [item.id, freeze(clone(item))]))), elements: readMap(new Map(elements.map(item => [item.id, freeze(clone(item))]))), tombstones: readMap(tombstones) });
 }
 export function geometryValid(value: Geometry): boolean {
   return !!value && typeof value === 'object' && [value.x, value.y, value.width, value.height, value.rotation].every(Number.isFinite) && value.width > 0 && value.height > 0 && value.rotation >= 0 && value.rotation < 360;
@@ -115,7 +121,7 @@ export function applyChanges(document: Document, changes: readonly Change[], con
       } else canvases.delete(change.id);
     } else throw fault('invalid_change');
   }
-  return Object.freeze({ projectId: document.projectId, elements: readMap(elements), canvases: readMap(canvases), tombstones: readMap(tombstones) });
+  return documentEnvelope({ projectId: document.projectId, elements: readMap(elements), canvases: readMap(canvases), tombstones: readMap(tombstones) });
 }
 export function invert(changes: readonly Change[]): readonly Change[] {
   return changes.slice().reverse().map(change => {

src/lib/canvas-runtime/import-identities.ts

@@ -0,0 +1,27 @@
+import type {Id} from '../canvas-core/types.ts';
+import {fault} from '../canvas-core/values.ts';
+type Ids=Readonly<Record<string,Id>>;
+/** One runner's last immutable ID index. Copies are deferred until allocation. */
+export function createImportIdentities(){
+ let cached:{ids:Ids;used:Set<Id>}|undefined;
+ const capture=(input:Ids,createId:()=>Id)=>{
+  let identity=cached?.ids===input?cached:undefined;
+  if(!identity){
+   const reusable=Object.isFrozen(input)&&Object.values(Object.getOwnPropertyDescriptors(input)).every(property=>Object.hasOwn(property,'value')&&typeof property.value==='string');
+   const ids=reusable?input:Object.assign(Object.create(null),input),used=new Set<Id>(Object.values(ids));identity={ids,used};if(reusable)cached=identity;
+  }
+  let ids=identity.ids,used=identity.used,owned=ids!==input;
+  return Object.freeze({
+   reserve(key:string){
+    if(typeof key!=='string'||!key)throw fault('invalid_import_key');
+    if(Object.hasOwn(ids,key))return ids[key];
+    const id=createId();if(typeof id!=='string'||!id||used.has(id))throw fault('duplicate_import_identity');
+    if(!owned){ids=Object.assign(Object.create(null),ids);used=new Set(used);owned=true;}
+    (ids as Record<string,Id>)[key]=id;used.add(id);return id;
+   },
+   has:(id:Id)=>used.has(id),
+   finish:():Ids=>owned?{...ids}:ids,
+  });
+ };
+ return Object.freeze(Object.assign(capture,{clear(){cached=undefined;}}));
+}

src/lib/canvas-runtime/imports.ts

@@ -1,3 +1,4 @@
+import {createImportIdentities} from './import-identities.ts';
 import {validateImportReadProof,type ImportReadProof} from './import-read-proof.ts';
 import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
 import {createImportBudget,type ImportScheduling} from './import-budget.ts';
@@ -130,6 +131,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
   if(initial.source?.kind==='duplicate-snapshot')throw fault('import_interpreter_unavailable');
   if(initial.source&&(initial.source.integrityVersion===2?!ports.checkpointSource:!initial.source.sha256||initial.source.integrityVersion!==1))throw fault('import_source_integrity_unavailable');
   if(!Number.isFinite(budget)||budget<=0||![maxPending,maxBytes,maxBatch].every(value=>Number.isSafeInteger(value)&&value>0))throw fault('invalid_import_budget');
+  const identities=createImportIdentities(),allocateIdentity=()=>ports.createId();
   let costPerUnit:number|undefined;
   let active:Promise<void>|undefined,controller:AbortController|undefined,closed=false,restart=false;const holds=new Set<symbol>();
   const report=(error:unknown)=>{try{ports.onError(error);}catch{/* Reporting cannot change imported content. */}};
@@ -146,13 +148,12 @@ export function createImportRunner(ports:ImportRunnerPorts){
   function next(job:ImportJob,patch:Partial<ImportJob>):ImportJob{const value={...job,...patch,version:3 as const,generation:job.generation+1};if(value.error===undefined)delete value.error;return freeze(checkpointEvidence(value));}
   function publish(job:ImportJob,patch:Partial<ImportJob>){const value=next(job,patch);ports.publish(value,job.generation);if(!equal(current(),value))throw fault('invalid_import_publish');return value;}
   function prepare(line:ImportLine,job:ImportJob){
-    const ids:Record<string,Id>=Object.assign(Object.create(null),job.stableIds),used=new Set(Object.values(ids));
-    const reserve=(key:string)=>{if(typeof key!=='string'||!key)throw fault('invalid_import_key');if(Object.hasOwn(ids,key))return ids[key];const id=ports.createId();if(typeof id!=='string'||!id||used.has(id))throw fault('duplicate_import_identity');ids[key]=id;used.add(id);return id;};
-    const unit=ports.prepare(line,job,reserve);
+    const ids=identities(job.stableIds,allocateIdentity);
+    const unit=ports.prepare(line,job,ids.reserve);
     if(!validJson(unit)||typeof unit.key!=='string'||!unit.key||!Array.isArray(unit.changes)||!Array.isArray(unit.dependencies)||!unit.changes.length||unit.changes.length>10000||new Set(unit.dependencies).size!==unit.dependencies.length||unit.dependencies.some(key=>typeof key!=='string'||!key))throw fault('invalid_import_unit');
-    for(const change of unit.changes)if(change.kind!=='field'&&change.before===null&&change.after&&!used.has(change.id))throw fault('unstable_import_identity');
+    for(const change of unit.changes)if(change.kind!=='field'&&change.before===null&&change.after&&!ids.has(change.id))throw fault('unstable_import_identity');
     if(new TextEncoder().encode(JSON.stringify(unit)).byteLength>maxBytes)throw fault('import_unit_capacity');
-    return{unit:freeze(clone(unit)),ids:{...ids}};
+    return{unit:freeze(clone(unit)),ids:ids.finish()};
   }
   const checkpoint=async(stage:'parsed'|'committed'='committed')=>{if(ports.optimisticLocal&&ports.awaitDurabilityBudget)await ports.awaitDurabilityBudget(stage);else await ports.persist();};
   const digest=async(unit:ImportUnit)=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(unit))))].map(value=>value.toString(16).padStart(2,'0')).join('');
@@ -273,7 +274,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
       if(closed||controller?.signal.aborted)return;
       try{const job=current();if(['running','completed','partial'].includes(job.status)){publish(job,{status:'paused',error:(error as {code?:string}).code??'import_failed'});try{await ports.persist();}catch{/* Storage may still be unavailable. */}}}catch{/* A replaced identity cannot receive an old error. */}
       report(error);
-    }).finally(()=>{active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}});
+    }).finally(()=>{identities.clear();active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}});
     return active;
   }
   return Object.freeze({

src/lib/canvas-runtime/recovery.ts

@@ -13,7 +13,7 @@ import type { Change, CoreSnapshot, Document, HistoryEntry, HistoryState, Id, Js
 import { validateHistoryReplay, replayHistoryEvidence, sameHistory } from './history-replay.ts';
 import type { RecoveryAudit } from './conflicts.ts';
 import { applyHistoryTransition, validateHistoryTransition, type HistoryTransition } from '../canvas-core/history-transition.ts';
-import { createDocument, validateDocument } from '../canvas-core/document.ts';
+import { createDocument, validateDocument, isFactoryDocument } from '../canvas-core/document.ts';
 import { fault, freeze, readMap, validJson, equal } from '../canvas-core/values.ts';
 import type { EngineSnapshot, RuntimeState } from './contracts.ts';
 import type { FrozenRequest, LedgerSnapshot, OperationRecord } from './ledger.ts';
@@ -21,7 +21,16 @@ import { validateRunInput, type RunRecord } from './runs.ts';
 import type { ResourceRecord } from './resources.ts';
 import type { RecoveryImage } from './storage-port.ts';
 
-const documentValue = (document: Document) => ({ projectId: document.projectId, canvases: [...document.canvases.values()], elements: [...document.elements.values()], tombstones: [...document.tombstones] });
+function encodeDocumentValue(document:Document){
+  // Freeze our own containers only. The codec still independently proves whether
+  // each nested value can be reused, including external mutable records.
+  return Object.freeze({projectId:document.projectId,canvases:Object.freeze([...document.canvases.values()]),elements:Object.freeze([...document.elements.values()]),tombstones:Object.freeze([...document.tombstones].map(pair=>Object.freeze(pair)))});
+}
+const documentViews=new WeakMap<Document,ReturnType<typeof encodeDocumentValue>>();
+function documentValue(document:Document){
+  if(!isFactoryDocument(document))return encodeDocumentValue(document);
+  let value=documentViews.get(document);if(!value){value=encodeDocumentValue(document);documentViews.set(document,value);}return value;
+}
 /** Plain records only: functions, credentials, renderer objects, URLs and Map methods never serialize. */
 export function encodeRecovery(snapshot: EngineSnapshot): RecoveryImage {
   const payload = {

tests/headless/import-identities.test.mjs

@@ -0,0 +1,14 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {createImportIdentities} from '../../src/lib/canvas-runtime/import-identities.ts';
+test('preallocated 10000-ID tables keep their identity across records without allocating or rescanning',()=>{
+ let reads=0;const raw=Object.freeze(Object.fromEntries(Array.from({length:10000},(_,i)=>['key'+i,'id'+i]))),ids=new Proxy(raw,{ownKeys(target){reads++;return Reflect.ownKeys(target);}}),capture=createImportIdentities(),allocate=()=>{throw Error('unexpected allocation');};
+ const first=capture(ids,allocate);assert.equal(first.reserve('key0'),'id0');assert.strictEqual(first.finish(),ids);const checked=reads;for(let i=0;i<10000;i++){const row=capture(ids,allocate);assert.equal(row.reserve('key'+i),'id'+i);assert.ok(row.has('id'+i));assert.strictEqual(row.finish(),ids);}assert.equal(reads,checked);capture.clear();capture(ids,allocate);assert.ok(reads>checked);
+});
+test('first allocation detaches the dictionary and reverse set, and a late callback cannot change its published result',()=>{
+ const ids=Object.freeze({old:'old-id'}),capture=createImportIdentities();let next=0;const row=capture(ids,()=>`new-${++next}`);row.reserve('added');const result=row.finish();assert.notStrictEqual(result,ids);row.reserve('late');assert.deepEqual(result,{old:'old-id',added:'new-1'});assert.deepEqual(ids,{old:'old-id'});const sibling=capture(ids,()=> 'new-1');assert.equal(sibling.has('new-1'),false);assert.equal(sibling.reserve('other'),'new-1');assert.throws(()=>capture(ids,()=> 'old-id').reserve('duplicate'),/duplicate_import_identity/);
+});
+test('mutable and frozen getter dictionaries are captured defensively rather than reused',()=>{
+ const capture=createImportIdentities(),mutable={one:'first'},first=capture(mutable,()=> 'new');mutable.one='second';assert.equal(first.reserve('one'),'first');assert.deepEqual(first.finish(),{one:'first'});assert.equal(capture(mutable,()=> 'new').reserve('one'),'second');
+ let current='old',reads=0;const dynamic=Object.freeze({get one(){reads++;return current;}}),one=capture(dynamic,()=> 'new');current='changed';assert.equal(one.reserve('one'),'old');assert.equal(reads,1);assert.equal(capture(dynamic,()=> 'new').reserve('one'),'changed');assert.equal(reads,2);
+});

tests/headless/imports.test.mjs

@@ -143,3 +143,11 @@ test('async preparation denial isolates independent groups without publishing th
 test('a generation change during async preparation prevents the old attempt from committing or changing progress',async()=>{const gate=deferred(),entered=deferred(),f=fixture([{id:'later'}],{ports:{async beforeCommit(){entered.resolve();await gate.promise;return{enabled:true};}}});const running=f.runner.start();await entered.promise;f.root={...f.root,job:{...f.root.job,generation:f.root.job.generation+1}};const generation=f.root.job.generation;gate.resolve();await running;assert.equal(f.commits.length,0);assert.equal(f.root.job.generation,generation);assert.equal(f.root.job.committedSourceKeys.length,0);await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.document.elements.size,1);await f.runner.close();});
 
 test('NDJSON checks elapsed time before returning a short scanned record and honors cancellation',async()=>{const controller=new AbortController();let clock=0,yields=0;const source=createNdjsonImportSource({bytes:new Blob([JSON.stringify({text:'x'.repeat(2048)})+'\n']),fingerprint:'timed',chunkBytes:256,now:()=>clock+=10,frameBudgetMs:6,yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(source.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);});
+
+test('a resumed preallocated import retains the exact stable-ID table through parsed and committed checkpoints',async()=>{
+ const stableIds=Object.freeze(Object.fromEntries([...Array.from({length:1000},(_,i)=>['reserved-'+i,'unused-'+i]),['one','id-one'],['two','id-two']])),job={...createImportJob({id:'import',projectId:'project',sourceFingerprint:'immutable-source',parserVersion:1}),stableIds},f=fixture([{id:'one'},{id:'two'}],{root:{document:base,job},ports:{createId(){throw Error('preallocated ID was replaced');}}});await f.runner.start();assert.equal(f.root.job.status,'completed');assert.strictEqual(f.root.job.stableIds,stableIds);for(const cut of f.commits)assert.strictEqual(cut.job.stableIds,stableIds);assert.deepEqual(f.root.job.committedSourceKeys,['one','two']);assert.equal(f.errors.length,0);await f.runner.close();
+});
+
+test('identity allocation keeps the caller port method receiver',async()=>{
+ const f=fixture([{id:'bound'}],{ports:{allocated:0,createId(){return `bound-${++this.allocated}`;}}});await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.job.stableIds.bound,'bound-1');assert.equal(f.ports.allocated,1);assert.equal(f.errors.length,0);await f.runner.close();
+});

tests/headless/recovery-views.test.mjs

@@ -0,0 +1,16 @@
+import {test} from 'node:test';
+import assert from 'node:assert/strict';
+import {createDocument,applyChanges} from '../../src/lib/canvas-core/document.ts';
+import {createLedger} from '../../src/lib/canvas-runtime/ledger.ts';
+import {encodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts';
+const node=id=>({id,incarnation:1,ownerCanvasId:'root',componentId:'text',dataVersion:1,structure:'node',data:{title:id,text:''},geometry:{x:0,y:0,width:100,height:100,rotation:0}}),doc=()=>createDocument('root',[{id:'root',parent:null}],[node('A'),node('B')]);
+const snapshot=(document,rootRevision=0)=>({rootRevision,revision:0,document,operations:new Map(),history:{undo:[],redo:[],entries:new Map()},runtime:{acknowledged:document,versions:new Map(),runs:new Map(),resources:new Map(),resourcePins:new Map(),receipts:new Map(),conflicts:new Map(),ledger:createLedger().getSnapshot()}});
+test('metadata-only roots reuse the document encoding and preserve prior images through a later edit',()=>{
+ const document=doc(),one=encodeRecovery(snapshot(document));let two;const values=Map.prototype.values;try{Map.prototype.values=function(){if(this.size===2)throw Error('metadata root enumerated document records');return values.call(this);};two=encodeRecovery(snapshot(document,1));}finally{Map.prototype.values=values;}assert.strictEqual(one.payload.document,two.payload.document);assert.strictEqual(one.payload.document,one.payload.runtime.acknowledged);const changed=applyChanges(document,[{kind:'field',target:{id:'A',incarnation:1},field:'data.text',before:'',after:'new'}]),three=encodeRecovery(snapshot(changed,2));assert.equal(three.payload.document.elements[0].data.text,'new');assert.equal(one.payload.document.elements[0].data.text,'');assert.strictEqual(three.payload.document.elements[1],one.payload.document.elements[1]);assert.strictEqual(three.payload.document,encodeRecovery(snapshot(changed,3)).payload.document);
+});
+test('unowned document envelopes reread mutable map membership and nested data without freezing callers',()=>{
+ const base=doc(),mutable=node('mutable'),elements=new Map([['mutable',mutable]]),document=Object.freeze({...base,elements}),one=encodeRecovery(snapshot(document));mutable.data.text='changed';elements.set('other',node('other'));const two=encodeRecovery(snapshot(document,1));assert.equal(two.payload.document.elements.length,2);assert.equal(two.payload.document.elements[0].data.text,'changed');assert.equal(one.payload.document.elements.length,1);assert.equal(one.payload.document.elements[0].data.text,'');assert.equal(Object.isFrozen(mutable),false);const lookalike=Object.freeze({...base});assert.notStrictEqual(encodeRecovery(snapshot(lookalike)).payload.document,encodeRecovery(snapshot(lookalike,1)).payload.document);
+});
+test('factory table ownership does not imply deep immutability of values inherited from an external document',()=>{
+ const base=doc(),mutable=node('external'),source={...base,elements:new Map([['external',mutable]])},document=applyChanges(source,[]),one=encodeRecovery(snapshot(document));mutable.data.text='later';source.elements.set('new',node('new'));const two=encodeRecovery(snapshot(document,1));assert.equal(two.payload.document.elements.length,1);assert.equal(two.payload.document.elements[0].data.text,'later');assert.equal(one.payload.document.elements[0].data.text,'');assert.equal(Object.isFrozen(mutable),false);
+});