diff --git a/headless/README.md b/headless/README.md index 56ccbcf..5c0f29c 100644 --- a/headless/README.md +++ b/headless/README.md @@ -751,3 +751,28 @@ The 10000-child copy probe still exceeded its 240-second completion window after this change. The exact hashed-input browser test proves removal of whole-image hashing for ordinary adjacent delta writes; this is not a passed end-to-end copy budget or a claim that encoding, rendering and dispatch costs are all resolved. + + +Import identity capture now reuses a frozen, own-data string dictionary when no +new ID is reserved. One active runner retains its last ID-table reference and a +reverse Set; only the first actual allocation copies both. Mutable dictionaries +and frozen getters are captured defensively, and a late reserve callback cannot +mutate an already published result. The index is cleared when the pump ends, +including pause/cancel/close/restart; the durable `stableIds` remains on ImportJob. +Allocation keeps the caller port method receiver. No task or recovery version +changes, and first-item publication/durability barriers remain in place. + +Recovery also reuses frozen plain document views across metadata-only Roots. +`isFactoryDocument` identifies an envelope with fixed table membership produced +by core constructors; it is not a content-validation or deep-immutability test. +A WeakMap owns only a derived plain view. Its array/tuple containers are frozen +without freezing caller-owned records. The codec still independently checks deep +immutability, so mutable values inherited through `applyChanges` are reevaluated. +Unowned envelopes reread mutable map membership. A test prohibits document-record +enumeration during a metadata-only encode after the initial view is built. + +The 1000-child copy probe completes with these paths, but the 10000-child copy +still exceeds 240 seconds. Removing ID-table copies and metadata-only document +scans is directly verified; a general throughput improvement at 10000 nodes is +not established. Per-record commit, ACK and remaining table traversal costs remain +part of the larger implementation goal. diff --git a/src/lib/canvas-core/document.ts b/src/lib/canvas-core/document.ts index 6a3397d..8a1023f 100644 --- a/src/lib/canvas-core/document.ts +++ b/src/lib/canvas-core/document.ts @@ -1,11 +1,17 @@ import type { CanvasRecord, Change, Document, ElementRecord, Geometry, Id, Registry, Target } from './types.ts'; import { clone, equal, fault, freeze, readMap, validJson } from './values.ts'; +// This owns only the envelope and fixed table membership, not validation or +// immutability of values inherited by applyChanges from an external document. +const factoryDocuments=new WeakSet(); +function documentEnvelope(document:Document):Document{const value=Object.freeze(document);factoryDocuments.add(value);return value;} +export const isFactoryDocument=(document:Document)=>factoryDocuments.has(document); + export function createDocument(projectId: Id, canvases: readonly CanvasRecord[], elements: readonly ElementRecord[] = [], tombstones: ReadonlyMap = new Map()): Document { for (const element of elements) { const id = element.id; if (!validJson(element)) throw fault('invalid_element', [id]); } for (const canvas of canvases) if (!validJson(canvas)) throw fault('invalid_canvas'); if (!projectId || new Set(canvases.map(item => item.id)).size !== canvases.length || new Set(elements.map(item => item.id)).size !== elements.length) throw fault('duplicate_identity'); - return Object.freeze({ projectId, canvases: readMap(new Map(canvases.map(item => [item.id, freeze(clone(item))]))), elements: readMap(new Map(elements.map(item => [item.id, freeze(clone(item))]))), tombstones: readMap(tombstones) }); + return documentEnvelope({ projectId, canvases: readMap(new Map(canvases.map(item => [item.id, freeze(clone(item))]))), elements: readMap(new Map(elements.map(item => [item.id, freeze(clone(item))]))), tombstones: readMap(tombstones) }); } export function geometryValid(value: Geometry): boolean { return !!value && typeof value === 'object' && [value.x, value.y, value.width, value.height, value.rotation].every(Number.isFinite) && value.width > 0 && value.height > 0 && value.rotation >= 0 && value.rotation < 360; @@ -115,7 +121,7 @@ export function applyChanges(document: Document, changes: readonly Change[], con } else canvases.delete(change.id); } else throw fault('invalid_change'); } - return Object.freeze({ projectId: document.projectId, elements: readMap(elements), canvases: readMap(canvases), tombstones: readMap(tombstones) }); + return documentEnvelope({ projectId: document.projectId, elements: readMap(elements), canvases: readMap(canvases), tombstones: readMap(tombstones) }); } export function invert(changes: readonly Change[]): readonly Change[] { return changes.slice().reverse().map(change => { diff --git a/src/lib/canvas-runtime/import-identities.ts b/src/lib/canvas-runtime/import-identities.ts new file mode 100644 index 0000000..f5eb7ee --- /dev/null +++ b/src/lib/canvas-runtime/import-identities.ts @@ -0,0 +1,27 @@ +import type {Id} from '../canvas-core/types.ts'; +import {fault} from '../canvas-core/values.ts'; +type Ids=Readonly>; +/** One runner's last immutable ID index. Copies are deferred until allocation. */ +export function createImportIdentities(){ + let cached:{ids:Ids;used:Set}|undefined; + const capture=(input:Ids,createId:()=>Id)=>{ + let identity=cached?.ids===input?cached:undefined; + if(!identity){ + const reusable=Object.isFrozen(input)&&Object.values(Object.getOwnPropertyDescriptors(input)).every(property=>Object.hasOwn(property,'value')&&typeof property.value==='string'); + const ids=reusable?input:Object.assign(Object.create(null),input),used=new Set(Object.values(ids));identity={ids,used};if(reusable)cached=identity; + } + let ids=identity.ids,used=identity.used,owned=ids!==input; + return Object.freeze({ + reserve(key:string){ + if(typeof key!=='string'||!key)throw fault('invalid_import_key'); + if(Object.hasOwn(ids,key))return ids[key]; + const id=createId();if(typeof id!=='string'||!id||used.has(id))throw fault('duplicate_import_identity'); + if(!owned){ids=Object.assign(Object.create(null),ids);used=new Set(used);owned=true;} + (ids as Record)[key]=id;used.add(id);return id; + }, + has:(id:Id)=>used.has(id), + finish:():Ids=>owned?{...ids}:ids, + }); + }; + return Object.freeze(Object.assign(capture,{clear(){cached=undefined;}})); +} diff --git a/src/lib/canvas-runtime/imports.ts b/src/lib/canvas-runtime/imports.ts index b2b81b8..24a2de0 100644 --- a/src/lib/canvas-runtime/imports.ts +++ b/src/lib/canvas-runtime/imports.ts @@ -1,3 +1,4 @@ +import {createImportIdentities} from './import-identities.ts'; import {validateImportReadProof,type ImportReadProof} from './import-read-proof.ts'; import {readImportRange,type ImportByteReader} from './import-byte-reader.ts'; import {createImportBudget,type ImportScheduling} from './import-budget.ts'; @@ -130,6 +131,7 @@ export function createImportRunner(ports:ImportRunnerPorts){ if(initial.source?.kind==='duplicate-snapshot')throw fault('import_interpreter_unavailable'); if(initial.source&&(initial.source.integrityVersion===2?!ports.checkpointSource:!initial.source.sha256||initial.source.integrityVersion!==1))throw fault('import_source_integrity_unavailable'); if(!Number.isFinite(budget)||budget<=0||![maxPending,maxBytes,maxBatch].every(value=>Number.isSafeInteger(value)&&value>0))throw fault('invalid_import_budget'); + const identities=createImportIdentities(),allocateIdentity=()=>ports.createId(); let costPerUnit:number|undefined; let active:Promise|undefined,controller:AbortController|undefined,closed=false,restart=false;const holds=new Set(); const report=(error:unknown)=>{try{ports.onError(error);}catch{/* Reporting cannot change imported content. */}}; @@ -146,13 +148,12 @@ export function createImportRunner(ports:ImportRunnerPorts){ function next(job:ImportJob,patch:Partial):ImportJob{const value={...job,...patch,version:3 as const,generation:job.generation+1};if(value.error===undefined)delete value.error;return freeze(checkpointEvidence(value));} function publish(job:ImportJob,patch:Partial){const value=next(job,patch);ports.publish(value,job.generation);if(!equal(current(),value))throw fault('invalid_import_publish');return value;} function prepare(line:ImportLine,job:ImportJob){ - const ids:Record=Object.assign(Object.create(null),job.stableIds),used=new Set(Object.values(ids)); - const reserve=(key:string)=>{if(typeof key!=='string'||!key)throw fault('invalid_import_key');if(Object.hasOwn(ids,key))return ids[key];const id=ports.createId();if(typeof id!=='string'||!id||used.has(id))throw fault('duplicate_import_identity');ids[key]=id;used.add(id);return id;}; - const unit=ports.prepare(line,job,reserve); + const ids=identities(job.stableIds,allocateIdentity); + const unit=ports.prepare(line,job,ids.reserve); if(!validJson(unit)||typeof unit.key!=='string'||!unit.key||!Array.isArray(unit.changes)||!Array.isArray(unit.dependencies)||!unit.changes.length||unit.changes.length>10000||new Set(unit.dependencies).size!==unit.dependencies.length||unit.dependencies.some(key=>typeof key!=='string'||!key))throw fault('invalid_import_unit'); - for(const change of unit.changes)if(change.kind!=='field'&&change.before===null&&change.after&&!used.has(change.id))throw fault('unstable_import_identity'); + for(const change of unit.changes)if(change.kind!=='field'&&change.before===null&&change.after&&!ids.has(change.id))throw fault('unstable_import_identity'); if(new TextEncoder().encode(JSON.stringify(unit)).byteLength>maxBytes)throw fault('import_unit_capacity'); - return{unit:freeze(clone(unit)),ids:{...ids}}; + return{unit:freeze(clone(unit)),ids:ids.finish()}; } const checkpoint=async(stage:'parsed'|'committed'='committed')=>{if(ports.optimisticLocal&&ports.awaitDurabilityBudget)await ports.awaitDurabilityBudget(stage);else await ports.persist();}; const digest=async(unit:ImportUnit)=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(unit))))].map(value=>value.toString(16).padStart(2,'0')).join(''); @@ -273,7 +274,7 @@ export function createImportRunner(ports:ImportRunnerPorts){ if(closed||controller?.signal.aborted)return; try{const job=current();if(['running','completed','partial'].includes(job.status)){publish(job,{status:'paused',error:(error as {code?:string}).code??'import_failed'});try{await ports.persist();}catch{/* Storage may still be unavailable. */}}}catch{/* A replaced identity cannot receive an old error. */} report(error); - }).finally(()=>{active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}}); + }).finally(()=>{identities.clear();active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}}); return active; } return Object.freeze({ diff --git a/src/lib/canvas-runtime/recovery.ts b/src/lib/canvas-runtime/recovery.ts index 2b164b8..5134b16 100644 --- a/src/lib/canvas-runtime/recovery.ts +++ b/src/lib/canvas-runtime/recovery.ts @@ -13,7 +13,7 @@ import type { Change, CoreSnapshot, Document, HistoryEntry, HistoryState, Id, Js import { validateHistoryReplay, replayHistoryEvidence, sameHistory } from './history-replay.ts'; import type { RecoveryAudit } from './conflicts.ts'; import { applyHistoryTransition, validateHistoryTransition, type HistoryTransition } from '../canvas-core/history-transition.ts'; -import { createDocument, validateDocument } from '../canvas-core/document.ts'; +import { createDocument, validateDocument, isFactoryDocument } from '../canvas-core/document.ts'; import { fault, freeze, readMap, validJson, equal } from '../canvas-core/values.ts'; import type { EngineSnapshot, RuntimeState } from './contracts.ts'; import type { FrozenRequest, LedgerSnapshot, OperationRecord } from './ledger.ts'; @@ -21,7 +21,16 @@ import { validateRunInput, type RunRecord } from './runs.ts'; import type { ResourceRecord } from './resources.ts'; import type { RecoveryImage } from './storage-port.ts'; -const documentValue = (document: Document) => ({ projectId: document.projectId, canvases: [...document.canvases.values()], elements: [...document.elements.values()], tombstones: [...document.tombstones] }); +function encodeDocumentValue(document:Document){ + // Freeze our own containers only. The codec still independently proves whether + // each nested value can be reused, including external mutable records. + return Object.freeze({projectId:document.projectId,canvases:Object.freeze([...document.canvases.values()]),elements:Object.freeze([...document.elements.values()]),tombstones:Object.freeze([...document.tombstones].map(pair=>Object.freeze(pair)))}); +} +const documentViews=new WeakMap>(); +function documentValue(document:Document){ + if(!isFactoryDocument(document))return encodeDocumentValue(document); + let value=documentViews.get(document);if(!value){value=encodeDocumentValue(document);documentViews.set(document,value);}return value; +} /** Plain records only: functions, credentials, renderer objects, URLs and Map methods never serialize. */ export function encodeRecovery(snapshot: EngineSnapshot): RecoveryImage { const payload = { diff --git a/tests/headless/import-identities.test.mjs b/tests/headless/import-identities.test.mjs new file mode 100644 index 0000000..ec622d9 --- /dev/null +++ b/tests/headless/import-identities.test.mjs @@ -0,0 +1,14 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createImportIdentities} from '../../src/lib/canvas-runtime/import-identities.ts'; +test('preallocated 10000-ID tables keep their identity across records without allocating or rescanning',()=>{ + let reads=0;const raw=Object.freeze(Object.fromEntries(Array.from({length:10000},(_,i)=>['key'+i,'id'+i]))),ids=new Proxy(raw,{ownKeys(target){reads++;return Reflect.ownKeys(target);}}),capture=createImportIdentities(),allocate=()=>{throw Error('unexpected allocation');}; + const first=capture(ids,allocate);assert.equal(first.reserve('key0'),'id0');assert.strictEqual(first.finish(),ids);const checked=reads;for(let i=0;i<10000;i++){const row=capture(ids,allocate);assert.equal(row.reserve('key'+i),'id'+i);assert.ok(row.has('id'+i));assert.strictEqual(row.finish(),ids);}assert.equal(reads,checked);capture.clear();capture(ids,allocate);assert.ok(reads>checked); +}); +test('first allocation detaches the dictionary and reverse set, and a late callback cannot change its published result',()=>{ + const ids=Object.freeze({old:'old-id'}),capture=createImportIdentities();let next=0;const row=capture(ids,()=>`new-${++next}`);row.reserve('added');const result=row.finish();assert.notStrictEqual(result,ids);row.reserve('late');assert.deepEqual(result,{old:'old-id',added:'new-1'});assert.deepEqual(ids,{old:'old-id'});const sibling=capture(ids,()=> 'new-1');assert.equal(sibling.has('new-1'),false);assert.equal(sibling.reserve('other'),'new-1');assert.throws(()=>capture(ids,()=> 'old-id').reserve('duplicate'),/duplicate_import_identity/); +}); +test('mutable and frozen getter dictionaries are captured defensively rather than reused',()=>{ + const capture=createImportIdentities(),mutable={one:'first'},first=capture(mutable,()=> 'new');mutable.one='second';assert.equal(first.reserve('one'),'first');assert.deepEqual(first.finish(),{one:'first'});assert.equal(capture(mutable,()=> 'new').reserve('one'),'second'); + let current='old',reads=0;const dynamic=Object.freeze({get one(){reads++;return current;}}),one=capture(dynamic,()=> 'new');current='changed';assert.equal(one.reserve('one'),'old');assert.equal(reads,1);assert.equal(capture(dynamic,()=> 'new').reserve('one'),'changed');assert.equal(reads,2); +}); diff --git a/tests/headless/imports.test.mjs b/tests/headless/imports.test.mjs index 6005746..7cdd037 100644 --- a/tests/headless/imports.test.mjs +++ b/tests/headless/imports.test.mjs @@ -143,3 +143,11 @@ test('async preparation denial isolates independent groups without publishing th test('a generation change during async preparation prevents the old attempt from committing or changing progress',async()=>{const gate=deferred(),entered=deferred(),f=fixture([{id:'later'}],{ports:{async beforeCommit(){entered.resolve();await gate.promise;return{enabled:true};}}});const running=f.runner.start();await entered.promise;f.root={...f.root,job:{...f.root.job,generation:f.root.job.generation+1}};const generation=f.root.job.generation;gate.resolve();await running;assert.equal(f.commits.length,0);assert.equal(f.root.job.generation,generation);assert.equal(f.root.job.committedSourceKeys.length,0);await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.document.elements.size,1);await f.runner.close();}); test('NDJSON checks elapsed time before returning a short scanned record and honors cancellation',async()=>{const controller=new AbortController();let clock=0,yields=0;const source=createNdjsonImportSource({bytes:new Blob([JSON.stringify({text:'x'.repeat(2048)})+'\n']),fingerprint:'timed',chunkBytes:256,now:()=>clock+=10,frameBudgetMs:6,yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(source.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);}); + +test('a resumed preallocated import retains the exact stable-ID table through parsed and committed checkpoints',async()=>{ + const stableIds=Object.freeze(Object.fromEntries([...Array.from({length:1000},(_,i)=>['reserved-'+i,'unused-'+i]),['one','id-one'],['two','id-two']])),job={...createImportJob({id:'import',projectId:'project',sourceFingerprint:'immutable-source',parserVersion:1}),stableIds},f=fixture([{id:'one'},{id:'two'}],{root:{document:base,job},ports:{createId(){throw Error('preallocated ID was replaced');}}});await f.runner.start();assert.equal(f.root.job.status,'completed');assert.strictEqual(f.root.job.stableIds,stableIds);for(const cut of f.commits)assert.strictEqual(cut.job.stableIds,stableIds);assert.deepEqual(f.root.job.committedSourceKeys,['one','two']);assert.equal(f.errors.length,0);await f.runner.close(); +}); + +test('identity allocation keeps the caller port method receiver',async()=>{ + const f=fixture([{id:'bound'}],{ports:{allocated:0,createId(){return `bound-${++this.allocated}`;}}});await f.runner.start();assert.equal(f.root.job.status,'completed');assert.equal(f.root.job.stableIds.bound,'bound-1');assert.equal(f.ports.allocated,1);assert.equal(f.errors.length,0);await f.runner.close(); +}); diff --git a/tests/headless/recovery-views.test.mjs b/tests/headless/recovery-views.test.mjs new file mode 100644 index 0000000..5dbd6f2 --- /dev/null +++ b/tests/headless/recovery-views.test.mjs @@ -0,0 +1,16 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createDocument,applyChanges} from '../../src/lib/canvas-core/document.ts'; +import {createLedger} from '../../src/lib/canvas-runtime/ledger.ts'; +import {encodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts'; +const node=id=>({id,incarnation:1,ownerCanvasId:'root',componentId:'text',dataVersion:1,structure:'node',data:{title:id,text:''},geometry:{x:0,y:0,width:100,height:100,rotation:0}}),doc=()=>createDocument('root',[{id:'root',parent:null}],[node('A'),node('B')]); +const snapshot=(document,rootRevision=0)=>({rootRevision,revision:0,document,operations:new Map(),history:{undo:[],redo:[],entries:new Map()},runtime:{acknowledged:document,versions:new Map(),runs:new Map(),resources:new Map(),resourcePins:new Map(),receipts:new Map(),conflicts:new Map(),ledger:createLedger().getSnapshot()}}); +test('metadata-only roots reuse the document encoding and preserve prior images through a later edit',()=>{ + const document=doc(),one=encodeRecovery(snapshot(document));let two;const values=Map.prototype.values;try{Map.prototype.values=function(){if(this.size===2)throw Error('metadata root enumerated document records');return values.call(this);};two=encodeRecovery(snapshot(document,1));}finally{Map.prototype.values=values;}assert.strictEqual(one.payload.document,two.payload.document);assert.strictEqual(one.payload.document,one.payload.runtime.acknowledged);const changed=applyChanges(document,[{kind:'field',target:{id:'A',incarnation:1},field:'data.text',before:'',after:'new'}]),three=encodeRecovery(snapshot(changed,2));assert.equal(three.payload.document.elements[0].data.text,'new');assert.equal(one.payload.document.elements[0].data.text,'');assert.strictEqual(three.payload.document.elements[1],one.payload.document.elements[1]);assert.strictEqual(three.payload.document,encodeRecovery(snapshot(changed,3)).payload.document); +}); +test('unowned document envelopes reread mutable map membership and nested data without freezing callers',()=>{ + const base=doc(),mutable=node('mutable'),elements=new Map([['mutable',mutable]]),document=Object.freeze({...base,elements}),one=encodeRecovery(snapshot(document));mutable.data.text='changed';elements.set('other',node('other'));const two=encodeRecovery(snapshot(document,1));assert.equal(two.payload.document.elements.length,2);assert.equal(two.payload.document.elements[0].data.text,'changed');assert.equal(one.payload.document.elements.length,1);assert.equal(one.payload.document.elements[0].data.text,'');assert.equal(Object.isFrozen(mutable),false);const lookalike=Object.freeze({...base});assert.notStrictEqual(encodeRecovery(snapshot(lookalike)).payload.document,encodeRecovery(snapshot(lookalike,1)).payload.document); +}); +test('factory table ownership does not imply deep immutability of values inherited from an external document',()=>{ + const base=doc(),mutable=node('external'),source={...base,elements:new Map([['external',mutable]])},document=applyChanges(source,[]),one=encodeRecovery(snapshot(document));mutable.data.text='later';source.elements.set('new',node('new'));const two=encodeRecovery(snapshot(document,1));assert.equal(two.payload.document.elements.length,1);assert.equal(two.payload.document.elements[0].data.text,'later');assert.equal(one.payload.document.elements[0].data.text,'');assert.equal(Object.isFrozen(mutable),false); +});