diff --git a/src/lib/canvas-core/duplicate.ts b/src/lib/canvas-core/duplicate.ts index ed39d2f..dca1eec 100644 --- a/src/lib/canvas-core/duplicate.ts +++ b/src/lib/canvas-core/duplicate.ts @@ -7,7 +7,7 @@ import { clone, fault, freeze } from './values.ts'; export type DuplicatePlan = { readonly version: 1; readonly roots: readonly Id[]; readonly elementIds: Readonly>; readonly canvasIds: Readonly>; readonly changes: readonly Change[] }; export type DuplicateOptions = { readonly createId: () => Id; readonly worldOffset?: Point; readonly mapping?: { readonly elements: Readonly>; readonly canvases: Readonly> } }; -function copyData(element: ElementRecord, registry: Registry, elements: ReadonlyMap, canvases: ReadonlyMap, document: Document): Data { +export function remapCopyData(element: ElementRecord, registry: Registry, elements: ReadonlyMap, canvases: ReadonlyMap, document: Document): Data { const definition = registry.components.get(element.componentId), schema = definition?.copy; if (registry.resolve(element) !== 'known' || !schema) throw fault('copy_schema_unavailable', [element.id]); const data = clone(element.data) as Record; @@ -84,7 +84,7 @@ export function planDuplicate(document: Document, registry: Registry, selection: const transform = canvasTransform(document, old.ownerCanvasId), zero = toLocal(transform, { x: 0, y: 0 }), shift = toLocal(transform, offset); geometry = { ...geometry, x: geometry.x + shift.x - zero.x, y: geometry.y + shift.y - zero.y }; } - const after = { ...clone(old), id: elementIds.get(old.id)!, incarnation: 1, ownerCanvasId: canvasIds.get(old.ownerCanvasId) ?? old.ownerCanvasId, data: copyData(old, registry, elementIds, canvasIds, document), geometry }; + const after = { ...clone(old), id: elementIds.get(old.id)!, incarnation: 1, ownerCanvasId: canvasIds.get(old.ownerCanvasId) ?? old.ownerCanvasId, data: remapCopyData(old, registry, elementIds, canvasIds, document), geometry }; changes.push({ kind: 'element', id: after.id, before: null, after }); } } @@ -105,7 +105,7 @@ export function planDuplicate(document: Document, registry: Registry, selection: const from = displacement(edge.source.elementId, source.elementId), to = displacement(edge.target.elementId, target.elementId), knots = path.knots; path = { ...path, knots: knots.map((raw, index) => { const knot = raw as Record, point = knot.position as { x: number; y: number }, world = toWorld(transform, point), t = (index + 1) / (knots.length + 1); return { ...knot, position: toLocal(transform, { x: world.x + from.x * (1 - t) + to.x * t, y: world.y + from.y * (1 - t) + to.y * t }) }; }) }; } - const after: ElementRecord = { ...clone(edge), id: elementIds.get(edge.id)!, incarnation: 1, ownerCanvasId: owner, source, target, path, data: copyData(edge, registry, elementIds, canvasIds, document) }; + const after: ElementRecord = { ...clone(edge), id: elementIds.get(edge.id)!, incarnation: 1, ownerCanvasId: owner, source, target, path, data: remapCopyData(edge, registry, elementIds, canvasIds, document) }; changes.push({ kind: 'element', id: after.id, before: null, after }); } validateDocument(applyChanges(document, changes), registry); diff --git a/src/lib/canvas-runtime/import-native.ts b/src/lib/canvas-runtime/import-native.ts new file mode 100644 index 0000000..e416b67 --- /dev/null +++ b/src/lib/canvas-runtime/import-native.ts @@ -0,0 +1,51 @@ +import type { Document, ElementRecord, Id, Json, Registry } from '../canvas-core/types.ts'; +import { remapCopyData } from '../canvas-core/duplicate.ts'; +import { clone, fault } from '../canvas-core/values.ts'; +import type { ImportJob, ImportLine, ImportUnit } from './imports.ts'; + +export function assertNativeImportTarget(job:ImportJob,document:Document){ + const source=job.source,canvas=source?document.canvases.get(source.targetCanvasId):undefined; + if(!source||!canvas)throw fault('import_target_missing'); + if(source.targetEntry?(canvas.parent?.entryElementId!==source.targetEntry.id||document.elements.get(source.targetEntry.id)?.incarnation!==source.targetEntry.incarnation):canvas.parent!==null)throw fault('import_target_changed'); + return source; +} +export function assertNativeImportDependencies(job:ImportJob,document:Document,dependencies:readonly string[]){ + for(const key of dependencies)if(job.committedSourceKeys.includes(key)){ + const id=job.stableIds[key],binding=job.bindings?.[id]; + if(!binding)throw fault('import_dependency_changed'); + if(binding.kind==='element'){if(document.elements.get(id)?.incarnation!==binding.incarnation)throw fault('import_dependency_changed',[id]);} + else{const canvas=document.canvases.get(id),entry=binding.entryElementId,entryBinding=entry?job.bindings?.[entry]:undefined;if(!canvas||canvas.parent?.entryElementId!==entry||entry&&(entryBinding?.kind!=='element'||document.elements.get(entry)?.incarnation!==entryBinding.incarnation))throw fault('import_dependency_changed',[id]);} + } +} + +/** Native NDJSON records use source IDs; $target names the explicitly chosen existing canvas. */ +export function prepareNativeImport(line:ImportLine,job:ImportJob,reserve:(key:string)=>Id,document:Document,registry:Registry):ImportUnit { + const source=assertNativeImportTarget(job,document); + const raw=line.value as Record|undefined; + if(!raw||Array.isArray(raw)||!['element','canvas'].includes(raw.kind as string)||typeof raw.key!=='string'||!raw.key||raw.key==='$target'||!raw.record||typeof raw.record!=='object'||Array.isArray(raw.record))throw fault('invalid_native_import_record'); + const record=clone(raw.record) as Record;if(record.id!==raw.key)throw fault('import_source_identity_mismatch'); + const dependencies=new Set(),elementIds=new Map(),canvasIds=new Map(); + const element=(id:Json):Id=>{if(typeof id!=='string'||!id)throw fault('invalid_import_reference');const key=`element:${id}`;dependencies.add(key);const mapped=reserve(key);elementIds.set(id,mapped);return mapped;}; + const owner=(id:Json):Id=>{if(id==='$target'){canvasIds.set('$target',source.targetCanvasId);return source.targetCanvasId;}if(typeof id!=='string'||!id)throw fault('invalid_import_reference');const key=`canvas:${id}`;dependencies.add(key);const mapped=reserve(key);canvasIds.set(id,mapped);return mapped;}; + const key=`${raw.kind}:${raw.key}`,id=reserve(key); + if(raw.kind==='canvas'){ + const parent=record.parent as Record|null; + if(!parent||typeof parent!=='object'||Array.isArray(parent))throw fault('invalid_import_canvas'); + const after={...record,id,parent:{...parent,canvasId:owner(parent.canvasId),entryElementId:element(parent.entryElementId)}}; + dependencies.delete(key);return{key,dependencies:[...dependencies],changes:[{kind:'canvas',id,before:null,after:after as never}]}; + } + const original=record as unknown as ElementRecord,definition=registry.components.get(original.componentId); + if(registry.resolve(original)!=='known'||!definition?.copy||!original.data||typeof original.data!=='object'||Array.isArray(original.data))throw fault('import_component_unavailable'); + for(const ref of definition.copy.references){ + let value:Json|undefined=original.data;for(const part of ref.path){if(!value||typeof value!=='object'||!Object.hasOwn(value,part)){value=undefined;break;}value=(value as Record)[part];} + if(value===undefined||value===null)continue; + if(ref.many&&!Array.isArray(value))throw fault('invalid_import_reference'); + for(const item of ref.many?value as Json[]:[value]){if(typeof item!=='string'||!item)throw fault('invalid_import_reference');if(ref.target==='element')element(item);else owner(item);} + } + const after={...record,id,incarnation:1,ownerCanvasId:owner(original.ownerCanvasId),data:remapCopyData(original,registry,elementIds,canvasIds,document)}; + if(original.structure==='connection'){ + if(!original.source||!original.target)throw fault('invalid_import_endpoint'); + Object.assign(after,{source:{...original.source,elementId:element(original.source.elementId)},target:{...original.target,elementId:element(original.target.elementId)}}); + }else if(original.structure!=='node')throw fault('invalid_import_structure'); + dependencies.delete(key);return{key,dependencies:[...dependencies],changes:[{kind:'element',id,before:null,after:after as ElementRecord}]}; +} diff --git a/src/lib/canvas-runtime/imports.ts b/src/lib/canvas-runtime/imports.ts new file mode 100644 index 0000000..2966449 --- /dev/null +++ b/src/lib/canvas-runtime/imports.ts @@ -0,0 +1,192 @@ +import type { Change, Id, Json, Result } from '../canvas-core/types.ts'; +import { clone, equal, fault, freeze, validJson } from '../canvas-core/values.ts'; + +export type ImportLine = { readonly cursor: number; readonly nextCursor: number; readonly key: string; readonly value?: Json; readonly error?: string; readonly skip?: boolean }; +export type ImportSource = { readonly fingerprint: string; readonly parserVersion: number; read(cursor: number, signal: AbortSignal): Promise }; +export type ImportUnit = { readonly key: string; readonly dependencies: readonly string[]; readonly changes: readonly Change[] }; +export type ImportLocator = { readonly key: string; readonly cursor: number; readonly nextCursor: number; readonly dependencies: readonly string[]; readonly digest: string; readonly byteLength: number; readonly changeCount: number }; +export type ImportFailure = { readonly key: string; readonly cursor: number; readonly code: string; readonly dependencies?: readonly string[] }; +export type NativeImportSource = { readonly kind: 'native-ndjson'; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} }; +export type ImportBinding = {readonly kind:'element';readonly incarnation:number}|{readonly kind:'canvas';readonly entryElementId?:Id}; +export type ImportJob = { + readonly version: 1; readonly id: Id; readonly projectId: Id; readonly sourceFingerprint: string; readonly parserVersion: number; + readonly source?: NativeImportSource; + readonly generation: number; readonly status: 'paused' | 'running' | 'completed' | 'partial' | 'cancelled'; readonly cursor: number; readonly eof: boolean; + readonly bindings?: Readonly>; + readonly stableIds: Readonly>; readonly pending: readonly ImportLocator[]; readonly committedSourceKeys: readonly string[]; readonly failures: readonly ImportFailure[]; readonly error?: string; +}; +export function createImportJob(input: Pick): ImportJob { + if (![input.id,input.projectId,input.sourceFingerprint].every(value=>typeof value==='string'&&!!value) || !Number.isSafeInteger(input.parserVersion) || input.parserVersion<1) throw fault('invalid_import_job'); + return freeze({ id: input.id, projectId: input.projectId, sourceFingerprint: input.sourceFingerprint, parserVersion: input.parserVersion, ...(input.source?{source:clone(input.source)}:{}), version: 1, generation: 0, status: 'paused', cursor: 0, eof: false, bindings: {}, stableIds: {}, pending: [], committedSourceKeys: [], failures: [] }); +} +export function validateImportJob(job: ImportJob): void { + if(!job||typeof job!=='object'||!validJson(job)||!job.stableIds||typeof job.stableIds!=='object'||Array.isArray(job.stableIds)||![job.pending,job.committedSourceKeys,job.failures].every(Array.isArray)||job.error!==undefined&&typeof job.error!=='string')throw fault('invalid_import_job'); + if (job.version !== 1) throw fault('import_interpreter_unavailable'); + createImportJob(job); + if (!Number.isSafeInteger(job.generation) || job.generation<0 || !Number.isSafeInteger(job.cursor) || job.cursor<0 || typeof job.eof!=='boolean' || !['paused','running','completed','partial','cancelled'].includes(job.status)) throw fault('invalid_import_job'); + if(job.source&&(job.source.kind!=='native-ndjson'||job.source.name!==undefined&&(typeof job.source.name!=='string'||job.source.name.length>255)||!job.source.fileId||!job.source.targetCanvasId||!Number.isSafeInteger(job.source.byteLength)||job.source.byteLength<0||job.source.targetEntry&&(!job.source.targetEntry.id||!Number.isSafeInteger(job.source.targetEntry.incarnation)||job.source.targetEntry.incarnation<1)))throw fault('invalid_import_source'); + const keys=new Set(),ids=Object.values(job.stableIds),stableIdSet=new Set(ids); + if (stableIdSet.size!==ids.length || ids.some(id=>typeof id!=='string'||!id) || Object.keys(job.stableIds).some(key=>!key)) throw fault('invalid_import_job'); + for(const[id,binding]of Object.entries(job.bindings??{}))if(!id||!binding||!stableIdSet.has(id)||!['element','canvas'].includes(binding.kind)||binding.kind==='element'&&(!Number.isSafeInteger(binding.incarnation)||binding.incarnation<1)||binding.kind==='canvas'&&binding.entryElementId!==undefined&&typeof binding.entryElementId!=='string')throw fault('invalid_import_job'); + for(const key of job.committedSourceKeys){if(!key||keys.has(key))throw fault('invalid_import_job');keys.add(key);} + if(job.source)for(const key of job.committedSourceKeys){const id=job.stableIds[key],binding=job.bindings?.[id];if(!Object.hasOwn(job.stableIds,key)||!binding||!key.startsWith(`${binding.kind}:`)||binding.kind==='canvas'&&(!binding.entryElementId||job.bindings?.[binding.entryElementId]?.kind!=='element'))throw fault('invalid_import_job');} + for(const unit of job.pending){if(!unit||typeof unit.key!=='string'||!unit.key||!Array.isArray(unit.dependencies)||keys.has(unit.key)||!Number.isSafeInteger(unit.cursor)||unit.cursor<0||!Number.isSafeInteger(unit.nextCursor)||unit.nextCursor<=unit.cursor||unit.nextCursor>job.cursor||!(/^[a-f0-9]{64}$/.test(unit.digest))||!Number.isSafeInteger(unit.byteLength)||unit.byteLength<1||!Number.isSafeInteger(unit.changeCount)||unit.changeCount<1||unit.changeCount>10000||new Set(unit.dependencies).size!==unit.dependencies.length||unit.dependencies.some(key=>typeof key!=='string'||!key))throw fault('invalid_import_job');keys.add(unit.key);} + if(job.status==='partial'&&!job.eof)throw fault('invalid_import_job'); + if(job.status==='completed'&&(!job.eof||job.pending.length||job.failures.length))throw fault('invalid_import_job'); + for(const failure of job.failures)if(!failure||typeof failure.key!=='string'||!failure.key||typeof failure.code!=='string'||!failure.code||!Number.isSafeInteger(failure.cursor)||failure.cursor<0||failure.cursor>job.cursor)throw fault('invalid_import_job'); +} +const yieldTask=()=>new Promise(resolve=>setTimeout(resolve,0)); + +/** Seekable bounded NDJSON parser. Fingerprint must identify the host's immutable, durable byte source. */ +export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number;yieldTask?:()=>Promise}):ImportSource { + const max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16*1024; + if(!(options.bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source'); + const sourceBytes=options.bytes,sourceYield=options.yieldTask??yieldTask; + let cache={offset:-1,bytes:new Uint8Array()}; + return Object.freeze({fingerprint:options.fingerprint,parserVersion:1, + async read(cursor:number,signal:AbortSignal):Promise{ + if(!Number.isSafeInteger(cursor)||cursor<0||cursor>sourceBytes.size)throw fault('invalid_import_cursor'); + if(signal.aborted)throw fault('import_aborted');if(cursor===sourceBytes.size)return null; + const parts:Uint8Array[]=[],start=cursor;let total=0,oversized=false,scanned=0; + while(cursor=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:new Uint8Array(await sourceBytes.slice(cursor,cursor+chunk).arrayBuffer())}; + if(signal.aborted)throw fault('import_aborted'); + const begin=cursor-cache.offset,newline=cache.bytes.indexOf(10,begin),end=newline<0?cache.bytes.length:newline,part=cache.bytes.subarray(begin,end);total+=part.length; + if(total>max){oversized=true;parts.length=0;}else if(!oversized)parts.push(part); + cursor=cache.offset+end+(newline<0?0:1);scanned+=end-begin; + if(newline>=0||cursor===sourceBytes.size)break; + if(scanned>=64*1024){scanned=0;await sourceYield();} + } + const base={cursor:start,nextCursor:cursor,key:`byte:${start}`}; + if(oversized)return{...base,error:'record_too_large'}; + const bytes=new Uint8Array(total);let offset=0;for(const part of parts){bytes.set(part,offset);offset+=part.length;} + let text:string;try{text=new TextDecoder('utf-8',{fatal:true}).decode(bytes);}catch{return{...base,error:'invalid_utf8'};} + if(!text.trim())return{...base,skip:true}; + try{const value=JSON.parse(text);return validJson(value)?{...base,value}:{...base,error:'invalid_json_value'};}catch{return{...base,error:'invalid_json'};} + }, + }); +} + +export type ImportRunnerPorts={ + source:ImportSource;getJob():ImportJob;createId():Id; + /** Synchronous compare-and-publish; never write a second private job store. */ + publish(next:ImportJob,expectedGeneration:number):void; + /** Pure decoding; reserve() returns stable source-key identities. No upload/run effects here. */ + prepare(line:ImportLine,job:ImportJob,reserve:(sourceKey:string)=>Id):ImportUnit; + /** Publish all document changes AND next job in the same StateRoot, or publish nothing. */ + commit(units:readonly ImportUnit[],next:ImportJob,expectedGeneration:number):Result; + persist():Promise;onError(error:unknown):void; + /** Host must durably gate every network write on the same document/job cut. */ + optimisticLocal?:boolean; + now?:()=>number;yieldTask?:()=>Promise;frameBudgetMs?:number;maxPending?:number;maxPendingBytes?:number;maxBatch?:number; +}; + +/** Cooperative producer/consumer. Pending work stores seekable locators, not unbounded decoded payloads. */ +export function createImportRunner(ports:ImportRunnerPorts){ + const initial=ports.getJob(),identity={id:initial.id,projectId:initial.projectId,sourceFingerprint:initial.sourceFingerprint,parserVersion:initial.parserVersion},budget=ports.frameBudgetMs??6,maxPending=ports.maxPending??4096,maxBytes=ports.maxPendingBytes??1024*1024,maxBatch=ports.maxBatch??32,now=ports.now??(()=>performance.now()); + validateImportJob(initial); + if(!Number.isFinite(budget)||budget<=0||![maxPending,maxBytes,maxBatch].every(value=>Number.isSafeInteger(value)&&value>0))throw fault('invalid_import_budget'); + let costPerUnit:number|undefined; + let active:Promise|undefined,controller:AbortController|undefined,closed=false,restart=false;const holds=new Set(); + const report=(error:unknown)=>{try{ports.onError(error);}catch{/* Reporting cannot change imported content. */}}; + function current(){const job=ports.getJob();if(job.id!==identity.id||job.projectId!==identity.projectId||job.sourceFingerprint!==identity.sourceFingerprint||job.parserVersion!==identity.parserVersion)throw fault('import_identity_changed');return job;} + function next(job:ImportJob,patch:Partial):ImportJob{const value={...job,...patch,generation:job.generation+1};if(value.error===undefined)delete value.error;return freeze(value);} + function publish(job:ImportJob,patch:Partial){const value=next(job,patch);ports.publish(value,job.generation);if(!equal(current(),value))throw fault('invalid_import_publish');return value;} + function prepare(line:ImportLine,job:ImportJob){ + const ids:Record=Object.assign(Object.create(null),job.stableIds),used=new Set(Object.values(ids)); + const reserve=(key:string)=>{if(typeof key!=='string'||!key)throw fault('invalid_import_key');if(Object.hasOwn(ids,key))return ids[key];const id=ports.createId();if(typeof id!=='string'||!id||used.has(id))throw fault('duplicate_import_identity');ids[key]=id;used.add(id);return id;}; + const unit=ports.prepare(line,job,reserve); + if(!validJson(unit)||typeof unit.key!=='string'||!unit.key||!Array.isArray(unit.changes)||!Array.isArray(unit.dependencies)||!unit.changes.length||unit.changes.length>10000||new Set(unit.dependencies).size!==unit.dependencies.length||unit.dependencies.some(key=>typeof key!=='string'||!key))throw fault('invalid_import_unit'); + for(const change of unit.changes)if(change.kind!=='field'&&change.before===null&&change.after&&!used.has(change.id))throw fault('unstable_import_identity'); + if(new TextEncoder().encode(JSON.stringify(unit)).byteLength>maxBytes)throw fault('import_unit_capacity'); + return{unit:freeze(clone(unit)),ids:{...ids}}; + } + const digest=async(unit:ImportUnit)=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(unit))))].map(value=>value.toString(16).padStart(2,'0')).join(''); + function completed(job:ImportJob,units:readonly ImportUnit[],pending:readonly ImportLocator[]){ + const bindings:Record=Object.assign(Object.create(null),job.bindings??{}); + for(const unit of units)for(const change of unit.changes)if(change.kind==='element'&&change.before===null&&change.after)bindings[change.id]={kind:'element',incarnation:change.after.incarnation};else if(change.kind==='canvas'&&change.before===null&&change.after)bindings[change.id]={kind:'canvas',...(change.after.parent?{entryElementId:change.after.parent.entryElementId}:{})}; + return next(job,{pending,committedSourceKeys:[...job.committedSourceKeys,...units.map(unit=>unit.key)],bindings:{...bindings}}); + } + function commit(units:readonly ImportUnit[],job:ImportJob,value:ImportJob){const start=now(),result=ports.commit(units,value,job.generation),cost=Math.max(.01,(now()-start)/units.length);costPerUnit=costPerUnit===undefined?cost:Math.max(cost,costPerUnit*.8+cost*.2);if(!['applied','rejected'].includes(result?.status)||!equal(current(),result.status==='applied'?value:job))throw fault('invalid_import_commit');return result;} + async function pump(signal:AbortSignal){ + let frame=now(),didWork=false; + if(!ports.optimisticLocal)await ports.persist(); + while(!closed&&!signal.aborted&&!holds.size){ + let job=current();if(job.status!=='running')return; + if(ports.source.fingerprint!==job.sourceFingerprint||ports.source.parserVersion!==job.parserVersion)throw fault('import_source_mismatch'); + if(didWork&&now()-frame>=budget){await(ports.yieldTask??yieldTask)();frame=now();didWork=false;continue;} + didWork=true; + const committed=new Set(job.committedSourceKeys),chosen:ImportLocator[]=[];let chosenBytes=0,chosenChanges=0; + const batchLimit=Math.max(1,Math.min(maxBatch,costPerUnit===undefined?1:Math.floor(budget/(costPerUnit*1.5)))); + for(const locator of job.pending)if(chosen.lengthcommitted.has(key)||chosen.some(unit=>unit.key===key))){chosen.push(locator);chosenBytes+=locator.byteLength;chosenChanges+=locator.changeCount;} + if(chosen.length){ + const units:ImportUnit[]=[];let failed:ImportFailure|undefined; + for(const locator of chosen){ + const line=await ports.source.read(locator.cursor,signal); + if(closed||signal.aborted||current().generation!==job.generation)return; + if(!line||line.error||line.skip||line.nextCursor!==locator.nextCursor)throw fault('import_source_changed'); + const value=prepare(line,job); + if(value.unit.key!==locator.key||!equal(value.unit.dependencies,locator.dependencies)||!equal(value.ids,job.stableIds)||value.unit.changes.length!==locator.changeCount||new TextEncoder().encode(JSON.stringify(value.unit)).byteLength!==locator.byteLength||await digest(value.unit)!==locator.digest)throw fault('import_source_changed'); + if(closed||signal.aborted||current().generation!==job.generation)return; + units.push(value.unit); + } + let proposed=completed(job,units,job.pending.filter(unit=>!chosen.includes(unit))); + let result=commit(units,job,proposed); + if(result.status==='rejected'){ + // Isolate a failed record without discarding valid independent records in the same candidate batch. + for(let index=0;indexjob.committedSourceKeys.includes(key)))continue; + proposed=completed(job,[unit],job.pending.filter(value=>value.key!==unit.key)); + result=commit([unit],job,proposed); + if(result.status==='rejected'){failed={key:unit.key,cursor:locator.cursor,code:result.code};publish(job,{pending:job.pending.filter(value=>value.key!==unit.key),failures:[...job.failures,failed]});} + await ports.persist();if(closed||signal.aborted||current().status!=='running')return; + if(now()-frame>=budget){await(ports.yieldTask??yieldTask)();frame=now();} + } + }else await ports.persist(); + continue; + } + if(job.eof){ + const unresolved=job.pending.map(unit=>({key:unit.key,cursor:unit.cursor,code:'unresolved_dependency',dependencies:unit.dependencies.filter(key=>!committed.has(key))})); + publish(job,{status:job.failures.length||unresolved.length?'partial':'completed',failures:[...job.failures,...unresolved]});await ports.persist();return; + } + if(job.pending.length>=maxPending||new TextEncoder().encode(JSON.stringify(job.pending)).byteLength>=maxBytes){publish(job,{status:'paused',error:'dependency_capacity'});await ports.persist();return;} + const line=await ports.source.read(job.cursor,signal); + if(closed||signal.aborted||current().generation!==job.generation)return; + if(!line){publish(job,{eof:true});await ports.persist();continue;} + if(line.cursor!==job.cursor||!Number.isSafeInteger(line.nextCursor)||line.nextCursor<=job.cursor)throw fault('invalid_import_cursor'); + if(line.skip){publish(job,{cursor:line.nextCursor});await ports.persist();continue;} + if(line.error){publish(job,{cursor:line.nextCursor,failures:[...job.failures,{key:line.key,cursor:line.cursor,code:line.error}]});await ports.persist();continue;} + let patch:Partial; + try{ + const prepared=prepare(line,job),key=prepared.unit.key; + if(job.committedSourceKeys.includes(key)||job.pending.some(unit=>unit.key===key)||job.failures.some(failure=>failure.key===key))throw fault('duplicate_import_source_key'); + const locator={key,cursor:line.cursor,nextCursor:line.nextCursor,dependencies:prepared.unit.dependencies,digest:await digest(prepared.unit),byteLength:new TextEncoder().encode(JSON.stringify(prepared.unit)).byteLength,changeCount:prepared.unit.changes.length}; + patch={cursor:line.nextCursor,stableIds:prepared.ids,pending:[...job.pending,locator]}; + }catch(error){patch={cursor:line.nextCursor,failures:[...job.failures,{key:line.key,cursor:line.cursor,code:(error as {code?:string}).code??'import_decode_failed'}]};} + if(closed||signal.aborted||current().generation!==job.generation)return; + publish(job,patch); + const staged=patch.pending?.at(-1),immediatelyReady=staged?.dependencies.every(key=>job.committedSourceKeys.includes(key)); + if(!ports.optimisticLocal||!immediatelyReady)await ports.persist(); + } + } + function launch(){ + if(closed||holds.size)return Promise.resolve();if(active){restart=true;return active;} + restart=false;controller=new AbortController(); + active=pump(controller.signal).catch(async error=>{ + if(closed||controller?.signal.aborted)return; + try{const job=current();if(job.status==='running'){publish(job,{status:'paused',error:(error as {code?:string}).code??'import_failed'});try{await ports.persist();}catch{/* Storage may still be unavailable. */}}}catch{/* A replaced identity cannot receive an old error. */} + report(error); + }).finally(()=>{active=undefined;controller=undefined;if(restart&&!closed&&!holds.size){try{if(current().status==='running')void launch();}catch{/* Replaced scope stays untouched. */}}}); + return active; + } + return Object.freeze({ + start(){if(closed)throw fault('import_runner_closed');const job=current();if(['completed','partial','cancelled'].includes(job.status))return Promise.resolve();if(job.status!=='running')publish(job,{status:'running',error:undefined});return launch();}, + hold(){const token=Symbol();holds.add(token);controller?.abort();return()=>{if(!holds.delete(token)||holds.size||closed)return;if(current().status==='running')void launch();};}, + async pause(){const job=current();if(job.status==='running'){controller?.abort();publish(job,{status:'paused'});await ports.persist();}}, + async cancel(){const job=current();if(['running','paused'].includes(job.status)){controller?.abort();publish(job,{status:'cancelled',error:undefined});await ports.persist();}}, + async settle(){while(active)await active;}, + async close(){closed=true;controller?.abort();while(active)await active;}, + }); +} diff --git a/tests/headless/imports.test.mjs b/tests/headless/imports.test.mjs new file mode 100644 index 0000000..a2f01e7 --- /dev/null +++ b/tests/headless/imports.test.mjs @@ -0,0 +1,68 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createDocument,applyChanges,validateDocument,createRegistry,builtinComponents,validJson,fault} from '../../src/lib/canvas-core/index.ts'; +import {createImportJob,validateImportJob,createNdjsonImportSource,createImportRunner} from '../../src/lib/canvas-runtime/imports.ts'; +const registry=createRegistry(builtinComponents()),base=createDocument('project',[{id:'canvas',parent:null}],[]); +const deferred=()=>{let resolve;const promise=new Promise(yes=>resolve=yes);return{promise,resolve};}; +const ticks=async()=>{for(let i=0;i<30;i++)await Promise.resolve();}; +const bytes=records=>new Blob([records.map(record=>typeof record==='string'?record:JSON.stringify(record)).join('\n')]); +function fixture(records,options={}){ + const source=options.source??createNdjsonImportSource({bytes:bytes(records),fingerprint:'immutable-source'}),calls=[],errors=[],saved=[],commits=[];let n=0,time=0,root=options.root??{document:base,job:createImportJob({id:'import',projectId:'project',sourceFingerprint:source.fingerprint,parserVersion:1})}; + const f={source,calls,errors,saved,commits,get root(){return root;},set root(value){root=value;},ports:{source,getJob:()=>root.job,createId:()=>`new-${++n}`,now:()=>time, + publish(job,expected){assert.equal(root.job.generation,expected);root={...root,job};}, + prepare(line,job,reserve){time+=2;const input=line.value;if(!input||typeof input!=='object'||!input.id)throw fault('invalid_record');const id=reserve(input.id);return{key:input.id,dependencies:input.dependencies??[],changes:[{kind:'element',id,before:null,after:{id,incarnation:1,structure:'node',ownerCanvasId:'canvas',componentId:'text',dataVersion:1,data:{title:input.title??input.id,text:''},geometry:{x:0,y:0,width:input.bad?-1:100,height:100,rotation:0}}}]};}, + commit(units,job,expected){time+=2;assert.equal(root.job.generation,expected);calls.push(units.map(unit=>unit.key));const before=root;try{const document=applyChanges(root.document,units.flatMap(unit=>unit.changes));validateDocument(document,registry);root={document,job};commits.push(root);return{status:'applied',revision:commits.length,operationIds:[]};}catch(error){assert.equal(root,before);return{status:'rejected',code:error.code??'invalid',targets:[]};}}, + async persist(){validateImportJob(root.job);assert.equal(validJson(root.job),true);saved.push(structuredClone({job:root.job,document:{projectId:root.document.projectId,canvases:[...root.document.canvases.values()],elements:[...root.document.elements.values()],tombstones:[...root.document.tombstones]}}));}, + onError:error=>errors.push(error),yieldTask:async()=>{f.yields++;},...options.ports},yields:0}; + f.runner=createImportRunner(f.ports);return f; +} + +test('NDJSON parses bounded UTF-8 chunks, seeks by byte cursor and reports malformed/oversized records without losing the next line',async()=>{ + const source=createNdjsonImportSource({bytes:bytes([{id:'δΈ­ζ–‡ πŸ˜€'},'{bad}','x'.repeat(30),{id:'ok'}]),fingerprint:'file-1',maxRecordBytes:24,chunkBytes:3}),signal=new AbortController().signal; + let cursor=0;const rows=[];for(;;){const row=await source.read(cursor,signal);if(!row)break;rows.push(row);cursor=row.nextCursor;} + assert.equal(rows[0].value.id,'δΈ­ζ–‡ πŸ˜€');assert.equal(rows[1].error,'invalid_json');assert.equal(rows[2].error,'record_too_large');assert.equal(rows[3].value.id,'ok');assert.deepEqual(await source.read(rows[0].cursor,signal),rows[0]); + const input={bytes:bytes([{id:'old'}]),fingerprint:'fixed'},owned=createNdjsonImportSource(input);input.bytes=bytes([{id:'replacement'}]);assert.equal((await owned.read(0,signal)).value.id,'old'); +}); + +test('records become visible before EOF, task yields real scheduling points and cancellation preserves the committed prefix',async()=>{ + const gate=deferred(),entered=deferred(),raw=bytes([{id:'first'},{id:'second'}]),initial=createNdjsonImportSource({bytes:raw,fingerprint:'immutable-source'});let firstEnd; + firstEnd=(await initial.read(0,new AbortController().signal)).nextCursor; + const source={...initial,async read(cursor,signal){if(cursor===firstEnd){entered.resolve();await gate.promise;}return initial.read(cursor,signal);}},f=fixture([],{source,ports:{frameBudgetMs:3}}); + const running=f.runner.start();await entered.promise;assert.equal(f.root.document.elements.size,1);assert.equal(f.root.job.committedSourceKeys[0],'first');assert.ok(f.yields>0); + await f.runner.cancel();gate.resolve();await running;await f.runner.settle();assert.equal(f.root.job.status,'cancelled');assert.equal(f.root.document.elements.size,1);assert.deepEqual(f.root.job.committedSourceKeys,['first']);await f.runner.close(); +}); + +test('a child encountered before its parent waits by source key, then joins without overwriting user edits',async()=>{ + const f=fixture([{id:'child',dependencies:['parent']},{id:'parent'}]);const real=f.ports.commit;f.ports.commit=(units,next,expected)=>{const result=real(units,next,expected);if(result.status==='applied'&&units.some(unit=>unit.key==='parent')){const id=f.root.job.stableIds.parent,element=f.root.document.elements.get(id);f.root={...f.root,document:applyChanges(f.root.document,[{kind:'field',target:{id,incarnation:1},field:'data.title',before:element.data.title,after:'user edit'}])};}return result;}; + await f.runner.start();assert.equal(f.root.job.status,'completed');assert.deepEqual(f.root.job.committedSourceKeys,['parent','child']);assert.equal(f.root.document.elements.get(f.root.job.stableIds.parent).data.title,'user edit');assert.equal(f.root.job.pending.length,0);assert.equal(f.errors.length,0); +}); + +test('a rejected multi-record batch publishes no prefix, isolates the invalid record and retains independent content',async()=>{ + const f=fixture([{id:'good',dependencies:['parent']},{id:'bad',dependencies:['parent'],bad:true},{id:'parent'},{id:'tail'}]);await f.runner.start();assert.equal(f.root.job.status,'partial');assert.ok(f.calls.some(keys=>keys.includes('good')&&keys.includes('bad')));assert.deepEqual(f.root.job.committedSourceKeys,['parent','good','tail']);assert.equal(f.root.document.elements.has(f.root.job.stableIds.bad),false);assert.equal(f.root.job.failures.find(failure=>failure.key==='bad').code,'invalid_geometry');assert.equal(f.errors.length,0); +}); + +test('record 101 failure preserves prior content and later valid records; missing dependencies report partial without dangling insertion',async()=>{ + const input=Array.from({length:100},(_,index)=>({id:`n-${index}`}));input.push('{bad}',{id:'after'},{id:'edge',dependencies:['missing']});const f=fixture(input);await f.runner.start();assert.equal(f.root.job.status,'partial');assert.equal(f.root.document.elements.size,101);assert.equal(f.root.job.failures.length,2);assert.ok(f.root.job.failures.some(failure=>failure.code==='invalid_json'));assert.ok(f.root.job.failures.some(failure=>failure.code==='unresolved_dependency'));assert.equal(f.root.document.elements.has(f.root.job.stableIds.edge),false);validateImportJob(JSON.parse(JSON.stringify(f.root.job))); +}); + +test('a durable pending locator resumes with the same IDs; a committed key is not inserted again after deletion',async()=>{ + const gate=deferred(),entered=deferred(),f=fixture([{id:'one',title:'original'}]);const persist=f.ports.persist;f.ports.persist=async()=>{await persist();if(f.root.job.pending.length&&!f.root.job.committedSourceKeys.length){entered.resolve();await gate.promise;}}; + const task=f.runner.start();await entered.promise;const cut=f.saved.at(-1),closing=f.runner.close();gate.resolve();await task;await closing; + const document=createDocument(cut.document.projectId,cut.document.canvases,cut.document.elements,new Map(cut.document.tombstones)),resumed=fixture([],{source:f.source,root:{document,job:cut.job},ports:{createId(){throw Error('reallocated identity');}}});await resumed.runner.start();assert.equal(resumed.root.job.status,'completed');assert.equal(resumed.root.document.elements.size,1);assert.equal(resumed.root.job.stableIds.one,cut.job.stableIds.one); + const id=resumed.root.job.stableIds.one,element=resumed.root.document.elements.get(id),deleted=applyChanges(resumed.root.document,[{kind:'element',id,before:element,after:null}]),again=fixture([],{source:f.source,root:{document:deleted,job:{...resumed.root.job,status:'paused',eof:false}},ports:{createId(){throw Error('reallocated identity');}}});await again.runner.start();assert.equal(again.root.document.elements.has(id),false);assert.equal(again.calls.length,0); +}); + +test('changed source bytes or source identity cannot claim a durable pending item',async()=>{ + const oldSource=createNdjsonImportSource({bytes:bytes([{id:'one',title:'aaa'}]),fingerprint:'immutable-source'}),gate=deferred(),entered=deferred(),f=fixture([],{source:oldSource});const persist=f.ports.persist;f.ports.persist=async()=>{await persist();if(f.root.job.pending.length){entered.resolve();await gate.promise;}}; + const task=f.runner.start();await entered.promise;const root=f.root,closing=f.runner.close();gate.resolve();await task;await closing; + for(const [title,fingerprint,code]of[['bbb','immutable-source','import_source_changed'],['aaa','different-source','import_source_mismatch']]){const source=createNdjsonImportSource({bytes:bytes([{id:'one',title}]),fingerprint}),r=fixture([],{source,root});await r.runner.start();assert.equal(r.root.job.status,'paused');assert.equal(r.root.job.error,code);assert.equal(r.root.document.elements.size,0);assert.equal(r.calls.length,0);} +}); + +test('dependency backpressure pauses with locators intact and a larger budget resumes the same cursor and mapping',async()=>{ + const f=fixture([{id:'child',dependencies:['parent']},{id:'parent'}],{ports:{maxPending:1}});await f.runner.start();assert.equal(f.root.job.status,'paused');assert.equal(f.root.job.error,'dependency_capacity');assert.equal(f.root.job.pending.length,1);const id=f.root.job.stableIds.child; + const r=fixture([],{source:f.source,root:f.root,ports:{maxPending:8,createId:()=> 'parent-id'}});await r.runner.start();assert.equal(r.root.job.status,'completed');assert.equal(r.root.job.stableIds.child,id);assert.equal(r.root.document.elements.size,2); +}); + +test('future interpreter versions and malformed job cuts fail before any source read',()=>{ + const job=createImportJob({id:'i',projectId:'p',sourceFingerprint:'s',parserVersion:1});assert.throws(()=>validateImportJob({...job,version:2}),/import_interpreter_unavailable/);assert.throws(()=>validateImportJob({...job,stableIds:{a:'same',b:'same'}}),/invalid_import_job/);assert.throws(()=>validateImportJob({...job,status:'completed'}),/invalid_import_job/); +});