Decode duplicate payloads on demand from indexed snapshots

src/lib/canvas-core/document.ts

@@ -35,7 +35,7 @@ export function commonCanvas(document: Document, left: Id, right: Id): Id {
   return result;
 }
 /** Cooperative validation over an immutable document; synchronous callers drain the same checks. */
-export function* validateDocumentSteps(document: Document, registry: Registry): Generator<void, void> {
+function* validateDocumentChecks(document: Document, registry: Registry, componentData:boolean): Generator<void, void> {
   if (!document.canvases.size) throw fault('missing_root');
   for (const [id, incarnation] of document.tombstones) { yield; if (!id || !Number.isSafeInteger(incarnation) || incarnation < 1) throw fault('invalid_tombstone', [id]); }
   const entries = new Set<Id>();
@@ -56,7 +56,7 @@ export function* validateDocumentSteps(document: Document, registry: Registry):
     const dead = document.tombstones.get(element.id);
     if (dead !== undefined && element.incarnation <= dead) throw fault('stale_incarnation', [element.id]);
     const definition = registry.components.get(element.componentId);
-    if (registry.resolve(element) === 'known' && (!definition || definition.structure !== element.structure || !definition.validate(element.data))) throw fault('invalid_component_data', [element.id]);
+    if (componentData && registry.resolve(element) === 'known' && (!definition || definition.structure !== element.structure || !definition.validate(element.data))) throw fault('invalid_component_data', [element.id]);
     if (element.structure === 'node') {
       if (!geometryValid(element.geometry)) throw fault('invalid_geometry', [element.id]);
     } else {
@@ -67,6 +67,9 @@ export function* validateDocumentSteps(document: Document, registry: Registry):
     }
   }
 }
+/** Structural indexes contain no component payload; use only for source planning, never editor publication. */
+export function* validateDocumentTopologySteps(document:Document,registry:Registry):Generator<void,void>{yield* validateDocumentChecks(document,registry,false);}
+export function* validateDocumentSteps(document:Document,registry:Registry):Generator<void,void>{yield* validateDocumentChecks(document,registry,true);}
 export function validateDocument(document: Document, registry: Registry): void {
   for (const _ of validateDocumentSteps(document, registry)) { /* Drain identical validation without scheduling. */ }
 }

src/lib/canvas-core/duplicate-checkpoint.ts

@@ -1,5 +1,5 @@
 import type { CanvasRecord, Change, Document, ElementRecord, Id, Point, Registry, Target } from './types.ts';
-import { ancestors, targetElement, validateDocumentSteps } from './document.ts';
+import { ancestors, targetElement, validateDocumentSteps, validateDocumentTopologySteps } from './document.ts';
 import { duplicateTreeChange, duplicateEdgeChange, duplicateOverlay, type DuplicateOptions, type DuplicatePlan } from './duplicate.ts';
 import { clone, equal, fault, freeze, validJson } from './values.ts';
 const phases=['selection','canvases','members','roots','seed','dfs','edges','allocate-elements','allocate-canvases','mapping','emit','emit-edges','validate','done'] as const;
@@ -17,7 +17,7 @@ export function validateDuplicateCheckpoint(value:DuplicateCheckpoint):void{
   for(const list of [value.children,value.elementIds,value.canvasIds])if(!Array.isArray(list)||list.some(pair=>!ids(pair)||pair.length!==2)||new Set(list.map(pair=>pair[0])).size!==list.length)throw fault('invalid_copy_checkpoint');
   if(!Array.isArray(value.members)||value.members.some(pair=>!Array.isArray(pair)||pair.length!==2||typeof pair[0]!=='string'||!pair[0]||!ids(pair[1]))||new Set(value.members.map(pair=>pair[0])).size!==value.members.length||value.emitted>value.order.length||value.emittedEdges>value.edges.length||value.emittedEdges>0&&value.emitted!==value.order.length||value.expanding&&(typeof value.expanding.canvas!=='string'||!value.expanding.canvas||!Number.isSafeInteger(value.expanding.index)||value.expanding.index< -1))throw fault('invalid_copy_checkpoint');
 }
-export function createDuplicatePlanner(document:Document,registry:Registry,selection:readonly Target[],options:DuplicateOptions,binding:{sourceToken:string;schemaToken:string;checkpoint?:DuplicateCheckpoint}){
+export function createDuplicatePlanner(document:Document,registry:Registry,selection:readonly Target[],options:DuplicateOptions,binding:{sourceToken:string;schemaToken:string;checkpoint?:DuplicateCheckpoint;topologyOnly?:boolean}){
   const offset=options.worldOffset??{x:32,y:32},idFactory=options.createId,suppliedMapping=options.mapping?freeze(clone(options.mapping)):undefined;
   const state:MutableCheckpoint=binding.checkpoint?clone(binding.checkpoint) as MutableCheckpoint:{version:1,projectId:document.projectId,sourceToken:binding.sourceToken,schemaToken:binding.schemaToken,selection:clone(selection) as Target[],offset:clone(offset),...(suppliedMapping?{mapping:suppliedMapping}:{}),phase:'selection',cursor:0,steps:0,selected:[],children:[],members:[],roots:[],copied:[],canvases:[],order:[],work:[],edges:[],elementIds:[],canvasIds:[],emitted:0,emittedEdges:0};
   validateDuplicateCheckpoint(state);
@@ -33,7 +33,7 @@ export function createDuplicatePlanner(document:Document,registry:Registry,selec
   const limits:Partial<Record<DuplicateCheckpoint['phase'],number>>={selection:state.selection.length,canvases:sourceCanvases.length,members:sourceElements.length,roots:state.selected.length,seed:state.roots.length,edges:sourceElements.length,'allocate-elements':state.copied.length,'allocate-canvases':state.canvases.length};if(limits[state.phase]!==undefined&&state.cursor>limits[state.phase]!)throw fault('invalid_copy_checkpoint');
   const supplied=suppliedMapping?[...Object.keys(suppliedMapping.elements).map(id=>({id,kind:'element' as const})),...Object.keys(suppliedMapping.canvases).map(id=>({id,kind:'canvas' as const}))]:[];
   const outputElements=new Map<Id,ElementRecord>(),outputCanvases=new Map<Id,CanvasRecord>(),output=duplicateOverlay(document,outputElements,outputCanvases),changes:Change[]=[];
-  let treeCache=0,edgeCache=0,sourceChecks:Generator<void>|undefined=validateDocumentSteps(document,registry),outputChecks:Generator<void>|undefined,validated=false;
+  let treeCache=0,edgeCache=0,sourceChecks:Generator<void>|undefined=(binding.topologyOnly?validateDocumentTopologySteps:validateDocumentSteps)(document,registry),outputChecks:Generator<void>|undefined,validated=false;
   const phase=(next:DuplicateCheckpoint['phase'],cursor=0)=>{state.phase=next;state.cursor=cursor;};
   function add(change:Exclude<Change,{kind:'field'}>){freeze(change);changes.push(change);if(change.kind==='element')outputElements.set(change.id,change.after!);else outputCanvases.set(change.id,change.after!);}
   function tree(index:number){const task=state.order[index];add(duplicateTreeChange(document,registry,task.kind,(task.kind==='element'?document.elements:document.canvases).get(task.id)!,elementIds,canvasIds,canvases,state.offset));}

src/lib/canvas-runtime/copy-snapshot-index.ts

@@ -0,0 +1,58 @@
+import type {CanvasRecord,Document,ElementRecord,Id,Json,Registry,Data} from '../canvas-core/types.ts';
+import {createDocument,validateDocumentTopologySteps} from '../canvas-core/document.ts';
+import {equal,fault,freeze,validJson} from '../canvas-core/values.ts';
+import {createNdjsonImportSource} from './imports.ts';
+const task=()=>new Promise<void>(resolve=>setTimeout(resolve,0));
+/** Only ownership, endpoint and geometry fields needed to decide scope/coordinates. No component content. */
+export function copyTopology(record:ElementRecord):ElementRecord{
+  const raw=record.data.presentation,presentation=raw&&typeof raw==='object'&&!Array.isArray(raw)?Object.fromEntries(['contentScale','unitScale'].filter(key=>Object.hasOwn(raw,key)).map(key=>[key,raw[key]])):{};
+  const data:Data=Object.keys(presentation).length?{presentation}:{};
+  const common={id:record.id,incarnation:record.incarnation,ownerCanvasId:record.ownerCanvasId,componentId:record.componentId,dataVersion:record.dataVersion,data};
+  return record.structure==='node'?{...common,structure:'node',geometry:record.geometry}:{...common,structure:'connection',source:record.source,target:record.target,path:null};
+}
+/** Format 2 places a compact topology/byte-locator index before the uninterpreted payload region. */
+export async function encodeIndexedCopySnapshot(document:Document,signal:AbortSignal):Promise<Blob>{
+  const index:BlobPart[]=[],payload:BlobPart[]=[];let indexBytes=0,payloadBytes=0,count=0;
+  function encode(value:unknown){if(signal.aborted)throw fault('copy_aborted');const text=JSON.stringify(value)+'\n',size=new TextEncoder().encode(text).byteLength;if(size>256*1024)throw fault('record_too_large');return{text,size};}
+  function append(value:unknown){const part=encode(value);index.push(part.text);indexBytes+=part.size;if(indexBytes+payloadBytes>64*1024*1024)throw fault('invalid_import_source');}
+  append({kind:'header',version:2,projectId:document.projectId});
+  for(const record of document.canvases.values()){append({kind:'canvas',record});if(++count%32===0)await task();}
+  for(const record of document.elements.values()){
+    const part=encode({kind:'element',record});append({kind:'element-index',record:copyTopology(record),offset:payloadBytes,byteLength:part.size});payload.push(part.text);payloadBytes+=part.size;if(indexBytes+payloadBytes>64*1024*1024)throw fault('invalid_import_source');if(++count%32===0)await task();
+  }
+  for(const record of document.tombstones){append({kind:'tombstone',record});if(++count%32===0)await task();}
+  append({kind:'payload',byteLength:payloadBytes});if(signal.aborted)throw fault('copy_aborted');return new Blob([...index,...payload],{type:'application/x-ndjson'});
+}
+export type IndexedCopySnapshot={readonly document:Document;readElement(id:Id,signal:AbortSignal):Promise<ElementRecord>};
+/** Scan only the small index. Component JSON/schema checks occur just before each record enters the runner. */
+export async function openIndexedCopySnapshot(bytes:Blob,projectId:Id,registry:Registry,signal:AbortSignal):Promise<IndexedCopySnapshot>{
+  if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source');
+  const source=createNdjsonImportSource({bytes,fingerprint:'indexed-copy-snapshot'}),canvases:CanvasRecord[]=[],elements:ElementRecord[]=[],tombstones=new Map<Id,number>(),locators=new Map<Id,{offset:number;byteLength:number}>();let cursor=0,header=false,payloadOffset=-1,expectedOffset=0,count=0;
+  while(true){
+    if(signal.aborted)throw fault('copy_aborted');const line=await source.read(cursor,signal);if(!line||line.error||line.skip||!line.value)throw fault('invalid_copy_snapshot');cursor=line.nextCursor;
+    const value=line.value as Record<string,Json>;
+    if(!header){if(value.kind!=='header'||value.version!==2||value.projectId!==projectId)throw fault('invalid_copy_snapshot');header=true;continue;}
+    if(value.kind==='payload'){if(value.byteLength!==expectedOffset||cursor+expectedOffset!==bytes.size)throw fault('invalid_copy_snapshot');payloadOffset=cursor;break;}
+    if(value.kind==='canvas')canvases.push(value.record as unknown as CanvasRecord);
+    else if(value.kind==='element-index'){
+      const record=value.record as unknown as ElementRecord,offset=value.offset as number,byteLength=value.byteLength as number;
+      if(!record||!Number.isSafeInteger(offset)||offset!==expectedOffset||!Number.isSafeInteger(byteLength)||byteLength<1||byteLength>256*1024||locators.has(record.id)||!equal(copyTopology(record),record))throw fault('invalid_copy_snapshot');
+      locators.set(record.id,{offset,byteLength});elements.push(record);expectedOffset+=byteLength;if(expectedOffset>bytes.size)throw fault('invalid_copy_snapshot');
+    }else if(value.kind==='tombstone'){const row=value.record as [Id,number];if(!Array.isArray(row)||row.length!==2||tombstones.has(row[0]))throw fault('invalid_copy_snapshot');tombstones.set(row[0],row[1]);}else throw fault('invalid_copy_snapshot');
+    if(++count%32===0)await task();
+  }
+  const document=createDocument(projectId,canvases,elements,tombstones);let checks=0;
+  for(const _ of validateDocumentTopologySteps(document,registry)){if(signal.aborted)throw fault('copy_aborted');if(++checks%512===0)await task();}
+  // Cache raw byte windows, never decoded component objects. Preserve parser batching across Blob task boundaries.
+  const windows=new Map<number,Uint8Array>(),windowBytes=64*1024;let cachedBytes=0;
+  return Object.freeze({document,async readElement(id:Id,signal:AbortSignal){
+    if(signal.aborted)throw fault('copy_aborted');const locator=locators.get(id);if(!locator)throw fault('element_missing',[id]);
+    const start=Math.floor(locator.offset/windowBytes)*windowBytes,required=locator.offset+locator.byteLength-start;let window=windows.get(start);
+    if(!window||window.byteLength<required){const raw=await bytes.slice(payloadOffset+start,Math.min(bytes.size,payloadOffset+start+Math.max(windowBytes,required))).arrayBuffer();if(signal.aborted)throw fault('copy_aborted');const previous=windows.get(start);if(previous)cachedBytes-=previous.byteLength;window=new Uint8Array(raw);windows.delete(start);windows.set(start,window);cachedBytes+=window.byteLength;while(windows.size>4||cachedBytes>1024*1024){const[key,value]=windows.entries().next().value!;windows.delete(key);cachedBytes-=value.byteLength;}}
+    const raw=window.subarray(locator.offset-start,required);let value:{kind:string;record:ElementRecord};
+    try{value=JSON.parse(new TextDecoder('utf-8',{fatal:true}).decode(raw));}catch{throw fault('invalid_copy_snapshot');}
+    if(!value||value.kind!=='element'||!validJson(value.record)||!value.record.data||typeof value.record.data!=='object'||Array.isArray(value.record.data)||!equal(copyTopology(value.record),document.elements.get(id)))throw fault('invalid_copy_snapshot');
+    const record=value.record,definition=registry.components.get(record.componentId);if(registry.resolve(record)==='known'&&(!definition||definition.structure!==record.structure||!definition.validate(record.data)))throw fault('invalid_component_data',[id]);
+    return freeze(record);
+  }});
+}

src/lib/canvas-runtime/duplicate-source.ts

@@ -8,10 +8,10 @@ import { duplicateImportLine } from './import-duplicate.ts';
 import type { ImportSource } from './imports.ts';
 
 /** A seekable record cursor over the original SHA-bound snapshot, not a second mapped Blob. */
-export function createDuplicatePlanSource(document:Document,registry:Registry,state:DuplicateCheckpoint,fingerprint:string):ImportSource{
+export function createDuplicatePlanSource(document:Document,registry:Registry,state:DuplicateCheckpoint,fingerprint:string,readElement?:(id:Id,signal:AbortSignal)=>Promise<ElementRecord>):ImportSource{
   if(state.phase!=='emit'||state.emitted!==0||state.emittedEdges!==0||state.schemaToken!==copyRegistrySignature(document,registry))throw fault('copy_interpreter_changed');
   // Structural/source binding checks; the decoded source has already passed document validation.
-  createDuplicatePlanner(document,registry,state.selection,{createId:()=>{throw fault('unstable_import_identity');},worldOffset:state.offset,...(state.mapping?{mapping:state.mapping}:{})},{sourceToken:state.sourceToken,schemaToken:state.schemaToken,checkpoint:state});
+  createDuplicatePlanner(document,registry,state.selection,{createId:()=>{throw fault('unstable_import_identity');},worldOffset:state.offset,...(state.mapping?{mapping:state.mapping}:{})},{sourceToken:state.sourceToken,schemaToken:state.schemaToken,checkpoint:state,topologyOnly:!!readElement});
   const elements=new Map(state.elementIds),canvases=new Map(state.canvasIds),copiedCanvases=new Set(state.canvases),keys=new Map<Id,string>(),reverseElements=new Map(state.elementIds.map(([id,mapped])=>[mapped,id])),reverseCanvases=new Map(state.canvasIds.map(([id,mapped])=>[mapped,id]));
   for(const[id,mapped]of elements)keys.set(mapped,`element:${id}`);for(const[id,mapped]of canvases)keys.set(mapped,`canvas:${id}`);
   // Edge geometry needs mapped endpoint/owner records. Compute these on demand; retain a bounded cache.
@@ -24,7 +24,8 @@ export function createDuplicatePlanSource(document:Document,registry:Registry,st
   return Object.freeze({fingerprint,parserVersion:2,async read(cursor:number,signal:AbortSignal){
     if(signal.aborted)throw fault('import_aborted');if(!Number.isSafeInteger(cursor)||cursor<0||cursor>length)throw fault('invalid_import_cursor');if(cursor===length)return null;
     const task=state.order[cursor],key=task?`${task.kind}:${task.id}`:`element:${state.edges[cursor-state.order.length]}`;
-    try{const change=cursor<state.order.length?duplicateTreeChange(document,registry,task.kind,(task.kind==='element'?document.elements:document.canvases).get(task.id)!,elements,canvases,copiedCanvases,state.offset):duplicateEdgeChange(document,registry,document.elements.get(state.edges[cursor-state.order.length])!,elements,canvases,projected),line=duplicateImportLine(document,registry,change,keys);
+    try{const original=task?.kind==='canvas'?document.canvases.get(task.id)!:readElement?await readElement(task?.id??state.edges[cursor-state.order.length],signal):document.elements.get(task?.id??state.edges[cursor-state.order.length])!;
+    if(signal.aborted)throw fault('import_aborted');const change=cursor<state.order.length?duplicateTreeChange(document,registry,task.kind,original,elements,canvases,copiedCanvases,state.offset):duplicateEdgeChange(document,registry,original as ElementRecord,elements,canvases,projected),line=duplicateImportLine(document,registry,change,keys);
     if(new TextEncoder().encode(JSON.stringify(line)).byteLength>256*1024)throw fault('record_too_large');
     return{cursor,nextCursor:cursor+1,key:line.key,value:line as unknown as Json};}catch(error){return{cursor,nextCursor:cursor+1,key,error:(error as {code?:string}).code??'copy_record_failed'};}
   }});

src/lib/canvas-runtime/duplicate.ts

@@ -62,7 +62,7 @@ export async function planDuplicateResumable(document:Document,registry:Registry
 
 
 /** Stop before record conversion: the stable order/mappings are sufficient for seekable lazy output. */
-export async function prepareDuplicateStream(document:Document,registry:Registry,selection:readonly Target[],options:DuplicateOptions,binding:{sourceToken:string;schemaToken:string;checkpoint?:DuplicateCheckpoint},scheduling:Omit<DuplicateScheduling,'checkpoint'>&{checkpoint?:(state:DuplicateCheckpoint)=>Promise<void>}={}):Promise<DuplicateCheckpoint>{
+export async function prepareDuplicateStream(document:Document,registry:Registry,selection:readonly Target[],options:DuplicateOptions,binding:{sourceToken:string;schemaToken:string;checkpoint?:DuplicateCheckpoint;topologyOnly?:boolean},scheduling:Omit<DuplicateScheduling,'checkpoint'>&{checkpoint?:(state:DuplicateCheckpoint)=>Promise<void>}={}):Promise<DuplicateCheckpoint>{
   const budget=scheduling.frameBudgetMs??6,limit=scheduling.maxStepsPerSlice??512,now=scheduling.now??(()=>performance.now()),yieldTask=scheduling.yieldTask??(()=>new Promise<void>(resolve=>setTimeout(resolve,0)));
   if(!Number.isFinite(budget)||budget<=0||!Number.isSafeInteger(limit)||limit<1)throw fault('invalid_copy_budget');
   const check=()=>{if(scheduling.signal?.aborted)throw fault('copy_aborted');};check();

src/lib/canvas-runtime/engine.ts

@@ -1,4 +1,5 @@
-import { encodeCopySnapshot, decodeCopySnapshot, copyRegistrySignature } from './copy-snapshot.ts';
+import { encodeIndexedCopySnapshot, openIndexedCopySnapshot, type IndexedCopySnapshot } from './copy-snapshot-index.ts';
+import { decodeCopySnapshot, copyRegistrySignature } from './copy-snapshot.ts';
 import { createDuplicatePlanSource } from './duplicate-source.ts';
 import { planDuplicateAsync, planDuplicateResumable, prepareDuplicateStream } from './duplicate.ts';
 import { importSourceDigest } from './import-integrity.ts';
@@ -329,15 +330,15 @@ export async function openProjectEngine(options: EngineOptions) {
       const admitted=getImport(id),preparation=admitted.preparation;
       if(!preparation||admitted.source?.kind!=='duplicate-snapshot')throw fault('invalid_copy_preparation');
       const check=()=>{const job=getImport(id);if(controller.signal.aborted||closing||closed||importHolds.size||job.status!=='preparing'||job.sourceFingerprint!==admitted.sourceFingerprint)throw fault('copy_aborted');return job;};
-      await persist();check();const document=await decodeCopySnapshot(bytes,options.scope.projectId,options.registry,controller.signal);
+      await persist();check();const indexed=admitted.source.snapshotVersion===2?await openIndexedCopySnapshot(bytes,options.scope.projectId,options.registry,controller.signal):undefined,document=indexed?.document??await decodeCopySnapshot(bytes,options.scope.projectId,options.registry,controller.signal);
       if(copyRegistrySignature(document,options.registry)!==preparation.registrySignature)throw fault('copy_interpreter_changed');
-      if(preparation.version===3){
-        const state=await prepareDuplicateStream(document,options.registry,preparation.selection,{worldOffset:preparation.worldOffset,createId},{sourceToken:admitted.source.sha256!,schemaToken:preparation.registrySignature,checkpoint:preparation.planner},{signal:controller.signal,checkpoint:async state=>{
+      if(preparation.version===3||preparation.version===4){
+        const state=await prepareDuplicateStream(document,options.registry,preparation.selection,{worldOffset:preparation.worldOffset,createId},{sourceToken:admitted.source.sha256!,schemaToken:preparation.registrySignature,checkpoint:preparation.planner,topologyOnly:!!indexed},{signal:controller.signal,checkpoint:async state=>{
           const job=check(),previous=job.preparation!;if(previous.planner&&state.steps<=previous.steps)return;
           publishImport(freeze({...job,generation:job.generation+1,preparation:{...previous,steps:state.steps,allocationIds:[...state.elementIds,...state.canvasIds].map(pair=>pair[1]),planner:state}}),job.generation);await persist();check();
         }}),job=check();assertNativeImportTarget(job,editor.getSnapshot().document);
         const stableIds=Object.fromEntries([...state.elementIds.map(([old,id])=>[`element:${old}`,id]),...state.canvasIds.map(([old,id])=>[`canvas:${old}`,id])]),next={...job,generation:job.generation+1,parserVersion:2,status:'running' as const,stableIds,source:{...job.source!,kind:'duplicate-plan' as const,plan:state,copy:{sourceRevision:job.source!.copy!.sourceRevision,roots:state.roots.map(old=>stableIds[`element:${old}`])}}};delete next.preparation;delete next.error;
-        publishImport(freeze(next),job.generation);return{job:getImport(id),bytes,document};
+        publishImport(freeze(next),job.generation);return{job:getImport(id),bytes,document,indexed};
       }
       const allocationIds=[...preparation.allocationIds];let cursor=0;
       const plan=preparation.version===2?await planDuplicateResumable(document,options.registry,preparation.selection,{worldOffset:preparation.worldOffset,createId},{sourceToken:admitted.source.sha256!,schemaToken:preparation.registrySignature,checkpoint:preparation.planner},{signal:controller.signal,checkpoint:async state=>{
@@ -355,7 +356,7 @@ export async function openProjectEngine(options: EngineOptions) {
       const stableIds=Object.fromEntries([...Object.entries(plan.elementIds).map(([old,id])=>[`element:${old}`,id]),...Object.entries(plan.canvasIds).map(([old,id])=>[`canvas:${old}`,id])]);
       const next={...job,generation:job.generation+1,status:'running' as const,stableIds,sourceFingerprint:`local-file:${fileId}:${encoded.size}`,source:{...job.source!,kind:'duplicate-ndjson' as const,fileId,byteLength:encoded.size,sha256,integrityVersion:1 as const,copy:{sourceRevision:job.source!.copy!.sourceRevision,roots:plan.roots}}};delete next.preparation;delete next.error;
       fileWrites.set(fileId,encoded);try{publishImport(freeze(next),job.generation);}catch(error){fileWrites.delete(fileId);throw error;}
-      return{job:getImport(id),bytes:encoded,document:undefined};
+      return{job:getImport(id),bytes:encoded,document:undefined,indexed:undefined};
     }
     function loadImport(id:Id):Promise<void>{
       if(importLoads.has(id))return importLoads.get(id)!;
@@ -376,9 +377,10 @@ export async function openProjectEngine(options: EngineOptions) {
         if(importHolds.size){deferredImportLoads.add(id);return;}
         if(source.sha256&&source.sha256!==sha256)throw fault('import_source_changed');
         if(!source.sha256){publishImport(freeze({...verified,generation:verified.generation+1,source:{...source,sha256}}),verified.generation);job=getImport(id);source=job.source!;}else job=verified;
-        let copyDocument:Document|undefined;
-        if(source.kind==='duplicate-snapshot'){if(preparationController.signal.aborted||importHolds.size||latest.status!=='preparing')return;const materialized=await materializeCopy(id,bytes,preparationController);job=materialized.job;source=job.source!;bytes=materialized.bytes;copyDocument=materialized.document;expectedFingerprint=job.sourceFingerprint;}
-        const importSource=source.kind==='duplicate-plan'?createDuplicatePlanSource(copyDocument??await decodeCopySnapshot(bytes,options.scope.projectId,options.registry,preparationController.signal),options.registry,source.plan!,job.sourceFingerprint):createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint});
+        let copyDocument:Document|undefined,indexed:IndexedCopySnapshot|undefined;
+        if(source.kind==='duplicate-snapshot'){if(preparationController.signal.aborted||importHolds.size||latest.status!=='preparing')return;const materialized=await materializeCopy(id,bytes,preparationController);job=materialized.job;source=job.source!;bytes=materialized.bytes;copyDocument=materialized.document;indexed=materialized.indexed;expectedFingerprint=job.sourceFingerprint;}
+        if(source.kind==='duplicate-plan'&&source.snapshotVersion===2){indexed??=await openIndexedCopySnapshot(bytes,options.scope.projectId,options.registry,preparationController.signal);copyDocument=indexed.document;}
+        const importSource=source.kind==='duplicate-plan'?createDuplicatePlanSource(copyDocument??await decodeCopySnapshot(bytes,options.scope.projectId,options.registry,preparationController.signal),options.registry,source.plan!,job.sourceFingerprint,indexed?.readElement):createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint});
         if(closing||closed||preparationController.signal.aborted||getImport(id).generation!==job.generation)return;
         if(importHolds.size){deferredImportLoads.add(id);return;}
         const runner=createImportRunner({optimisticLocal:true,awaitDurabilityBudget:()=>persistenceFailed||pending.size>=32?persist():undefined,source:importSource,getJob:()=>getImport(id),createId,
@@ -413,12 +415,12 @@ export async function openProjectEngine(options: EngineOptions) {
         signal?.addEventListener('abort',abort,{once:true});if(signal?.aborted)controller.abort();
         const task=(async()=>{
           if(!target||!targets.length)throw fault('empty_copy_selection');
-          const bytes=await encodeCopySnapshot(captured.document,controller.signal);assertWritable();if(controller.signal.aborted)throw fault('copy_aborted');
+          const bytes=await encodeIndexedCopySnapshot(captured.document,controller.signal);assertWritable();if(controller.signal.aborted)throw fault('copy_aborted');
           const canvas=captured.document.canvases.get(target);if(!canvas)throw fault('canvas_not_loaded',[target]);
           const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;
           if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
           const entry=canvas.parent?captured.document.elements.get(canvas.parent.entryElementId)!:undefined;
-          const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'duplicate-snapshot',name:'复制选中内容',fileId,byteLength:bytes.size,targetCanvasId:target,copy:{sourceRevision:captured.revision,roots:[]},...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'preparing' as const,preparation:{version:3 as const,registrySignature:copyRegistrySignature(captured.document,options.registry),selection:targets,worldOffset:offset,allocationIds:[],steps:0}});
+          const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'duplicate-snapshot',snapshotVersion:2,name:'复制选中内容',fileId,byteLength:bytes.size,targetCanvasId:target,copy:{sourceRevision:captured.revision,roots:[]},...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'preparing' as const,preparation:{version:4 as const,registrySignature:copyRegistrySignature(captured.document,options.registry),selection:targets,worldOffset:offset,allocationIds:[],steps:0}});
           fileWrites.set(fileId,bytes);const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});});
           if(result.status==='rejected'){fileWrites.delete(fileId);throw fault(result.code,result.targets);}void loadImport(id);return id;
         })();

src/lib/canvas-runtime/imports.ts

@@ -8,12 +8,12 @@ export type ImportSource = { readonly fingerprint: string; readonly parserVersio
 export type ImportUnit = { readonly key: string; readonly dependencies: readonly string[]; readonly changes: readonly Change[]; readonly guard?: Json };
 export type ImportLocator = { readonly key: string; readonly cursor: number; readonly nextCursor: number; readonly dependencies: readonly string[]; readonly digest: string; readonly byteLength: number; readonly changeCount: number };
 export type ImportFailure = { readonly key: string; readonly cursor: number; readonly code: string; readonly dependencies?: readonly string[] };
-export type NativeImportSource = { readonly kind: 'native-ndjson' | 'duplicate-ndjson' | 'duplicate-snapshot' | 'duplicate-plan'; readonly plan?:DuplicateCheckpoint; readonly copy?: {readonly sourceRevision:number; readonly roots:readonly Id[]}; readonly integrityVersion?:1;readonly sha256?:string; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
+export type NativeImportSource = { readonly kind: 'native-ndjson' | 'duplicate-ndjson' | 'duplicate-snapshot' | 'duplicate-plan'; readonly snapshotVersion?:2;readonly plan?:DuplicateCheckpoint; readonly copy?: {readonly sourceRevision:number; readonly roots:readonly Id[]}; readonly integrityVersion?:1;readonly sha256?:string; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
 export type ImportBinding = {readonly kind:'element';readonly incarnation:number}|{readonly kind:'canvas';readonly entryElementId?:Id};
 export type ImportJob = {
   readonly version: 1 | 2 | 3; readonly id: Id; readonly projectId: Id; readonly sourceFingerprint: string; readonly parserVersion: number;
   readonly source?: NativeImportSource;
-  readonly preparation?: {readonly version:1|2|3;readonly planner?:DuplicateCheckpoint;readonly registrySignature:string;readonly selection:readonly Target[];readonly worldOffset:Point;readonly allocationIds:readonly Id[];readonly steps:number};
+  readonly preparation?: {readonly version:1|2|3|4;readonly planner?:DuplicateCheckpoint;readonly registrySignature:string;readonly selection:readonly Target[];readonly worldOffset:Point;readonly allocationIds:readonly Id[];readonly steps:number};
   readonly generation: number; readonly status: 'preparing' | 'paused' | 'running' | 'completed' | 'partial' | 'cancelled'; readonly cursor: number; readonly eof: boolean;
   readonly bindings?: Readonly<Record<Id,ImportBinding>>;
   readonly stableIds: Readonly<Record<string, Id>>; readonly pending: readonly ImportLocator[]; readonly committedSourceKeys: readonly string[]; readonly failures: readonly ImportFailure[]; readonly error?: string;
@@ -29,14 +29,16 @@ export function validateImportJob(job: ImportJob): void {
   if (!Number.isSafeInteger(job.generation) || job.generation<0 || !Number.isSafeInteger(job.cursor) || job.cursor<0 || typeof job.eof!=='boolean' || !['preparing','paused','running','completed','partial','cancelled'].includes(job.status)) throw fault('invalid_import_job');
   if(job.source&&(!['native-ndjson','duplicate-ndjson','duplicate-snapshot','duplicate-plan'].includes(job.source.kind)||['duplicate-ndjson','duplicate-plan'].includes(job.source.kind)&&(!job.source.copy||!Number.isSafeInteger(job.source.copy.sourceRevision)||job.source.copy.sourceRevision<0||!Array.isArray(job.source.copy.roots)||!job.source.copy.roots.length||job.source.copy.roots.some(id=>typeof id!=='string'||!id))||job.source.name!==undefined&&(typeof job.source.name!=='string'||job.source.name.length>255)||!job.source.fileId||!job.source.targetCanvasId||!Number.isSafeInteger(job.source.byteLength)||job.source.byteLength<0||job.source.targetEntry&&(!job.source.targetEntry.id||!Number.isSafeInteger(job.source.targetEntry.incarnation)||job.source.targetEntry.incarnation<1)))throw fault('invalid_import_source');
   if(job.source){
+    if(job.source.snapshotVersion!==undefined&&(job.source.snapshotVersion!==2||!['duplicate-snapshot','duplicate-plan'].includes(job.source.kind)))throw fault('copy_snapshot_interpreter_unavailable');
     if(job.source.integrityVersion!==undefined&&job.source.integrityVersion!==1)throw fault('import_source_integrity_interpreter_unavailable');
     if(job.source.sha256!==undefined&&(typeof job.source.sha256!=='string'||!(/^[a-f0-9]{64}$/.test(job.source.sha256))))throw fault('invalid_import_source');
     if(job.source.integrityVersion===1&&!job.source.sha256&&(job.cursor!==0||job.eof||job.pending.length||job.committedSourceKeys.length||Object.keys(job.stableIds).length||(job.preparation?.allocationIds.length??0)>0||(job.preparation?.steps??0)>0))throw fault('import_source_integrity_missing');
   }
   if(job.source?.kind==='duplicate-snapshot'){
     const p=job.preparation;
-    if(!p||![1,2,3].includes(p.version)||typeof p.registrySignature!=='string'||!p.registrySignature||!Array.isArray(p.selection)||!p.selection.length||p.selection.some(t=>!t||typeof t.id!=='string'||!t.id||!Number.isSafeInteger(t.incarnation)||t.incarnation<1)||!p.worldOffset||![p.worldOffset.x,p.worldOffset.y].every(Number.isFinite)||!Array.isArray(p.allocationIds)||p.allocationIds.some(id=>typeof id!=='string'||!id)||new Set(p.allocationIds).size!==p.allocationIds.length||!Number.isSafeInteger(p.steps)||p.steps<0||!['preparing','paused','cancelled'].includes(job.status)||job.cursor!==0||job.eof||job.pending.length||job.committedSourceKeys.length||Object.keys(job.stableIds).length||!job.source.copy||!Number.isSafeInteger(job.source.copy.sourceRevision)||job.source.copy.sourceRevision<0)throw fault('invalid_copy_preparation');
-    if(p.planner){validateDuplicateCheckpoint(p.planner);if(![2,3].includes(p.version)||p.planner.sourceToken!==job.source.sha256||p.planner.schemaToken!==p.registrySignature||p.planner.projectId!==job.projectId||!equal(p.planner.selection,p.selection)||!equal(p.planner.offset,p.worldOffset)||p.planner.steps!==p.steps||!equal([...p.planner.elementIds,...p.planner.canvasIds].map(pair=>pair[1]),p.allocationIds))throw fault('invalid_copy_preparation');}
+    if(!p||![1,2,3,4].includes(p.version)||typeof p.registrySignature!=='string'||!p.registrySignature||!Array.isArray(p.selection)||!p.selection.length||p.selection.some(t=>!t||typeof t.id!=='string'||!t.id||!Number.isSafeInteger(t.incarnation)||t.incarnation<1)||!p.worldOffset||![p.worldOffset.x,p.worldOffset.y].every(Number.isFinite)||!Array.isArray(p.allocationIds)||p.allocationIds.some(id=>typeof id!=='string'||!id)||new Set(p.allocationIds).size!==p.allocationIds.length||!Number.isSafeInteger(p.steps)||p.steps<0||!['preparing','paused','cancelled'].includes(job.status)||job.cursor!==0||job.eof||job.pending.length||job.committedSourceKeys.length||Object.keys(job.stableIds).length||!job.source.copy||!Number.isSafeInteger(job.source.copy.sourceRevision)||job.source.copy.sourceRevision<0)throw fault('invalid_copy_preparation');
+    if((p.version===4)!==(job.source.snapshotVersion===2))throw fault('invalid_copy_preparation');
+    if(p.planner){validateDuplicateCheckpoint(p.planner);if(![2,3,4].includes(p.version)||p.planner.sourceToken!==job.source.sha256||p.planner.schemaToken!==p.registrySignature||p.planner.projectId!==job.projectId||!equal(p.planner.selection,p.selection)||!equal(p.planner.offset,p.worldOffset)||p.planner.steps!==p.steps||!equal([...p.planner.elementIds,...p.planner.canvasIds].map(pair=>pair[1]),p.allocationIds))throw fault('invalid_copy_preparation');}
     else if(p.version!==1&&(p.steps!==0||p.allocationIds.length))throw fault('invalid_copy_preparation');
   }else if(job.status==='preparing'||job.preparation!==undefined)throw fault('invalid_copy_preparation');
   const keys=new Set<string>(),ids=Object.values(job.stableIds),stableIdSet=new Set(ids);