feat(canvas): persist and execute deep duplicate tasks

This commit is contained in:
2026-11-08 15:59:15 +08:00
parent 379db2fe14
commit 7ed0aa61cb
8 changed files with 188 additions and 18 deletions
+31 -8
View File
@@ -1,7 +1,9 @@
import { planDuplicateAsync } from './duplicate.ts';
import { encodeDuplicateSource, prepareDuplicateImport, assertDuplicateExternal } from './import-duplicate.ts';
import { createImportJob, createImportRunner, createNdjsonImportSource, type ImportJob } from './imports.ts';
import { assertNativeImportDependencies, assertNativeImportTarget, prepareNativeImport } from './import-native.ts';
import { elisionBatchLimit, findElision } from './elision.ts';
import type { Document, Id, Registry, Result, RunInput, RunApplyInput, RunCancelInput, Json, OperationIntent, Data } from '../canvas-core/types.ts';
import type { Document, Id, Registry, Result, RunInput, RunApplyInput, RunCancelInput, Target, Point, Json, OperationIntent, Data } from '../canvas-core/types.ts';
import { createEditor, type EditorOptions, type Transaction } from '../canvas-core/editor.ts';
import { createDocument } from '../canvas-core/document.ts';
import { createSessions } from '../canvas-core/gestures.ts';
@@ -55,6 +57,7 @@ export async function openProjectEngine(options: EngineOptions) {
let staged: readonly ResourceRecord[] = [];
let applying: RunApplyInput | undefined;
let importing: { job: ImportJob; expectedGeneration: number } | undefined;
const copyPreparations=new Map<AbortController,Promise<Id>>();
const importRunners=new Map<Id,ReturnType<typeof createImportRunner>>(),importLoads=new Map<Id,Promise<void>>(),importHolds=new Map<symbol,Map<Id,()=>void>>();
let recovery: { controller: AbortController; release(): void; cancel():Promise<void> } | undefined;
const assertWritable = () => { if(closing||closed)throw fault('engine_closing');if(recovery)throw fault('recovery_in_progress'); };
@@ -321,18 +324,18 @@ export async function openProjectEngine(options: EngineOptions) {
if(importRunners.has(id)){void importRunners.get(id)!.start();return Promise.resolve();}
const expectedFingerprint=getImport(id).sourceFingerprint;
const task=(async()=>{
const job=getImport(id),source=job.source;if(!source||source.kind!=='native-ndjson'||job.parserVersion!==1)throw fault('import_interpreter_unavailable');
const job=getImport(id),source=job.source;if(!source||!['native-ndjson','duplicate-ndjson'].includes(source.kind)||job.parserVersion!==1)throw fault('import_interpreter_unavailable');
const bytes=fileWrites.get(source.fileId)??await options.storage.readFile(leader,source.fileId);
if(closing||closed)return;
const latest=getImport(id);if(latest.sourceFingerprint!==job.sourceFingerprint||latest.source?.fileId!==source.fileId||['completed','partial','cancelled'].includes(latest.status))return;
if(!bytes)throw fault('import_source_unavailable');if(bytes.size!==source.byteLength)throw fault('import_source_changed');
const runner=createImportRunner({optimisticLocal:true,source:createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint}),getJob:()=>getImport(id),createId,
publish:publishImport,prepare:(line,job,reserve)=>prepareNativeImport(line,job,reserve,editor.getSnapshot().document,options.registry),persist,onError:report,
publish:publishImport,prepare:(line,job,reserve)=>job.source?.kind==='duplicate-ndjson'?prepareDuplicateImport(line,job,reserve):prepareNativeImport(line,job,reserve,editor.getSnapshot().document,options.registry),persist,onError:report,
commit(units,next,expected){
assertWritable();if(importing)throw fault('reentrant_dispatch');const current=getImport(id);if(current.generation!==expected)throw fault('stale_import_job');
try{assertNativeImportTarget(current,editor.getSnapshot().document);for(const unit of units)assertNativeImportDependencies(current,editor.getSnapshot().document,unit.dependencies);}catch(error){return reject(error);}
try{assertNativeImportTarget(current,editor.getSnapshot().document);for(const unit of units){assertNativeImportDependencies(current,editor.getSnapshot().document,unit.dependencies);if(current.source?.kind==='duplicate-ndjson')assertDuplicateExternal({cursor:0,nextCursor:1,key:unit.key,value:unit.guard},current,editor.getSnapshot().document);}}catch(error){return reject(error);}
importing={job:next,expectedGeneration:expected};
try{return editor.transaction(tx=>{for(const unit of units)for(const change of unit.changes){if(change.kind==='element'&&change.before===null&&change.after)tx.dispatch({type:'element.create',element:change.after});else if(change.kind==='canvas'&&change.before===null&&change.after)tx.dispatch({type:'canvas.create',canvas:change.after});else throw fault('invalid_native_import_unit');}},{label:'Import canvas content',groupId:`import:${id}`});}finally{importing=undefined;}
try{return editor.transaction(tx=>{for(const unit of units)for(const change of unit.changes){if(change.kind==='element'&&change.before===null&&change.after)tx.dispatch({type:'element.create',element:change.after});else if(change.kind==='canvas'&&change.before===null&&change.after)tx.dispatch({type:'canvas.create',canvas:change.after});else throw fault('invalid_native_import_unit');}},{label:current.source?.kind==='duplicate-ndjson'?'Duplicate selection':'Import canvas content',groupId:`import:${id}`});}finally{importing=undefined;}
},
});
importRunners.set(id,runner);for(const releases of importHolds.values())releases.set(id,runner.hold());
@@ -343,7 +346,7 @@ export async function openProjectEngine(options: EngineOptions) {
const imports=Object.freeze({
getJobs:()=>readMap(editor.getSnapshot().runtime.imports??new Map()),
startNdjson(bytes:Blob,targetCanvasId:Id):Id{
assertWritable();const canvas=editor.getSnapshot().document.canvases.get(targetCanvasId);if(!canvas)throw fault('canvas_not_loaded',[targetCanvasId]);if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source');
assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);const canvas=editor.getSnapshot().document.canvases.get(targetCanvasId);if(!canvas)throw fault('canvas_not_loaded',[targetCanvasId]);if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source');
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
const entry=canvas.parent?editor.getSnapshot().document.elements.get(canvas.parent.entryElementId)!:undefined;
const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'native-ndjson',name:((bytes as Blob&{name?:string}).name||'画布数据').slice(0,255),fileId,byteLength:bytes.size,targetCanvasId,...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'running' as const});
@@ -351,6 +354,26 @@ export async function openProjectEngine(options: EngineOptions) {
const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});});
if(result.status==='rejected'){fileWrites.delete(fileId);throw fault(result.code,result.targets);}void loadImport(id);return id;
},
async duplicate(selection:readonly Target[],settings:{readonly worldOffset?:Point;readonly signal?:AbortSignal;readonly viewCanvasId?:Id}={}):Promise<Id>{
assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);const captured=editor.getSnapshot(),target=settings.viewCanvasId??(selection[0]?captured.document.elements.get(selection[0].id)?.ownerCanvasId:undefined),signal=settings.signal,controller=new AbortController(),abort=()=>controller.abort();
signal?.addEventListener('abort',abort,{once:true});if(signal?.aborted)controller.abort();
const task=(async()=>{
const plan=await planDuplicateAsync(captured.document,options.registry,selection,{createId,worldOffset:settings.worldOffset},{signal:controller.signal});
const bytes=await encodeDuplicateSource(captured.document,options.registry,plan,controller.signal);
assertWritable();if(controller.signal.aborted)throw fault('copy_aborted');const permission=options.policy?.({type:'changes.apply',changes:plan.changes},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);
// Task belongs to the initiating view; copied records retain their mapped document owners.
if(!target)throw fault('empty_copy_selection');
const canvas=editor.getSnapshot().document.canvases.get(target);if(!canvas)throw fault('copy_external_changed',[target]);
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;
if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
const entry=canvas.parent?editor.getSnapshot().document.elements.get(canvas.parent.entryElementId)!:undefined;
const stableIds=Object.fromEntries([...Object.entries(plan.elementIds).map(([old,id])=>[`element:${old}`,id]),...Object.entries(plan.canvasIds).map(([old,id])=>[`canvas:${old}`,id])]);
const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'duplicate-ndjson',name:'复制选中内容',fileId,byteLength:bytes.size,targetCanvasId:target,copy:{sourceRevision:captured.revision,roots:plan.roots},...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),stableIds,status:'running' as const});
fileWrites.set(fileId,bytes);const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});});
if(result.status==='rejected'){fileWrites.delete(fileId);throw fault(result.code,result.targets);}void loadImport(id);return id;
})();
copyPreparations.set(controller,task);void task.finally(()=>{copyPreparations.delete(controller);signal?.removeEventListener('abort',abort);}).catch(()=>{});return task;
},
async pause(id:Id){assertWritable();const runner=importRunners.get(id);if(runner)await runner.pause();else{const job=getImport(id);if(job.status==='running'){publishImport(freeze({...job,generation:job.generation+1,status:'paused'}),job.generation);await persist();}}},
async resume(id:Id){assertWritable();const job=getImport(id);if(['completed','partial','cancelled'].includes(job.status))return;if(job.status==='running'){await loadImport(id);return;}await importRunners.get(id)?.settle();await persist();await scheduler.resume();assertWritable();const current=getImport(id);if(current.status!=='paused')return;const next={...current,generation:current.generation+1,status:'running' as const};delete next.error;publishImport(freeze(next),current.generation);await loadImport(id);},
async cancel(id:Id){assertWritable();const runner=importRunners.get(id);if(runner)await runner.cancel();else{const job=getImport(id);if(['running','paused'].includes(job.status)){const next={...job,generation:job.generation+1,status:'cancelled' as const};delete next.error;publishImport(freeze(next),job.generation);await persist();}}},
@@ -520,10 +543,10 @@ export async function openProjectEngine(options: EngineOptions) {
cancelRecovery:async()=>{await recovery?.cancel();},
getDurability: () => Object.freeze({ rootRevision: editor.getSnapshot().rootRevision, durableRootRevision }),
retry: async () => { if(recovery)await recovery.cancel();assertWritable();await scheduler.resume();await runTracker.retry(); },
async settle() { await Promise.resolve();while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async settle() { await Promise.resolve();await Promise.allSettled([...copyPreparations.values()]);while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async close() {
if (closed) return;
closing = true; sessions.dispose(); scheduler.stop(); runTracker.stop();
closing = true;for(const controller of copyPreparations.keys())controller.abort();await Promise.allSettled([...copyPreparations.values()]); sessions.dispose(); scheduler.stop(); runTracker.stop();
await Promise.all([...importRunners.values()].map(runner=>runner.close()));while(importLoads.size)await Promise.all([...importLoads.values()]);
recovery?.controller.abort();recovery?.release();
for (const release of [...displays]) release();
@@ -0,0 +1,63 @@
import type { CanvasRecord, Change, Document, Id, Json, Registry } from '../canvas-core/types.ts';
import type { DuplicatePlan } from '../canvas-core/duplicate.ts';
import { clone, equal, fault, validJson } from '../canvas-core/values.ts';
import type { ImportJob, ImportLine, ImportUnit } from './imports.ts';
type ExternalElement = {id: Id; incarnation: number; ownerCanvasId: Id};
type ExternalCanvas = {id: Id; parent: CanvasRecord['parent']; entryIncarnation?: number};
type CopyLine = {key: string; change: Exclude<Change,{kind:'field'}>; dependencies: string[]; externalElements: ExternalElement[]; externalCanvases: ExternalCanvas[]};
/** Internal source: already mapped records, with explicit guards for every retained external reference. */
export async function encodeDuplicateSource(document: Document, registry: Registry, plan: DuplicatePlan, signal: AbortSignal): Promise<Blob> {
const keys = new Map<Id,string>(),parts: BlobPart[] = [];
for (const [id,mapped] of Object.entries(plan.elementIds)) keys.set(mapped,`element:${id}`);
for (const [id,mapped] of Object.entries(plan.canvasIds)) keys.set(mapped,`canvas:${id}`);
let bytes=0,count=0;
for (const change of plan.changes) {
if(signal.aborted)throw fault('copy_aborted');
if(change.kind==='field'||!change.after||change.before!==null)throw fault('invalid_duplicate_plan');
const dependencies=new Set<string>(),elements=new Map<Id,ExternalElement>(),canvases=new Map<Id,ExternalCanvas>();
const canvas=(id:Id)=>{
const key=keys.get(id);if(key){dependencies.add(key);return;}
let next:Id|undefined=id;
while(next&&!canvases.has(next)){
const original=document.canvases.get(next);if(!original)throw fault('copy_external_changed',[next]);
const entry=original.parent?document.elements.get(original.parent.entryElementId):undefined;
canvases.set(next,{id:next,parent:clone(original.parent),...(entry?{entryIncarnation:entry.incarnation}:{})});
if(entry)elements.set(entry.id,{id:entry.id,incarnation:entry.incarnation,ownerCanvasId:entry.ownerCanvasId});
next=original.parent?.canvasId;
}
};
const element=(id:Id)=>{const key=keys.get(id);if(key){dependencies.add(key);return;}const original=document.elements.get(id);if(!original)throw fault('copy_external_changed',[id]);elements.set(id,{id,incarnation:original.incarnation,ownerCanvasId:original.ownerCanvasId});canvas(original.ownerCanvasId);};
if(change.kind==='canvas'){if(!change.after.parent)throw fault('invalid_duplicate_plan');canvas(change.after.parent.canvasId);element(change.after.parent.entryElementId);}
else {
const record=change.after;canvas(record.ownerCanvasId);
if(record.structure==='connection'){element(record.source.elementId);element(record.target.elementId);}
const schema=registry.components.get(record.componentId)?.copy;if(!schema)throw fault('copy_schema_unavailable');
for(const ref of schema.references){let value:Json|undefined=record.data;for(const part of ref.path){if(!value||typeof value!=='object'||!Object.hasOwn(value,part)){value=undefined;break;}value=(value as Record<string,Json>)[part];}if(value===undefined||value===null)continue;for(const id of ref.many?value as Json[]:[value]){if(typeof id!=='string')throw fault('invalid_copy_reference');if(ref.target==='element')element(id);else canvas(id);}}
}
const key=keys.get(change.id);if(!key)throw fault('invalid_duplicate_plan');dependencies.delete(key);
const line:CopyLine={key,change,dependencies:[...dependencies],externalElements:[...elements.values()],externalCanvases:[...canvases.values()]};
const text=JSON.stringify(line)+'\n',size=new TextEncoder().encode(text).byteLength;if(size>256*1024)throw fault('record_too_large');bytes+=size;if(bytes>64*1024*1024)throw fault('invalid_import_source');parts.push(text);
if(++count%32===0)await new Promise<void>(resolve=>setTimeout(resolve,0));
}
if(signal.aborted)throw fault('copy_aborted');return new Blob(parts,{type:'application/x-ndjson'});
}
function copyLine(line:ImportLine,job:ImportJob):CopyLine {
const value=line.value as unknown as CopyLine;
if(job.source?.kind!=='duplicate-ndjson'||!value||!validJson(value)||typeof value.key!=='string'||!Array.isArray(value.dependencies)||!Array.isArray(value.externalElements)||!Array.isArray(value.externalCanvases)||!value.change||value.change.kind!=='element'&&value.change.kind!=='canvas'||value.change.before!==null||!value.change.after||!Object.hasOwn(job.stableIds,value.key)||job.stableIds[value.key]!==value.change.id||value.change.after.id!==value.change.id||!value.key.startsWith(`${value.change.kind}:`))throw fault('invalid_duplicate_source');
if(value.change.kind==='element'&&value.change.after?.incarnation!==1)throw fault('invalid_duplicate_source');
return value;
}
export function prepareDuplicateImport(line:ImportLine,job:ImportJob,reserve:(key:string)=>Id):ImportUnit {
const value=copyLine(line,job);if(reserve(value.key)!==value.change.id)throw fault('invalid_duplicate_source');
for(const key of value.dependencies)if(!Object.hasOwn(job.stableIds,key))throw fault('invalid_duplicate_source');
return{key:value.key,dependencies:value.dependencies,changes:[value.change],guard:clone(line.value!)};
}
/** Check at the synchronous commit boundary, never only when the source was first parsed. */
export function assertDuplicateExternal(line:ImportLine,job:ImportJob,document:Document){
const value=copyLine(line,job);
for(const guard of value.externalElements){const record=document.elements.get(guard.id);if(!record||record.incarnation!==guard.incarnation||record.ownerCanvasId!==guard.ownerCanvasId)throw fault('copy_external_changed',[guard.id]);}
for(const guard of value.externalCanvases){const record=document.canvases.get(guard.id);if(!record||!equal(record.parent,guard.parent)||guard.parent&&document.elements.get(guard.parent.entryElementId)?.incarnation!==guard.entryIncarnation)throw fault('copy_external_changed',[guard.id]);}
}
+4 -3
View File
@@ -3,10 +3,10 @@ import { clone, equal, fault, freeze, validJson } from '../canvas-core/values.ts
export type ImportLine = { readonly cursor: number; readonly nextCursor: number; readonly key: string; readonly value?: Json; readonly error?: string; readonly skip?: boolean };
export type ImportSource = { readonly fingerprint: string; readonly parserVersion: number; read(cursor: number, signal: AbortSignal): Promise<ImportLine | null> };
export type ImportUnit = { readonly key: string; readonly dependencies: readonly string[]; readonly changes: readonly Change[] };
export type ImportUnit = { readonly key: string; readonly dependencies: readonly string[]; readonly changes: readonly Change[]; readonly guard?: Json };
export type ImportLocator = { readonly key: string; readonly cursor: number; readonly nextCursor: number; readonly dependencies: readonly string[]; readonly digest: string; readonly byteLength: number; readonly changeCount: number };
export type ImportFailure = { readonly key: string; readonly cursor: number; readonly code: string; readonly dependencies?: readonly string[] };
export type NativeImportSource = { readonly kind: 'native-ndjson'; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
export type NativeImportSource = { readonly kind: 'native-ndjson' | 'duplicate-ndjson'; readonly copy?: {readonly sourceRevision:number; readonly roots:readonly Id[]}; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
export type ImportBinding = {readonly kind:'element';readonly incarnation:number}|{readonly kind:'canvas';readonly entryElementId?:Id};
export type ImportJob = {
readonly version: 1; readonly id: Id; readonly projectId: Id; readonly sourceFingerprint: string; readonly parserVersion: number;
@@ -24,9 +24,10 @@ export function validateImportJob(job: ImportJob): void {
if (job.version !== 1) throw fault('import_interpreter_unavailable');
createImportJob(job);
if (!Number.isSafeInteger(job.generation) || job.generation<0 || !Number.isSafeInteger(job.cursor) || job.cursor<0 || typeof job.eof!=='boolean' || !['paused','running','completed','partial','cancelled'].includes(job.status)) throw fault('invalid_import_job');
if(job.source&&(job.source.kind!=='native-ndjson'||job.source.name!==undefined&&(typeof job.source.name!=='string'||job.source.name.length>255)||!job.source.fileId||!job.source.targetCanvasId||!Number.isSafeInteger(job.source.byteLength)||job.source.byteLength<0||job.source.targetEntry&&(!job.source.targetEntry.id||!Number.isSafeInteger(job.source.targetEntry.incarnation)||job.source.targetEntry.incarnation<1)))throw fault('invalid_import_source');
if(job.source&&(!['native-ndjson','duplicate-ndjson'].includes(job.source.kind)||job.source.kind==='duplicate-ndjson'&&(!job.source.copy||!Number.isSafeInteger(job.source.copy.sourceRevision)||job.source.copy.sourceRevision<0||!Array.isArray(job.source.copy.roots)||!job.source.copy.roots.length||job.source.copy.roots.some(id=>typeof id!=='string'||!id))||job.source.name!==undefined&&(typeof job.source.name!=='string'||job.source.name.length>255)||!job.source.fileId||!job.source.targetCanvasId||!Number.isSafeInteger(job.source.byteLength)||job.source.byteLength<0||job.source.targetEntry&&(!job.source.targetEntry.id||!Number.isSafeInteger(job.source.targetEntry.incarnation)||job.source.targetEntry.incarnation<1)))throw fault('invalid_import_source');
const keys=new Set<string>(),ids=Object.values(job.stableIds),stableIdSet=new Set(ids);
if (stableIdSet.size!==ids.length || ids.some(id=>typeof id!=='string'||!id) || Object.keys(job.stableIds).some(key=>!key)) throw fault('invalid_import_job');
if(job.source?.kind==='duplicate-ndjson'&&(!job.source.copy||new Set(job.source.copy.roots).size!==job.source.copy.roots.length||job.source.copy.roots.some(id=>!stableIdSet.has(id))))throw fault('invalid_import_source');
for(const[id,binding]of Object.entries(job.bindings??{}))if(!id||!binding||!stableIdSet.has(id)||!['element','canvas'].includes(binding.kind)||binding.kind==='element'&&(!Number.isSafeInteger(binding.incarnation)||binding.incarnation<1)||binding.kind==='canvas'&&binding.entryElementId!==undefined&&typeof binding.entryElementId!=='string')throw fault('invalid_import_job');
for(const key of job.committedSourceKeys){if(!key||keys.has(key))throw fault('invalid_import_job');keys.add(key);}
if(job.source)for(const key of job.committedSourceKeys){const id=job.stableIds[key],binding=job.bindings?.[id];if(!Object.hasOwn(job.stableIds,key)||!binding||!key.startsWith(`${binding.kind}:`)||binding.kind==='canvas'&&(!binding.entryElementId||job.bindings?.[binding.entryElementId]?.kind!=='element'))throw fault('invalid_import_job');}