feat(canvas): persist native import jobs with atomic editor progress

This commit is contained in:
2026-11-08 15:40:10 +08:00
parent 160c16afb7
commit 72d6105e7c
6 changed files with 171 additions and 16 deletions
+2
View File
@@ -1,3 +1,4 @@
import type { ImportJob } from './imports.ts';
import type { Document, Id, Json, OperationIntent, Snapshot } from '../canvas-core/types.ts';
import type { LedgerSnapshot, FrozenRequest } from './ledger.ts';
import type { CompiledRequest, OperationStatus } from './scheduler.ts';
@@ -6,6 +7,7 @@ import type { ResourceRecord } from './resources.ts';
import type { RecoveryAudit } from './conflicts.ts';
export type RuntimeState = {
readonly imports?: ReadonlyMap<Id, ImportJob>;
readonly ledger: LedgerSnapshot;
readonly runs: ReadonlyMap<Id, RunRecord>;
readonly acknowledged: Document;
+62 -9
View File
@@ -1,3 +1,5 @@
import { createImportJob, createImportRunner, createNdjsonImportSource, type ImportJob } from './imports.ts';
import { assertNativeImportDependencies, assertNativeImportTarget, prepareNativeImport } from './import-native.ts';
import { elisionBatchLimit, findElision } from './elision.ts';
import type { Document, Id, Registry, Result, RunInput, RunApplyInput, RunCancelInput, Json, OperationIntent, Data } from '../canvas-core/types.ts';
import { createEditor, type EditorOptions, type Transaction } from '../canvas-core/editor.ts';
@@ -52,9 +54,11 @@ export async function openProjectEngine(options: EngineOptions) {
const fileWrites = new Map<Id, Blob>(), digestTasks = new Map<Id, Promise<void>>();
let staged: readonly ResourceRecord[] = [];
let applying: RunApplyInput | undefined;
let importing: { job: ImportJob; expectedGeneration: number } | undefined;
const importRunners=new Map<Id,ReturnType<typeof createImportRunner>>(),importLoads=new Map<Id,Promise<void>>(),importHolds=new Map<symbol,Map<Id,()=>void>>();
let recovery: { controller: AbortController; release(): void; cancel():Promise<void> } | undefined;
const assertWritable = () => { if(closing||closed)throw fault('engine_closing');if(recovery)throw fault('recovery_in_progress'); };
const initial: RuntimeState = recovered?.runtime ?? Object.freeze({ recoveries:readMap(new Map()), runs: readMap(new Map()), acknowledged: createDocument(options.document.projectId, [...options.document.canvases.values()], [...options.document.elements.values()], options.document.tombstones), versions: readMap(options.versions), ledger: createLedger().getSnapshot(), receipts: readMap(new Map()), conflicts: readMap(new Map()), resources: readMap(new Map()), resourcePins: readMap(new Map()) });
const initial: RuntimeState = recovered?.runtime ?? Object.freeze({ imports:readMap(new Map()), recoveries:readMap(new Map()), runs: readMap(new Map()), acknowledged: createDocument(options.document.projectId, [...options.document.canvases.values()], [...options.document.elements.values()], options.document.tombstones), versions: readMap(options.versions), ledger: createLedger().getSnapshot(), receipts: readMap(new Map()), conflicts: readMap(new Map()), resources: readMap(new Map()), resourcePins: readMap(new Map()) });
const editor = createEditor({ registry: options.registry, document: options.document, recovery: recovered, runtime: initial, createId: options.createId, historyLimit: options.historyLimit,
policy(command, document) { if (closing || closed || recovery) return { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] }; return options.policy?.(command, document) ?? { enabled: true }; },
traversalPolicy(entry, direction) { return closing || closed || recovery ? { enabled: false, code: closing||closed?'engine_closing':'recovery_in_progress', targets: [] } : options.traversalPolicy?.(entry, direction) ?? { enabled: true }; },
@@ -83,7 +87,9 @@ export async function openProjectEngine(options: EngineOptions) {
for (const change of intent.changes) if (change.kind === 'element') basePresence[change.id] ??= change.before ? 'exists' : 'absent'; else if (change.kind === 'field') basePresence[change.target.id] ??= 'exists';
ledger.add(freeze({ ...intent, dependencies: [...dependencies] }), basePresence);
}
return Object.freeze({ ...previous, resources: readMap(resources), ledger: ledger.getSnapshot() });
const imports=new Map(previous.imports??[]);
if(importing){if(imports.get(importing.job.id)?.generation!==importing.expectedGeneration)throw fault('stale_import_job');imports.set(importing.job.id,importing.job);}
return Object.freeze({ ...previous, imports:readMap(imports), resources: readMap(resources), ledger: ledger.getSnapshot() });
},
});
const pending = new Map<number, EngineSnapshot>();
@@ -94,7 +100,7 @@ export async function openProjectEngine(options: EngineOptions) {
const task = (async () => {
while (pending.size) {
const [revision, snapshot] = pending.entries().next().value!;
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)).map(([id, bytes]) => ({ id, bytes }));
const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||[...(snapshot.runtime.imports?.values()??[])].some(job=>job.source?.fileId===id)).map(([id, bytes]) => ({ id, bytes }));
const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files });
if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt');
durableRootRevision = revision; pending.delete(revision);
@@ -304,21 +310,67 @@ export async function openProjectEngine(options: EngineOptions) {
const result=editor.updateRuntime(previous=>{const ledger=createLedger(previous.ledger);ledger.add(freeze(intent),{});return Object.freeze({...previous,ledger:ledger.getSnapshot()});},root.rootRevision);
if(result.status==='rejected')return result;queueMicrotask(()=>{if(!closing&&!closed)scheduler.notify([id]);});return{status:'applied',revision:root.revision,operationIds:[id]};
}
function getImport(id:Id){const job=editor.getSnapshot().runtime.imports?.get(id);if(!job)throw fault('import_missing',[id]);return job;}
function publishImport(job:ImportJob,expectedGeneration:number){
assertWritable();const permission=options.policy?.({type:'changes.apply',changes:[]},editor.getSnapshot().document);if(permission&&!permission.enabled)throw fault(permission.code,permission.targets);
const result=editor.updateRuntime(previous=>{const imports=new Map(previous.imports??[]);if(imports.get(job.id)?.generation!==expectedGeneration)throw fault('stale_import_job');imports.set(job.id,job);return Object.freeze({...previous,imports:readMap(imports)});});if(result.status==='rejected')throw fault(result.code,result.targets);
}
function holdImports(){const token=Symbol(),releases=new Map<Id,()=>void>();importHolds.set(token,releases);for(const[id,runner]of importRunners)releases.set(id,runner.hold());return()=>{if(!importHolds.delete(token))return;for(const release of releases.values())release();if(!closing&&!closed)for(const job of editor.getSnapshot().runtime.imports?.values()??[])if(job.status==='running'&&!importRunners.has(job.id))void loadImport(job.id);};}
function loadImport(id:Id):Promise<void>{
if(importLoads.has(id))return importLoads.get(id)!;
if(importRunners.has(id)){void importRunners.get(id)!.start();return Promise.resolve();}
const expectedFingerprint=getImport(id).sourceFingerprint;
const task=(async()=>{
const job=getImport(id),source=job.source;if(!source||source.kind!=='native-ndjson'||job.parserVersion!==1)throw fault('import_interpreter_unavailable');
const bytes=fileWrites.get(source.fileId)??await options.storage.readFile(leader,source.fileId);
if(closing||closed)return;
const latest=getImport(id);if(latest.sourceFingerprint!==job.sourceFingerprint||latest.source?.fileId!==source.fileId||['completed','partial','cancelled'].includes(latest.status))return;
if(!bytes)throw fault('import_source_unavailable');if(bytes.size!==source.byteLength)throw fault('import_source_changed');
const runner=createImportRunner({optimisticLocal:true,source:createNdjsonImportSource({bytes,fingerprint:job.sourceFingerprint}),getJob:()=>getImport(id),createId,
publish:publishImport,prepare:(line,job,reserve)=>prepareNativeImport(line,job,reserve,editor.getSnapshot().document,options.registry),persist,onError:report,
commit(units,next,expected){
assertWritable();if(importing)throw fault('reentrant_dispatch');const current=getImport(id);if(current.generation!==expected)throw fault('stale_import_job');
try{assertNativeImportTarget(current,editor.getSnapshot().document);for(const unit of units)assertNativeImportDependencies(current,editor.getSnapshot().document,unit.dependencies);}catch(error){return reject(error);}
importing={job:next,expectedGeneration:expected};
try{return editor.transaction(tx=>{for(const unit of units)for(const change of unit.changes){if(change.kind==='element'&&change.before===null&&change.after)tx.dispatch({type:'element.create',element:change.after});else if(change.kind==='canvas'&&change.before===null&&change.after)tx.dispatch({type:'canvas.create',canvas:change.after});else throw fault('invalid_native_import_unit');}},{label:'Import canvas content',groupId:`import:${id}`});}finally{importing=undefined;}
},
});
importRunners.set(id,runner);for(const releases of importHolds.values())releases.set(id,runner.hold());
if(getImport(id).status==='running')void runner.start();
})().catch(error=>{if(!closing&&!closed){try{const job=getImport(id);if(job.sourceFingerprint!==expectedFingerprint||['completed','partial','cancelled'].includes(job.status))return;publishImport(freeze({...job,generation:job.generation+1,status:'paused',error:(error as {code?:string}).code??'import_failed'}),job.generation);}catch{/* Recovery/identity fences retain the original job. */}report(error);}}).finally(()=>{importLoads.delete(id);});
importLoads.set(id,task);return task;
}
const imports=Object.freeze({
getJobs:()=>readMap(editor.getSnapshot().runtime.imports??new Map()),
startNdjson(bytes:Blob,targetCanvasId:Id):Id{
assertWritable();const canvas=editor.getSnapshot().document.canvases.get(targetCanvasId);if(!canvas)throw fault('canvas_not_loaded',[targetCanvasId]);if(!(bytes instanceof Blob)||bytes.size>64*1024*1024)throw fault('invalid_import_source');
const id=createId(),fileId=createId(),current=editor.getSnapshot().runtime;if(!id||!fileId||id===fileId||current.imports?.has(id)||current.resources.has(fileId)||fileWrites.has(fileId)||[...(current.imports?.values()??[])].some(job=>job.source?.fileId===fileId))throw fault('import_identity_conflict');
const entry=canvas.parent?editor.getSnapshot().document.elements.get(canvas.parent.entryElementId)!:undefined;
const job=freeze({...createImportJob({id,projectId:options.scope.projectId,sourceFingerprint:`local-file:${fileId}:${bytes.size}`,parserVersion:1,source:{kind:'native-ndjson',name:((bytes as Blob&{name?:string}).name||'画布数据').slice(0,255),fileId,byteLength:bytes.size,targetCanvasId,...(entry?{targetEntry:{id:entry.id,incarnation:entry.incarnation}}:{})}}),status:'running' as const});
fileWrites.set(fileId,bytes);
const result=editor.updateRuntime(previous=>{const jobs=new Map(previous.imports??[]);jobs.set(id,job);return Object.freeze({...previous,imports:readMap(jobs)});});
if(result.status==='rejected'){fileWrites.delete(fileId);throw fault(result.code,result.targets);}void loadImport(id);return id;
},
async pause(id:Id){assertWritable();const runner=importRunners.get(id);if(runner)await runner.pause();else{const job=getImport(id);if(job.status==='running'){publishImport(freeze({...job,generation:job.generation+1,status:'paused'}),job.generation);await persist();}}},
async resume(id:Id){assertWritable();const job=getImport(id);if(['completed','partial','cancelled'].includes(job.status))return;if(job.status==='running'){await loadImport(id);return;}await importRunners.get(id)?.settle();await persist();await scheduler.resume();assertWritable();const current=getImport(id);if(current.status!=='paused')return;const next={...current,generation:current.generation+1,status:'running' as const};delete next.error;publishImport(freeze(next),current.generation);await loadImport(id);},
async cancel(id:Id){assertWritable();const runner=importRunners.get(id);if(runner)await runner.cancel();else{const job=getImport(id);if(['running','paused'].includes(job.status)){const next={...job,generation:job.generation+1,status:'cancelled' as const};delete next.error;publishImport(freeze(next),job.generation);await persist();}}},
});
scheduler.start(); prepareDigests(); runTracker.notify([...initial.runs.keys()]);
for(const job of initial.imports?.values()??[])if(job.status==='running')void loadImport(job.id);
return Object.freeze({
editor, sessions,
editor, sessions, imports,
/** Read/preview under a scheduling hold. No request is rewritten or declared resolved here. */
async prepareRecovery(): Promise<RecoverySession> {
assertWritable();
if(!options.persistence.read)throw fault('canonical_reader_unavailable');
const releaseScheduler=scheduler.hold(),releaseRuns=runTracker.hold(),controller=new AbortController();
const active={controller,release(){if(recovery!==active)return;recovery=undefined;releaseRuns();releaseScheduler();},async cancel(){controller.abort();active.release();}};
const releaseScheduler=scheduler.hold(),releaseRuns=runTracker.hold(),releaseImports=holdImports(),controller=new AbortController();
const active={controller,release(){if(recovery!==active)return;recovery=undefined;releaseRuns();releaseScheduler();releaseImports();},async cancel(){controller.abort();active.release();}};
recovery=active;
sessions.cancel('recovery');
const assertCurrent=()=>{if(closing||closed)throw fault('engine_closing');if(recovery!==active||controller.signal.aborted)throw fault('recovery_session_closed');};
try {
// Both holds are acquired before awaiting either lane; already admitted callbacks finish normally.
await Promise.all([scheduler.settle(),runTracker.settle()]);assertCurrent();
await Promise.all([...importLoads.values()]);assertCurrent();await Promise.all([scheduler.settle(),runTracker.settle(),...[...importRunners.values()].map(runner=>runner.settle())]);assertCurrent();
while(digestTasks.size){await Promise.all([...digestTasks.values()]);assertCurrent();}
await persist();assertCurrent();
const staged=stageAppliedRecovery(editor.getSnapshot(),{persistence:options.persistence,runs:options.runs,assets:options.assets,now:runClock.now()});
@@ -445,7 +497,7 @@ export async function openProjectEngine(options: EngineOptions) {
if (!(file.bytes instanceof Blob) || !file.bytes.size || file.bytes.size > maxResourceBytes || !file.bytes.type.startsWith('image/') || typeof file.name !== 'string' || !file.name) throw fault('invalid_staged_file');
return freeze({ id: createId(), operationId: createId(), name: file.name, mimeType: file.bytes.type, byteLength: file.bytes.size });
});
if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || fileWrites.has(resource.id))) throw fault('resource_identity_conflict');
if (new Set(resources.flatMap(resource => [resource.id, resource.operationId])).size !== resources.length * 2 || resources.some(resource => editor.getSnapshot().runtime.resources.has(resource.id) || fileWrites.has(resource.id) || [...(editor.getSnapshot().runtime.imports?.values()??[])].some(job=>job.source?.fileId===resource.id))) throw fault('resource_identity_conflict');
staged = resources; resources.forEach((resource, index) => fileWrites.set(resource.id, files[index].bytes));
try {
const handles = freeze(resources.map(resource => ({ id: resource.id, src: options.assets!.sourceURL(resource.id) })));
@@ -468,10 +520,11 @@ export async function openProjectEngine(options: EngineOptions) {
cancelRecovery:async()=>{await recovery?.cancel();},
getDurability: () => Object.freeze({ rootRevision: editor.getSnapshot().rootRevision, durableRootRevision }),
retry: async () => { if(recovery)await recovery.cancel();assertWritable();await scheduler.resume();await runTracker.retry(); },
async settle() { await Promise.resolve(); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async settle() { await Promise.resolve();while(importLoads.size)await Promise.all([...importLoads.values()]);await Promise.all([...importRunners.values()].map(runner=>runner.settle())); while (digestTasks.size) await Promise.all([...digestTasks.values()]); await scheduler.settle(); await runTracker.settle(); await persist(); },
async close() {
if (closed) return;
closing = true; sessions.dispose(); scheduler.stop(); runTracker.stop();
await Promise.all([...importRunners.values()].map(runner=>runner.close()));while(importLoads.size)await Promise.all([...importLoads.values()]);
recovery?.controller.abort();recovery?.release();
for (const release of [...displays]) release();
// Failure keeps the leader/recovery material available for an explicit retry of close.
+7 -4
View File
@@ -1,3 +1,4 @@
import { validateImportJob, type ImportJob } from './imports.ts';
import { validateElisions, withLocalLifecycles } from './elision.ts';
import type { Change, CoreSnapshot, Document, HistoryEntry, HistoryState, Id, Json, OperationIntent, Registry } from '../canvas-core/types.ts';
import { validateHistoryReplay, replayHistoryEvidence, sameHistory } from './history-replay.ts';
@@ -18,10 +19,10 @@ export function encodeRecovery(snapshot: EngineSnapshot): RecoveryImage {
document: documentValue(snapshot.document),
history: { undo: snapshot.history.undo, redo: snapshot.history.redo, entries: [...snapshot.history.entries] },
operations: [...snapshot.operations],
runtime: { recoveries:[...(snapshot.runtime.recoveries??[])], runs: [...snapshot.runtime.runs], resources: [...snapshot.runtime.resources], resourcePins: [...snapshot.runtime.resourcePins], acknowledged: documentValue(snapshot.runtime.acknowledged), versions: [...snapshot.runtime.versions], receipts: [...snapshot.runtime.receipts], conflicts: [...snapshot.runtime.conflicts], ledger: { ...snapshot.runtime.ledger, operations: [...snapshot.runtime.ledger.operations], requests: [...snapshot.runtime.ledger.requests] } },
runtime: { imports:[...(snapshot.runtime.imports??[])], recoveries:[...(snapshot.runtime.recoveries??[])], runs: [...snapshot.runtime.runs], resources: [...snapshot.runtime.resources], resourcePins: [...snapshot.runtime.resourcePins], acknowledged: documentValue(snapshot.runtime.acknowledged), versions: [...snapshot.runtime.versions], receipts: [...snapshot.runtime.receipts], conflicts: [...snapshot.runtime.conflicts], ledger: { ...snapshot.runtime.ledger, operations: [...snapshot.runtime.ledger.operations], requests: [...snapshot.runtime.ledger.requests] } },
})) as Json;
if (!validJson(payload)) throw fault('invalid_recovery_payload');
return freeze({ formatVersion: 1, rootRevision: snapshot.rootRevision, localRevision: snapshot.revision, payload });
return freeze({ formatVersion: snapshot.runtime.imports?.size ? 2 : 1, rootRevision: snapshot.rootRevision, localRevision: snapshot.revision, payload });
}
type ObjectValue = Record<string, unknown>;
function object(value: unknown): ObjectValue { if (!value || Array.isArray(value) || typeof value !== 'object') throw fault('corrupt_recovery_image'); return value as ObjectValue; }
@@ -84,13 +85,14 @@ function readHistory(history:ObjectValue,operations:ReadonlyMap<Id,OperationInte
return Object.freeze({undo:freeze(undo),redo:freeze(redo),entries});
}
export function decodeRecovery(image: RecoveryImage, registry: Registry): CoreSnapshot & { readonly rootRevision: number; readonly runtime: RuntimeState } {
if (image.formatVersion !== 1) throw fault('unsupported_recovery_version');
if (![1,2].includes(image.formatVersion)) throw fault('unsupported_recovery_version');
integer(image.rootRevision); integer(image.localRevision);
if (image.localRevision > image.rootRevision || !validJson(image.payload)) throw fault('corrupt_recovery_image');
// Own the parsed records; mutating a caller's RecoveryImage must not mutate live state.
const raw = object(JSON.parse(JSON.stringify(image.payload))), document = readDocument(raw.document, registry), history = object(raw.history), runtime = object(raw.runtime), ledger = object(runtime.ledger);
const operations = table(raw.operations, intent),restoredHistory=readHistory(history,operations);
for(const operation of operations.values())if(operation.historyTransition?.version===1&&(operation.historyTransition.rootRevision<1||operation.historyTransition.rootRevision>image.rootRevision))throw fault('corrupt_recovery_image');
if(image.formatVersion===1&&array(runtime.imports??[]).length)throw fault('unsupported_recovery_version');
const records = table<OperationRecord>(ledger.operations, (value, id) => {
const raw = object(value); intent(raw.intent, id); strings(raw.blockers); object(raw.basePresence);
if (((object(raw.intent).operationType === 'document.change'||object(raw.intent).operationType==='run.apply'&&array(object(raw.intent).changes).length>0) && !operations.has(id)) || !['pending', 'frozen', 'resolved', 'blocked'].includes(string(raw.phase))) throw fault('corrupt_recovery_image');
@@ -221,8 +223,9 @@ export function decodeRecovery(image: RecoveryImage, registry: Registry): CoreSn
for(const value of array(raw.outputEvents ?? [])){const event=object(value),step=string(event.step_id);if(event.run_id!==id||event.type!=='step.output'||outputIds.has(step)||integer(event.seq,1)>Math.max(integer(raw.cursor),raw.pendingPage===undefined?0:integer(object(raw.pendingPage).next)))throw fault('corrupt_recovery_image');object(event.output);outputIds.add(step);}
return freeze({...value as RunRecord,outputEvents:array(raw.outputEvents ?? []) as never});
});
const imports=table<ImportJob>(runtime.imports??[],(value,id)=>{const job=value as ImportJob;validateImportJob(job);if(job.id!==id||job.projectId!==document.projectId)throw fault('corrupt_recovery_image');return freeze(job);});
const restoredLedger: LedgerSnapshot = Object.freeze({ generation, durableGeneration, operations: records, requests });
return Object.freeze({ revision: image.localRevision, rootRevision: image.rootRevision, document, history:restoredHistory, operations,
runtime: Object.freeze({ recoveries, runs, resources, resourcePins, ledger: restoredLedger, acknowledged, versions, receipts, conflicts: table(runtime.conflicts, string) }),
runtime: Object.freeze({ imports, recoveries, runs, resources, resourcePins, ledger: restoredLedger, acknowledged, versions, receipts, conflicts: table(runtime.conflicts, string) }),
});
}
+2 -2
View File
@@ -2,7 +2,7 @@ import type { Json } from '../canvas-core/types.ts';
import { clone, equal, fault, validJson } from '../canvas-core/values.ts';
export type StorageScope = { readonly accountId: string; readonly workspaceId: string; readonly projectId: string };
export type RecoveryImage = { readonly formatVersion: 1; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json };
export type RecoveryImage = { readonly formatVersion: 1 | 2; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json };
export type DurableBatch = { readonly image: RecoveryImage; readonly files?: readonly { readonly id: string; readonly bytes: Blob }[] };
export type StorageLeader = { readonly scope: StorageScope; readonly fence: number; readonly active: boolean; release(): Promise<void> };
export type StoragePort = {
@@ -23,7 +23,7 @@ const keyFor = (scope: StorageScope) => {
return JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]);
};
function validateImage(image: RecoveryImage) {
if (image.formatVersion !== 1) throw fault('unsupported_recovery_version');
if (![1,2].includes(image.formatVersion)) throw fault('unsupported_recovery_version');
if (!Number.isSafeInteger(image.rootRevision) || image.rootRevision < 0 || !Number.isSafeInteger(image.localRevision) || image.localRevision < 0 || image.localRevision > image.rootRevision || !validJson(image.payload)) throw fault('corrupt_recovery_image');
}
const requestValue = <T>(request: IDBRequest<T>) => new Promise<T>((resolve, reject) => { request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); });