Stream native imports before full source verification with durable send seals

This commit is contained in:
2026-11-09 23:24:53 +08:00
parent f11ea36b48
commit 0ce3abbe04
17 changed files with 230 additions and 63 deletions
+22 -5
View File
@@ -1,3 +1,4 @@
import {validateImportReadProof,type ImportReadProof} from './import-read-proof.ts';
import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
import {validateImportFileRetention,type FileRetirement,type PreviousImportFile} from './import-file-retention.ts';
@@ -12,7 +13,7 @@ export type ImportUnit = { readonly key: string; readonly dependencies: readonly
export type ImportLocator = { readonly key: string; readonly cursor: number; readonly nextCursor: number; readonly dependencies: readonly string[]; readonly digest: string; readonly byteLength: number; readonly changeCount: number };
export type ImportFailure = { readonly key: string; readonly cursor: number; readonly code: string; readonly dependencies?: readonly string[] };
export type LoadingImportAdmission={readonly version:1;readonly sourceToken:string;readonly operationSequence:number;readonly requiredCanvasIds:readonly Id[]};
export type NativeImportSource = { readonly loadingAdmission?:LoadingImportAdmission; readonly kind: 'native-document' | 'native-json-array' | 'native-ndjson' | 'duplicate-ndjson' | 'duplicate-snapshot' | 'duplicate-plan'; readonly retirement?:FileRetirement;readonly previousFiles?:readonly PreviousImportFile[];readonly snapshotVersion?:2;readonly plan?:DuplicateCheckpoint; readonly copy?: {readonly sourceRevision:number; readonly roots:readonly Id[]}; readonly integrityVersion?:1;readonly sha256?:string; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
export type NativeImportSource = { readonly loadingAdmission?:LoadingImportAdmission; readonly kind: 'native-document' | 'native-json-array' | 'native-ndjson' | 'duplicate-ndjson' | 'duplicate-snapshot' | 'duplicate-plan'; readonly retirement?:FileRetirement;readonly previousFiles?:readonly PreviousImportFile[];readonly snapshotVersion?:2;readonly plan?:DuplicateCheckpoint; readonly copy?: {readonly sourceRevision:number; readonly roots:readonly Id[]}; readonly integrityVersion?:1|2;readonly readProof?:ImportReadProof;readonly integrityError?:string;readonly sha256?:string; readonly name?:string; readonly fileId: Id; readonly byteLength: number; readonly targetCanvasId: Id; readonly targetEntry?: {readonly id:Id;readonly incarnation:number} };
export type ImportBinding = {readonly kind:'element';readonly incarnation:number}|{readonly kind:'canvas';readonly entryElementId?:Id};
export type ImportJob = {
readonly version: 1 | 2 | 3; readonly id: Id; readonly projectId: Id; readonly sourceFingerprint: string; readonly parserVersion: number;
@@ -22,6 +23,7 @@ export type ImportJob = {
readonly bindings?: Readonly<Record<Id,ImportBinding>>;
readonly stableIds: Readonly<Record<string, Id>>; readonly pending: readonly ImportLocator[]; readonly committedSourceKeys: readonly string[]; readonly failures: readonly ImportFailure[]; readonly error?: string;
};
export const importNeedsSeal=(job:ImportJob)=>job.source?.integrityVersion===2&&!(job.status==='cancelled'&&!job.committedSourceKeys.length)&&(!job.source.sha256||!!job.source.integrityError);
export function createImportJob(input: Pick<ImportJob, 'id' | 'projectId' | 'sourceFingerprint' | 'parserVersion' | 'source'>): ImportJob {
if (![input.id,input.projectId,input.sourceFingerprint].every(value=>typeof value==='string'&&!!value) || !Number.isSafeInteger(input.parserVersion) || input.parserVersion<1) throw fault('invalid_import_job');
return freeze({ id: input.id, projectId: input.projectId, sourceFingerprint: input.sourceFingerprint, parserVersion: input.parserVersion, ...(input.source?{source:{...clone(input.source),integrityVersion:input.source.integrityVersion??1}}:{}), version: 3, generation: 0, status: 'paused', cursor: 0, eof: false, bindings: {}, stableIds: {}, pending: [], committedSourceKeys: [], failures: [] });
@@ -38,7 +40,12 @@ export function validateImportJob(job: ImportJob): void {
if(job.source){
validateImportFileRetention(job.source,job.status);
if(job.source.snapshotVersion!==undefined&&(job.source.snapshotVersion!==2||!['duplicate-snapshot','duplicate-plan'].includes(job.source.kind)))throw fault('copy_snapshot_interpreter_unavailable');
if(job.source.integrityVersion!==undefined&&job.source.integrityVersion!==1)throw fault('import_source_integrity_interpreter_unavailable');
if(job.source.integrityVersion!==undefined&&![1,2].includes(job.source.integrityVersion))throw fault('import_source_integrity_interpreter_unavailable');
if(job.source.integrityVersion===2){
if(!['native-document','native-json-array','native-ndjson'].includes(job.source.kind)||job.source.kind==='native-document'&&job.parserVersion!==5)throw fault('import_source_integrity_interpreter_unavailable');
const proof=validateImportReadProof(job.source.readProof,job.source.fileId,job.source.byteLength);if(proof.blocks.some((block,index)=>block.index!==index)||job.source.byteLength>0&&!proof.blocks.length&&(job.cursor>0||job.eof||job.pending.length||job.committedSourceKeys.length||Object.keys(job.stableIds).length))throw fault('import_source_integrity_missing');
if(job.source.integrityError!==undefined&&(typeof job.source.integrityError!=='string'||!job.source.integrityError))throw fault('invalid_import_source');
}else if(job.source.readProof!==undefined||job.source.integrityError!==undefined)throw fault('import_source_integrity_interpreter_unavailable');
if(job.source.sha256!==undefined&&(typeof job.source.sha256!=='string'||!(/^[a-f0-9]{64}$/.test(job.source.sha256))))throw fault('invalid_import_source');
if(job.source.integrityVersion===1&&!job.source.sha256&&(job.cursor!==0||job.eof||job.pending.length||job.committedSourceKeys.length||Object.keys(job.stableIds).length||(job.preparation?.allocationIds.length??0)>0||(job.preparation?.steps??0)>0))throw fault('import_source_integrity_missing');
}
@@ -98,6 +105,7 @@ export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;
}
export type ImportRunnerPorts={
checkpointSource?:(job:ImportJob)=>ImportJob;
source:ImportSource;getJob():ImportJob;createId():Id;
/** Synchronous compare-and-publish; never write a second private job store. */
publish(next:ImportJob,expectedGeneration:number):void;
@@ -120,13 +128,22 @@ export function createImportRunner(ports:ImportRunnerPorts){
const initial=ports.getJob(),identity={id:initial.id,projectId:initial.projectId,sourceFingerprint:initial.sourceFingerprint,parserVersion:initial.parserVersion},budget=ports.frameBudgetMs??6,maxPending=ports.maxPending??4096,maxBytes=ports.maxPendingBytes??1024*1024,maxBatch=ports.maxBatch??32,now=ports.now??(()=>performance.now());
validateImportJob(initial);
if(initial.source?.kind==='duplicate-snapshot')throw fault('import_interpreter_unavailable');
if(initial.source&&(!initial.source.sha256||initial.source.integrityVersion!==1))throw fault('import_source_integrity_unavailable');
if(initial.source&&(initial.source.integrityVersion===2?!ports.checkpointSource:!initial.source.sha256||initial.source.integrityVersion!==1))throw fault('import_source_integrity_unavailable');
if(!Number.isFinite(budget)||budget<=0||![maxPending,maxBytes,maxBatch].every(value=>Number.isSafeInteger(value)&&value>0))throw fault('invalid_import_budget');
let costPerUnit:number|undefined;
let active:Promise<void>|undefined,controller:AbortController|undefined,closed=false,restart=false;const holds=new Set<symbol>();
const report=(error:unknown)=>{try{ports.onError(error);}catch{/* Reporting cannot change imported content. */}};
function current(){const job=ports.getJob();if(job.id!==identity.id||job.projectId!==identity.projectId||job.sourceFingerprint!==identity.sourceFingerprint||job.parserVersion!==identity.parserVersion)throw fault('import_identity_changed');return job;}
function next(job:ImportJob,patch:Partial<ImportJob>):ImportJob{const value={...job,...patch,version:3 as const,generation:job.generation+1};if(value.error===undefined)delete value.error;return freeze(value);}
function checkpointEvidence(job:ImportJob):ImportJob{
if(!ports.checkpointSource)return job;
const value=ports.checkpointSource(job);if(!value||!equal({...job,source:null},{...value,source:null}))throw fault('invalid_import_checkpoint');
const strip=(source:ImportJob['source'])=>{if(!source)return source;const{readProof:_,sha256:__,integrityError:___,...identity}=source;return identity;};
if(!equal(strip(job.source),strip(value.source)))throw fault('invalid_import_checkpoint');
if(value.source?.integrityVersion===2){validateImportReadProof(value.source.readProof,value.source.fileId,value.source.byteLength);if(value.source.sha256!==undefined&&(typeof value.source.sha256!=='string'||!/^[a-f0-9]{64}$/.test(value.source.sha256)))throw fault('invalid_import_checkpoint');if(value.source.integrityError!==undefined&&(typeof value.source.integrityError!=='string'||!value.source.integrityError))throw fault('invalid_import_checkpoint');}
else if(!equal(job.source,value.source))throw fault('invalid_import_checkpoint');
return value;
}
function next(job:ImportJob,patch:Partial<ImportJob>):ImportJob{const value={...job,...patch,version:3 as const,generation:job.generation+1};if(value.error===undefined)delete value.error;return freeze(checkpointEvidence(value));}
function publish(job:ImportJob,patch:Partial<ImportJob>){const value=next(job,patch);ports.publish(value,job.generation);if(!equal(current(),value))throw fault('invalid_import_publish');return value;}
function prepare(line:ImportLine,job:ImportJob){
const ids:Record<string,Id>=Object.assign(Object.create(null),job.stableIds),used=new Set(Object.values(ids));
@@ -144,7 +161,7 @@ export function createImportRunner(ports:ImportRunnerPorts){
for(const unit of units)for(const change of unit.changes)if(change.kind==='element'&&change.before===null&&change.after)bindings[change.id]={kind:'element',incarnation:change.after.incarnation};else if(change.kind==='canvas'&&change.before===null&&change.after)bindings[change.id]={kind:'canvas',...(change.after.parent?{entryElementId:change.after.parent.entryElementId}:{})};
return next(job,{pending,committedSourceKeys:[...job.committedSourceKeys,...units.map(unit=>unit.key)],bindings:{...bindings}});
}
function commit(units:readonly ImportUnit[],job:ImportJob,value:ImportJob){const start=now(),result=ports.commit(units,value,job.generation),cost=Math.max(.01,(now()-start)/units.length);costPerUnit=costPerUnit===undefined?cost:Math.max(cost,costPerUnit*.8+cost*.2);if(!['applied','rejected'].includes(result?.status)||!equal(current(),result.status==='applied'?value:job))throw fault('invalid_import_commit');return result;}
function commit(units:readonly ImportUnit[],job:ImportJob,value:ImportJob){const base=current();if(base.generation!==job.generation)throw fault('stale_import_job');value=freeze(checkpointEvidence(value));const start=now(),result=ports.commit(units,value,job.generation),cost=Math.max(.01,(now()-start)/units.length);costPerUnit=costPerUnit===undefined?cost:Math.max(cost,costPerUnit*.8+cost*.2);if(!['applied','rejected'].includes(result?.status)||!equal(current(),result.status==='applied'?value:base))throw fault('invalid_import_commit');return result;}
async function pump(signal:AbortSignal){
let frame=now(),didWork=false;
async function attempt(units:readonly ImportUnit[],job:ImportJob,proposed:ImportJob):Promise<Result|undefined>{