Add proof-backed parser byte reads and preserve retryable source failures

This commit is contained in:
2026-11-09 22:47:49 +08:00
parent f0ee0736b7
commit f11ea36b48
9 changed files with 220 additions and 14 deletions
+3 -2
View File
@@ -1,3 +1,4 @@
import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
import {validateImportFileRetention,type FileRetirement,type PreviousImportFile} from './import-file-retention.ts';
import { validateDuplicateCheckpoint, type DuplicateCheckpoint } from '../canvas-core/duplicate-checkpoint.ts';
@@ -67,7 +68,7 @@ export function validateImportJob(job: ImportJob): void {
const yieldTask=()=>new Promise<void>(resolve=>setTimeout(resolve,0));
/** Seekable bounded NDJSON parser. Fingerprint must identify the host's immutable, durable byte source. */
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
const max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16*1024;
if(!(options.bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source');
const sourceBytes=options.bytes;
@@ -79,7 +80,7 @@ export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;
const parts:Uint8Array[]=[],start=cursor,budget=createImportBudget(signal,options);let total=0,oversized=false;
while(cursor<sourceBytes.size){
if(signal.aborted)throw fault('import_aborted');
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:new Uint8Array(await sourceBytes.slice(cursor,cursor+chunk).arrayBuffer())};
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:await readImportRange(sourceBytes,options.readBytes,cursor,cursor+chunk,signal)};
if(signal.aborted)throw fault('import_aborted');
const begin=cursor-cache.offset,newline=cache.bytes.indexOf(10,begin),end=newline<0?cache.bytes.length:newline,part=cache.bytes.subarray(begin,end);total+=part.length;
if(total>max){oversized=true;parts.length=0;}else if(!oversized)parts.push(part);