Add proof-backed parser byte reads and preserve retryable source failures
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import {validateImportFileRetention,type FileRetirement,type PreviousImportFile} from './import-file-retention.ts';
|
||||
import { validateDuplicateCheckpoint, type DuplicateCheckpoint } from '../canvas-core/duplicate-checkpoint.ts';
|
||||
@@ -67,7 +68,7 @@ export function validateImportJob(job: ImportJob): void {
|
||||
const yieldTask=()=>new Promise<void>(resolve=>setTimeout(resolve,0));
|
||||
|
||||
/** Seekable bounded NDJSON parser. Fingerprint must identify the host's immutable, durable byte source. */
|
||||
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
|
||||
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
|
||||
const max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16*1024;
|
||||
if(!(options.bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source');
|
||||
const sourceBytes=options.bytes;
|
||||
@@ -79,7 +80,7 @@ export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;
|
||||
const parts:Uint8Array[]=[],start=cursor,budget=createImportBudget(signal,options);let total=0,oversized=false;
|
||||
while(cursor<sourceBytes.size){
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:new Uint8Array(await sourceBytes.slice(cursor,cursor+chunk).arrayBuffer())};
|
||||
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:await readImportRange(sourceBytes,options.readBytes,cursor,cursor+chunk,signal)};
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
const begin=cursor-cache.offset,newline=cache.bytes.indexOf(10,begin),end=newline<0?cache.bytes.length:newline,part=cache.bytes.subarray(begin,end);total+=part.length;
|
||||
if(total>max){oversized=true;parts.length=0;}else if(!oversized)parts.push(part);
|
||||
|
||||
Reference in New Issue
Block a user