Add proof-backed parser byte reads and preserve retryable source failures
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
import {fault} from '../canvas-core/values.ts';
|
||||
|
||||
const failures=new Set(['import_source_changed','invalid_import_byte_range','import_read_capacity','import_source_closed','import_source_read_failed','invalid_import_read_proof','import_read_proof_interpreter_unavailable']);
|
||||
export const isImportByteFailure=(error:unknown)=>failures.has((error as {code?:string}|undefined)?.code??'');
|
||||
|
||||
/** Optional immutable-source port. Implementations return exactly the requested bytes. */
|
||||
export type ImportByteReader={readonly byteLength:number;read(start:number,end:number,signal:AbortSignal):Promise<Uint8Array>};
|
||||
export async function readImportRange(bytes:Blob,reader:ImportByteReader|undefined,start:number,end:number,signal:AbortSignal):Promise<Uint8Array<ArrayBuffer>>{
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||start>bytes.size||reader&&reader.byteLength!==bytes.size)throw fault('invalid_import_byte_range');
|
||||
const limit=Math.min(end,bytes.size);let value:Uint8Array;
|
||||
try{value=reader?await reader.read(start,limit,signal):new Uint8Array(await bytes.slice(start,end).arrayBuffer());}
|
||||
catch(error){if(signal.aborted)throw fault('import_aborted');if(isImportByteFailure(error))throw error;throw fault('import_source_read_failed');}
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
if(!(value instanceof Uint8Array)||value.byteLength!==limit-start)throw fault('import_source_changed');
|
||||
return reader?new Uint8Array(value):value as Uint8Array<ArrayBuffer>;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import {readImportRange,isImportByteFailure,type ImportByteReader} from './import-byte-reader.ts';
|
||||
import type {ImportLine,ImportSource} from './imports.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import {openDocumentImportSource} from './import-document.ts';
|
||||
@@ -6,7 +7,7 @@ import {fault,validJson} from '../canvas-core/values.ts';
|
||||
type Range={start:number;end:number};
|
||||
type ScanCache={start:number;bytes:Uint8Array};
|
||||
type Canvas={id:string;parent:null|{canvasId:string;entryElementId:string}};
|
||||
type Options=ImportScheduling&{bytes:Blob;fingerprint:string;signal:AbortSignal;maxRecordBytes?:number};
|
||||
type Options=ImportScheduling&{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;signal:AbortSignal;maxRecordBytes?:number};
|
||||
const allowed=new Set(['documentFormatVersion','projectId','canvases','elements','tombstones']);
|
||||
const whitespace=(byte:number)=>byte===9||byte===10||byte===13||byte===32;
|
||||
|
||||
@@ -17,7 +18,7 @@ function scanner(options:Options,signal:AbortSignal,start=0,cache:ScanCache={sta
|
||||
const check=()=>{if(signal.aborted)throw fault('import_aborted');};
|
||||
async function peek(){
|
||||
check();if(cursor>=bytes.size)return -1;
|
||||
if(cursor<cache.start||cursor>=cache.start+cache.bytes.length){const loaded=new Uint8Array(await bytes.slice(cursor,cursor+16384).arrayBuffer());cache.start=cursor;cache.bytes=loaded;check();}
|
||||
if(cursor<cache.start||cursor>=cache.start+cache.bytes.length){const loaded=await readImportRange(bytes,options.readBytes,cursor,cursor+16384,signal);cache.start=cursor;cache.bytes=loaded;check();}
|
||||
return cache.bytes[cursor-cache.start];
|
||||
}
|
||||
function step(){cursor++;return budget.step();}
|
||||
@@ -40,8 +41,8 @@ function scanner(options:Options,signal:AbortSignal,start=0,cache:ScanCache={sta
|
||||
}
|
||||
async function decode(range:Range,limit=options.maxRecordBytes??256*1024):Promise<any>{
|
||||
check();if(range.end-range.start>limit)throw fault('record_too_large');
|
||||
let text:string;
|
||||
try{text=new TextDecoder('utf-8',{fatal:true}).decode(await bytes.slice(range.start,range.end).arrayBuffer());}catch(error){check();throw fault('invalid_utf8');}
|
||||
const data=await readImportRange(bytes,options.readBytes,range.start,range.end,signal);let text:string;
|
||||
try{text=new TextDecoder('utf-8',{fatal:true}).decode(data);}catch(error){check();throw fault('invalid_utf8');}
|
||||
check();try{const value=JSON.parse(text);if(!validJson(value))throw Error();return value;}catch{throw fault('invalid_json');}
|
||||
}
|
||||
return{peek,step,spaces,take,value,decode,position:()=>cursor};
|
||||
@@ -124,7 +125,7 @@ export async function openStreamingDocumentImportSource(input:Options):Promise<I
|
||||
if(!Number.isSafeInteger(position)||position<0)throw fault('invalid_import_cursor');
|
||||
const elementIndex=position-children.length;
|
||||
while(elementIndex>=ranges.length&&phase!=='done'){
|
||||
try{await scanNext(readSignal);}catch(error){if(readSignal.aborted)throw fault('import_aborted');failure=(error as {code?:string}).code??'invalid_document_import';phase='done';}
|
||||
try{await scanNext(readSignal);}catch(error){if(readSignal.aborted)throw fault('import_aborted');if(isImportByteFailure(error))throw error;failure=(error as {code?:string}).code??'invalid_document_import';phase='done';}
|
||||
}
|
||||
const count=children.length+ranges.length;
|
||||
if(position>=count){
|
||||
@@ -137,7 +138,7 @@ export async function openStreamingDocumentImportSource(input:Options):Promise<I
|
||||
const kind=position<children.length?'canvas':'element',record=kind==='canvas'?children[position]:await scanner(options,readSignal).decode(ranges[elementIndex]);
|
||||
if(!record||typeof record!=='object'||Array.isArray(record)||typeof record.id!=='string'||!record.id)throw fault('invalid_native_import_record');
|
||||
return{...base,value:{kind,key:record.id,record,sourceRootId:rootId}};
|
||||
}catch(error){if(readSignal.aborted)throw fault('import_aborted');return{...base,error:(error as {code?:string}).code??'invalid_document_import'};}
|
||||
}catch(error){if(readSignal.aborted)throw fault('import_aborted');if(isImportByteFailure(error))throw error;return{...base,error:(error as {code?:string}).code??'invalid_document_import'};}
|
||||
}
|
||||
return Object.freeze({fingerprint:options.fingerprint,parserVersion:5,read(position:number,readSignal:AbortSignal){
|
||||
// A slow future scan must not block re-reading a pending unit for commit/recovery.
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
import {readImportRange,isImportByteFailure,type ImportByteReader} from './import-byte-reader.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import type {ImportSource,ImportLine} from './imports.ts';
|
||||
import {fault,validJson} from '../canvas-core/values.ts';
|
||||
type Range={start:number;end:number};
|
||||
/** Index the envelope without decoding element payloads. Original bytes remain the source.
|
||||
* Preparation scans O(bytes) and retains O(records) ranges; this is not constant memory. */
|
||||
export async function openDocumentImportSource(options:{bytes:Blob;fingerprint:string;signal:AbortSignal;maxRecordBytes?:number}&ImportScheduling):Promise<ImportSource>{
|
||||
export async function openDocumentImportSource(options:{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;signal:AbortSignal;maxRecordBytes?:number}&ImportScheduling):Promise<ImportSource>{
|
||||
const {bytes,signal}=options,max=options.maxRecordBytes??256*1024;
|
||||
if(!(bytes instanceof Blob)||bytes.size>64*1024*1024||!options.fingerprint||!Number.isSafeInteger(max)||max<1)throw fault('invalid_import_source');
|
||||
let cursor=0,cache={start:-1,bytes:new Uint8Array()};const budget=createImportBudget(signal,options);
|
||||
const space=(b:number)=>[9,10,13,32].includes(b);
|
||||
async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor<cache.start||cursor>=cache.start+cache.bytes.length){cache={start:cursor,bytes:new Uint8Array(await bytes.slice(cursor,cursor+16384).arrayBuffer())};if(signal.aborted)throw fault('import_aborted');}return cache.bytes[cursor-cache.start];}
|
||||
async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor<cache.start||cursor>=cache.start+cache.bytes.length){cache={start:cursor,bytes:await readImportRange(bytes,options.readBytes,cursor,cursor+16384,signal)};if(signal.aborted)throw fault('import_aborted');}return cache.bytes[cursor-cache.start];}
|
||||
function step(){cursor++;return budget.step();}
|
||||
async function spaces(){while(space(await peek()))await step();}
|
||||
async function take(byte:number){await spaces();if(await peek()!==byte)throw fault('invalid_document_import');await step();}
|
||||
@@ -19,7 +20,7 @@ export async function openDocumentImportSource(options:{bytes:Blob;fingerprint:s
|
||||
const ended=!quoted&&stack.length===0&&(b===34||b===93||b===125);await step();if(ended)break;
|
||||
}if(cursor===start)throw fault('invalid_document_import');return{start,end:cursor};
|
||||
}
|
||||
async function decode(range:Range,limit=max){if(range.end-range.start>limit)throw fault('record_too_large');let text:string;try{text=new TextDecoder('utf-8',{fatal:true}).decode(await bytes.slice(range.start,range.end).arrayBuffer());}catch{throw fault('invalid_utf8');}try{const result=JSON.parse(text);if(!validJson(result))throw Error();return result;}catch{throw fault('invalid_json');}}
|
||||
async function decode(range:Range,limit=max,readSignal=signal){if(range.end-range.start>limit)throw fault('record_too_large');const data=await readImportRange(bytes,options.readBytes,range.start,range.end,readSignal);let text:string;try{text=new TextDecoder('utf-8',{fatal:true}).decode(data);}catch{throw fault('invalid_utf8');}try{const result=JSON.parse(text);if(!validJson(result))throw Error();return result;}catch{throw fault('invalid_json');}}
|
||||
const fields=new Map<string,unknown>(),elements:Range[]=[];
|
||||
await take(123);await spaces();if(await peek()===125)throw fault('invalid_document_import');
|
||||
while(true){const key=await decode(await value(),1024);if(typeof key!=='string'||fields.has(key))throw fault('invalid_document_import');await take(58);
|
||||
@@ -38,6 +39,6 @@ export async function openDocumentImportSource(options:{bytes:Blob;fingerprint:s
|
||||
return Object.freeze({fingerprint:options.fingerprint,parserVersion:4,async read(position:number,readSignal:AbortSignal):Promise<ImportLine|null>{
|
||||
if(readSignal.aborted)throw fault('import_aborted');if(!Number.isSafeInteger(position)||position<0||position>count)throw fault('invalid_import_cursor');if(position===count)return null;
|
||||
const base={cursor:position,nextCursor:position+1,key:`record:${position}`};let record:any,kind:string;
|
||||
try{if(position<children.length){kind='canvas';record=children[position];}else{kind='element';record=await decode(elements[position-children.length]);}if(readSignal.aborted)throw fault('import_aborted');if(!record||typeof record!=='object'||Array.isArray(record)||typeof record.id!=='string'||!record.id)throw fault('invalid_native_import_record');return{...base,value:{kind,key:record.id,record,sourceRootId:rootId}};}catch(error){if(readSignal.aborted)throw fault('import_aborted');return{...base,error:(error as {code?:string}).code??'invalid_document_import'};}
|
||||
try{if(position<children.length){kind='canvas';record=children[position];}else{kind='element';record=await decode(elements[position-children.length],max,readSignal);}if(readSignal.aborted)throw fault('import_aborted');if(!record||typeof record!=='object'||Array.isArray(record)||typeof record.id!=='string'||!record.id)throw fault('invalid_native_import_record');return{...base,value:{kind,key:record.id,record,sourceRootId:rootId}};}catch(error){if(readSignal.aborted)throw fault('import_aborted');if(isImportByteFailure(error))throw error;return{...base,error:(error as {code?:string}).code??'invalid_document_import'};}
|
||||
}});
|
||||
}
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import type {ImportSource,ImportLine} from './imports.ts';
|
||||
import {fault,validJson} from '../canvas-core/values.ts';
|
||||
/** Seekable JSON array reader. Cursors are byte offsets after '[' or a record comma.
|
||||
* Structure errors end the source; previously published records remain explicit. */
|
||||
export function createJsonArrayImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource{
|
||||
export function createJsonArrayImportSource(options:{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource{
|
||||
const bytes=options.bytes,max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16384;
|
||||
if(!(bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source');
|
||||
let cache={offset:-1,value:new Uint8Array()};const whitespace=(b:number)=>[9,10,13,32].includes(b);
|
||||
@@ -12,7 +13,7 @@ export function createJsonArrayImportSource(options:{bytes:Blob;fingerprint:stri
|
||||
if(start>0&&start===bytes.size){if(signal.aborted)throw fault('import_aborted');return null;}
|
||||
if(bytes.size===0)throw fault('invalid_json_array');
|
||||
let cursor=start;const budget=createImportBudget(signal,options);
|
||||
async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor<cache.offset||cursor>=cache.offset+cache.value.length){cache={offset:cursor,value:new Uint8Array(await bytes.slice(cursor,cursor+chunk).arrayBuffer())};if(signal.aborted)throw fault('import_aborted');}return cache.value[cursor-cache.offset];}
|
||||
async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor<cache.offset||cursor>=cache.offset+cache.value.length){cache={offset:cursor,value:await readImportRange(bytes,options.readBytes,cursor,cursor+chunk,signal)};if(signal.aborted)throw fault('import_aborted');}return cache.value[cursor-cache.offset];}
|
||||
function advance(){cursor++;return budget.step();}
|
||||
async function spaces(){while(whitespace(await peek()))await advance();}
|
||||
const failure=(error:string):ImportLine=>({cursor:start,nextCursor:bytes.size,key:`byte:${start}`,error});
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import {fault,freeze,validJson} from '../canvas-core/values.ts';
|
||||
import {createSha256} from './sha256.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import type {ImportByteReader} from './import-byte-reader.ts';
|
||||
|
||||
export const IMPORT_PROOF_BLOCK_BYTES=16384;
|
||||
const MAX_SOURCE_BYTES=64*1024*1024;
|
||||
export type ImportReadProof={readonly version:1;readonly fileId:string;readonly byteLength:number;readonly blockBytes:16384;readonly blocks:readonly {readonly index:number;readonly sha256:string}[]};
|
||||
/** This binds only read blocks. It is not a whole-source digest or permission to send mutations. */
|
||||
export function validateImportReadProof(value:unknown,fileId:string,byteLength:number):ImportReadProof{
|
||||
if(typeof fileId!=='string'||!fileId||!Number.isSafeInteger(byteLength)||byteLength<0||byteLength>MAX_SOURCE_BYTES)throw fault('invalid_import_read_proof');
|
||||
const proof=value as ImportReadProof;
|
||||
if(!proof||!validJson(proof)||proof.version!==1)throw fault('import_read_proof_interpreter_unavailable');
|
||||
if(proof.fileId!==fileId||proof.byteLength!==byteLength||proof.blockBytes!==IMPORT_PROOF_BLOCK_BYTES||!Array.isArray(proof.blocks)||Object.keys(proof).some(key=>!['version','fileId','byteLength','blockBytes','blocks'].includes(key)))throw fault('invalid_import_read_proof');
|
||||
let last=-1;const count=Math.ceil(byteLength/IMPORT_PROOF_BLOCK_BYTES);
|
||||
if(proof.blocks.length>count)throw fault('invalid_import_read_proof');
|
||||
for(const block of proof.blocks){if(!block||!Number.isSafeInteger(block.index)||block.index<=last||block.index>=count||typeof block.sha256!=='string'||!/^[a-f0-9]{64}$/.test(block.sha256)||Object.keys(block).some(key=>!['index','sha256'].includes(key)))throw fault('invalid_import_read_proof');last=block.index;}
|
||||
return freeze({version:1 as const,fileId,byteLength,blockBytes:IMPORT_PROOF_BLOCK_BYTES,blocks:proof.blocks.map(block=>({...block}))});
|
||||
}
|
||||
|
||||
/** Bounded, seekable verified reads for parsers; checkpoints contain hashes, never byte buffers.
|
||||
* Restored evidence must be checked with verify() before a runner resumes business changes. */
|
||||
export function createImportReadProof(input:{bytes:Blob;fileId:string;proof?:ImportReadProof;maxResidentBlocks?:number;maxReadBytes?:number}&ImportScheduling){
|
||||
const options=Object.freeze({...input});
|
||||
const {bytes,fileId}=options,residentLimit=options.maxResidentBlocks??4,readLimit=options.maxReadBytes??4*1024*1024;
|
||||
if(!(bytes instanceof Blob)||!fileId||typeof fileId!=='string'||bytes.size>MAX_SOURCE_BYTES||!Number.isSafeInteger(residentLimit)||residentLimit<1||residentLimit>64||!Number.isSafeInteger(readLimit)||readLimit<1||readLimit>4*1024*1024)throw fault('invalid_import_read_source');
|
||||
const initial:ImportReadProof=options.proof?validateImportReadProof(options.proof,fileId,bytes.size):freeze({version:1 as const,fileId,byteLength:bytes.size,blockBytes:IMPORT_PROOF_BLOCK_BYTES,blocks:[]});
|
||||
const hashes=new Map(initial.blocks.map(block=>[block.index,block.sha256]));
|
||||
type Block={index:number;sha256:string;data:Uint8Array};
|
||||
const resident=new Map<number,Block>(),pending=new Map<number,Promise<Block>>(),lifecycle=new AbortController();
|
||||
let closed=false,activeReads=0,proof:ImportReadProof=initial;
|
||||
const check=(signal:AbortSignal)=>{if(closed)throw fault('import_source_closed');if(signal.aborted)throw fault('import_aborted');};
|
||||
function wait<T>(task:Promise<T>,signal:AbortSignal):Promise<T>{
|
||||
check(signal);return new Promise((resolve,reject)=>{const abort=()=>{try{check(signal);}catch(error){reject(error);}};signal.addEventListener('abort',abort,{once:true});task.then(value=>{signal.removeEventListener('abort',abort);try{check(signal);resolve(value);}catch(error){reject(error);}},error=>{signal.removeEventListener('abort',abort);reject(error);});});
|
||||
}
|
||||
function block(index:number):Promise<Block>{
|
||||
const cached=resident.get(index);if(cached){resident.delete(index);resident.set(index,cached);return Promise.resolve(cached);}
|
||||
const active=pending.get(index);if(active)return active;
|
||||
if(pending.size>=4)throw fault('import_read_capacity');
|
||||
const task=(async()=>{
|
||||
const start=index*IMPORT_PROOF_BLOCK_BYTES,end=Math.min(start+IMPORT_PROOF_BLOCK_BYTES,bytes.size),data=new Uint8Array(await bytes.slice(start,end).arrayBuffer());
|
||||
if(closed)throw fault('import_source_closed');if(data.byteLength!==end-start)throw fault('import_source_changed');
|
||||
const hash=createSha256();hash.update(data);const sha256=hash.hex(),expected=hashes.get(index);
|
||||
if(expected!==undefined&&expected!==sha256)throw fault('import_source_changed');
|
||||
const value={index,sha256,data};resident.set(index,value);while(resident.size>residentLimit)resident.delete(resident.keys().next().value!);return value;
|
||||
})();
|
||||
pending.set(index,task);void task.finally(()=>{if(pending.get(index)===task)pending.delete(index);}).catch(()=>{});return task;
|
||||
}
|
||||
const port:ImportByteReader=Object.freeze({byteLength:bytes.size,async read(start:number,end:number,signal:AbortSignal){
|
||||
const combined=AbortSignal.any([signal,lifecycle.signal]);check(combined);
|
||||
if(!Number.isSafeInteger(start)||!Number.isSafeInteger(end)||start<0||end<start||end>bytes.size||end-start>readLimit)throw fault('invalid_import_byte_range');
|
||||
if(start===end)return new Uint8Array();
|
||||
if(activeReads>=4||pending.size>=4&&!resident.has(Math.floor(start/IMPORT_PROOF_BLOCK_BYTES))&&!pending.has(Math.floor(start/IMPORT_PROOF_BLOCK_BYTES)))throw fault('import_read_capacity');
|
||||
activeReads++;try{
|
||||
const output=new Uint8Array(end-start),used:Pick<Block,'index'|'sha256'>[]=[],budget=createImportBudget(combined,options);
|
||||
for(let index=Math.floor(start/IMPORT_PROOF_BLOCK_BYTES);index*IMPORT_PROOF_BLOCK_BYTES<end;index++){
|
||||
const value=await wait(block(index),combined),offset=index*IMPORT_PROOF_BLOCK_BYTES,from=Math.max(start,offset),to=Math.min(end,offset+value.data.byteLength);
|
||||
output.set(value.data.subarray(from-offset,to-offset),from-start);used.push({index:value.index,sha256:value.sha256});await budget.step(128);
|
||||
}
|
||||
check(combined);for(const value of used)if(hashes.has(value.index)&&hashes.get(value.index)!==value.sha256)throw fault('import_source_changed');
|
||||
let changed=false;for(const value of used)if(!hashes.has(value.index)){hashes.set(value.index,value.sha256);changed=true;}
|
||||
if(changed)proof=freeze({version:1 as const,fileId,byteLength:bytes.size,blockBytes:IMPORT_PROOF_BLOCK_BYTES,blocks:[...hashes].sort(([a],[b])=>a-b).map(([index,sha256])=>({index,sha256}))});
|
||||
return output;
|
||||
}finally{activeReads--;}
|
||||
}});
|
||||
return Object.freeze({port,getProof:()=>proof,
|
||||
async verify(signal:AbortSignal){const combined=AbortSignal.any([signal,lifecycle.signal]),budget=createImportBudget(combined,options);check(combined);for(const index of [...hashes.keys()]){await wait(block(index),combined);await budget.step(128);}check(combined);},
|
||||
close(){if(closed)return;closed=true;resident.clear();lifecycle.abort();},
|
||||
stats:()=>Object.freeze({residentBlocks:resident.size,activeReads,pendingBlocks:pending.size,proofBlocks:hashes.size,closed}),
|
||||
});
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import {readImportRange,type ImportByteReader} from './import-byte-reader.ts';
|
||||
import {createImportBudget,type ImportScheduling} from './import-budget.ts';
|
||||
import {validateImportFileRetention,type FileRetirement,type PreviousImportFile} from './import-file-retention.ts';
|
||||
import { validateDuplicateCheckpoint, type DuplicateCheckpoint } from '../canvas-core/duplicate-checkpoint.ts';
|
||||
@@ -67,7 +68,7 @@ export function validateImportJob(job: ImportJob): void {
|
||||
const yieldTask=()=>new Promise<void>(resolve=>setTimeout(resolve,0));
|
||||
|
||||
/** Seekable bounded NDJSON parser. Fingerprint must identify the host's immutable, durable byte source. */
|
||||
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
|
||||
export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;readBytes?:ImportByteReader;maxRecordBytes?:number;chunkBytes?:number}&ImportScheduling):ImportSource {
|
||||
const max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16*1024;
|
||||
if(!(options.bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source');
|
||||
const sourceBytes=options.bytes;
|
||||
@@ -79,7 +80,7 @@ export function createNdjsonImportSource(options:{bytes:Blob;fingerprint:string;
|
||||
const parts:Uint8Array[]=[],start=cursor,budget=createImportBudget(signal,options);let total=0,oversized=false;
|
||||
while(cursor<sourceBytes.size){
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:new Uint8Array(await sourceBytes.slice(cursor,cursor+chunk).arrayBuffer())};
|
||||
if(cursor<cache.offset||cursor>=cache.offset+cache.bytes.length)cache={offset:cursor,bytes:await readImportRange(sourceBytes,options.readBytes,cursor,cursor+chunk,signal)};
|
||||
if(signal.aborted)throw fault('import_aborted');
|
||||
const begin=cursor-cache.offset,newline=cache.bytes.indexOf(10,begin),end=newline<0?cache.bytes.length:newline,part=cache.bytes.subarray(begin,end);total+=part.length;
|
||||
if(total>max){oversized=true;parts.length=0;}else if(!oversized)parts.push(part);
|
||||
|
||||
Reference in New Issue
Block a user