From c4dfaf50a519d612053683eab0466436f5e089e3 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 17:50:35 +0800 Subject: [PATCH] Add seekable bounded JSON array import source --- headless/README.md | 9 +++++ src/lib/canvas-runtime/import-json-array.ts | 37 +++++++++++++++++++++ src/lib/canvas-runtime/index.ts | 1 + tests/headless/import-json-array.test.mjs | 8 +++++ 4 files changed, 55 insertions(+) create mode 100644 src/lib/canvas-runtime/import-json-array.ts create mode 100644 tests/headless/import-json-array.test.mjs diff --git a/headless/README.md b/headless/README.md index 796abf0..881310e 100644 --- a/headless/README.md +++ b/headless/README.md @@ -328,3 +328,12 @@ transform. Unchanged nodes reuse their geometry result; removed nodes release cache entries. Viewport queries reuse the grid. `clear()` discards it. The cache still scans projected nodes and synchronizes rectangle membership; it is not a claim of constant-time full-frame updates or complete region-based drawing. + +`createJsonArrayImportSource` is a seekable parser building block for a UTF-8 JSON +array, with byte cursors, bounded record payloads, structural depth limits and +abort/yield checks. It uses parserVersion 3 and can feed a host-managed generic +ImportRunner. It does not yet interpret the exported document envelope, and +ProjectEngine's persisted imports/UI still accept their existing formats only. +A malformed structural tail ends with an explicit failure; previously emitted +records remain available. Empty source bytes reject, while `[]` is a valid empty +array. Array record errors are distinct from document schema/reference failures. diff --git a/src/lib/canvas-runtime/import-json-array.ts b/src/lib/canvas-runtime/import-json-array.ts new file mode 100644 index 0000000..174bdd3 --- /dev/null +++ b/src/lib/canvas-runtime/import-json-array.ts @@ -0,0 +1,37 @@ +import type {ImportSource,ImportLine} from './imports.ts'; +import {fault,validJson} from '../canvas-core/values.ts'; +/** Seekable JSON array reader. Cursors are byte offsets after '[' or a record comma. + * Structure errors end the source; previously published records remain explicit. */ +export function createJsonArrayImportSource(options:{bytes:Blob;fingerprint:string;maxRecordBytes?:number;chunkBytes?:number;yieldTask?:()=>Promise}):ImportSource{ + const bytes=options.bytes,max=options.maxRecordBytes??256*1024,chunk=options.chunkBytes??16384; + if(!(bytes instanceof Blob)||!options.fingerprint||!Number.isSafeInteger(max)||max<1||!Number.isSafeInteger(chunk)||chunk<1||chunk>max)throw fault('invalid_import_source'); + let cache={offset:-1,value:new Uint8Array()};const whitespace=(b:number)=>[9,10,13,32].includes(b),yieldTask=options.yieldTask??(()=>new Promise(resolve=>setTimeout(resolve,0))); + return Object.freeze({fingerprint:options.fingerprint,parserVersion:3,async read(start:number,signal:AbortSignal):Promise{ + if(!Number.isSafeInteger(start)||start<0||start>bytes.size)throw fault('invalid_import_cursor'); + if(start>0&&start===bytes.size){if(signal.aborted)throw fault('import_aborted');return null;} + if(bytes.size===0)throw fault('invalid_json_array'); + let cursor=start,work=0; + async function peek(){if(signal.aborted)throw fault('import_aborted');if(cursor>=bytes.size)return -1;if(cursor=cache.offset+cache.value.length){cache={offset:cursor,value:new Uint8Array(await bytes.slice(cursor,cursor+chunk).arrayBuffer())};if(signal.aborted)throw fault('import_aborted');}return cache.value[cursor-cache.offset];} + async function advance(){cursor++;if(++work>=16384){work=0;await yieldTask();if(signal.aborted)throw fault('import_aborted');}} + async function spaces(){while(whitespace(await peek()))await advance();} + const failure=(error:string):ImportLine=>({cursor:start,nextCursor:bytes.size,key:`byte:${start}`,error}); + await spaces();if(start===0){if(await peek()!==91)return failure('invalid_json_array');await advance();await spaces();} + if(await peek()===93){await advance();await spaces();if(cursor!==bytes.size)return failure('invalid_json_array');if(start!==0){/* Empty tail is valid only after a record, not a trailing comma. */return failure('invalid_json_array');}return null;} + if(await peek()===-1)return failure('incomplete_json_array'); + const recordStart=cursor,parts:number[]=[],stack:number[]=[];let quoted=false,escaped=false,oversized=false; + while(true){const byte=await peek();if(byte===-1)return failure('incomplete_json_array'); + if(!quoted&&stack.length===0&&(byte===44||byte===93))break; + if(cursor-recordStart256)return failure('json_nesting_limit');}else if(byte===125||byte===93){if(stack.pop()!==(byte===125?123:91))return failure('invalid_json_array');} + await advance(); + } + const delimiter=await peek();await advance(); + if(delimiter===93){await spaces();if(cursor!==bytes.size)return failure('invalid_json_array');} + else {await spaces();if(await peek()===93||await peek()===-1)return failure('invalid_json_array');} + const base={cursor:start,nextCursor:cursor,key:`byte:${start}`}; + if(oversized)return{...base,error:'record_too_large'}; + let text:string;try{text=new TextDecoder('utf-8',{fatal:true}).decode(new Uint8Array(parts));}catch{return{...base,error:'invalid_utf8'};} + try{const value=JSON.parse(text);return validJson(value)?{...base,value}:{...base,error:'invalid_json_value'};}catch{return{...base,error:'invalid_json'};} + }}); +} diff --git a/src/lib/canvas-runtime/index.ts b/src/lib/canvas-runtime/index.ts index 9f66732..c641d57 100644 --- a/src/lib/canvas-runtime/index.ts +++ b/src/lib/canvas-runtime/index.ts @@ -32,3 +32,4 @@ export type {CanvasElementCounts,ElementOwners} from './scope-counts.ts'; export {createAssetDepot} from './asset-depot.ts'; export type {AssetDepot,AssetAuthority,SharedAsset,SharedAssetBinding,AssetDepotPort,DepotCut,DepotCollection} from './asset-depot.ts'; +export {createJsonArrayImportSource} from './import-json-array.ts'; diff --git a/tests/headless/import-json-array.test.mjs b/tests/headless/import-json-array.test.mjs new file mode 100644 index 0000000..6b9c5c6 --- /dev/null +++ b/tests/headless/import-json-array.test.mjs @@ -0,0 +1,8 @@ +import {test} from 'node:test';import assert from 'node:assert/strict';import {createJsonArrayImportSource} from '../../src/lib/canvas-runtime/import-json-array.ts'; +const signal=()=>new AbortController().signal; +function source(text,options={}){return createJsonArrayImportSource({bytes:new Blob([text]),fingerprint:'immutable-source',...options});} +async function all(reader){let cursor=0;const rows=[];for(let i=0;i<100;i++){const row=await reader.read(cursor,signal());if(!row)return rows;assert.ok(row.nextCursor>cursor);rows.push(row);cursor=row.nextCursor;}throw Error('reader did not terminate');} +test('array reader handles UTF8, escaped strings and nested delimiters across one-byte chunks and resumes from the same byte cursor',async()=>{const text=' [ {"title":"图\\\"片,]","nested":[1,{"x":"]"}]},\n {"title":"second"} ] ',reader=source(text,{chunkBytes:1}),rows=await all(reader);assert.deepEqual(rows.map(r=>r.value),JSON.parse(text));assert.ok(rows.every(r=>!r.error));const reopened=source(text,{chunkBytes:3});assert.deepEqual(await reopened.read(rows[1].cursor,signal()),rows[1]);assert.equal(await reopened.read(rows[1].nextCursor,signal()),null);}); +test('structurally malformed tails report failure after the already accepted prefix',async()=>{for(const tail of [',]',',','','] garbage']){const rows=await all(source('[{"id":1},{"id":2}'+tail));assert.deepEqual(rows[0].value,{id:1});assert.ok(rows.at(-1).error,tail);}assert.deepEqual(await all(source('[]')),[]);await assert.rejects(all(source('')),/invalid_json_array/);assert.equal((await all(source('[1,]')))[0].error,'invalid_json_array');}); +test('oversized records skip bounded payload and allow a later valid record; bad UTF8 remains explicit',async()=>{const rows=await all(source(JSON.stringify(['x'.repeat(200),{ok:true}]),{maxRecordBytes:32,chunkBytes:8}));assert.equal(rows[0].error,'record_too_large');assert.deepEqual(rows[1].value,{ok:true});const reader=createJsonArrayImportSource({bytes:new Blob([new Uint8Array([91,34,255,34,93])]),fingerprint:'bad'});assert.equal((await all(reader))[0].error,'invalid_utf8');}); +test('long scans yield and honor abort without publishing a completed record',async()=>{const controller=new AbortController();let yields=0;const reader=source(JSON.stringify(['x'.repeat(40000)]),{yieldTask:async()=>{yields++;controller.abort();}});await assert.rejects(reader.read(0,controller.signal),/import_aborted/);assert.equal(yields,1);await assert.rejects(reader.read(-1,signal()),/invalid_import_cursor/);});