Bound chunked source hashing with workspace worker pools

This commit is contained in:
2026-11-08 20:44:29 +08:00
parent 4035956607
commit 9769f80cf5
15 changed files with 190 additions and 26 deletions
+4 -4
View File
@@ -68,20 +68,20 @@ test('late connection changes cancel preview without restoring stale path or unr
test('workspace retains a failed close for retry and stops edits before releasing its persistent owner', async () => {
const { createWorkspaceRuntime } = await import('../../src/lib/canvas-runtime/workspace.ts');
let image = null, leader, broken = false, closed = false, server = document, version = 1;
let image = null, leader, broken = false, closed = false, digestClosed=false, server = document, version = 1;
const receipts = new Map();
const storage = { async claim(scope) { leader = { scope, fence: 1, active: true, async release() { leader.active = false; } }; return leader; }, async recover() { return image; }, async commit(_leader, batch) { if (broken) throw new Error('quota'); image = structuredClone(batch.image); return { durableRootRevision: image.rootRevision }; }, async readFile() {}, close() { closed = true; } };
const persistence = { compile: (intent, requestId, snapshot) => ({ canvasIds: intent.canvasIds, expectedVersions: Object.fromEntries(intent.canvasIds.map(id => [id, snapshot.runtime.versions.get(id)])), payload: { intent, requestId } }),
async submit(request) { const { applyChanges } = await import('../../src/lib/canvas-core/document.ts'); server = applyChanges(server, JSON.parse(request.body).intent.changes); version++; receipts.set(request.id, { document: server, versions: new Map([['root', version], ['child', 1]]) }); return { outcome: 'applied' }; }, async query() { return { outcome: 'unresolved' }; }, canonical: request => receipts.get(request.id) };
const workspace = createWorkspaceRuntime({ accountId: 'account', workspaceId: 'workspace', authEpoch: 1, registry, storage, isCurrentIdentity: () => true, adapters: () => ({ persistence }) });
const workspace = createWorkspaceRuntime({ accountId: 'account', workspaceId: 'workspace', authEpoch: 1, registry, storage, isCurrentIdentity: () => true,digests:{digest:async()=> 'a'.repeat(64),close(){digestClosed=true;}}, adapters: () => ({ persistence }) });
const [a, b] = await Promise.all([workspace.open(document, new Map([['root', 1], ['child', 1]])), workspace.open(document, new Map([['root', 1], ['child', 1]]))]);
assert.equal(a, b); const detach = a.sessions.attach('first'); a.sessions.attach('second'); detach();
a.editor.dispatch({ type: 'element.data', target: { id: 'outside', incarnation: 1 }, patch: { title: 'survives view detach' } }); await a.settle();
broken = true; a.editor.dispatch({ type: 'element.data', target: { id: 'outside', incarnation: 1 }, patch: { title: 'not durable yet' } });
await assert.rejects(() => workspace.close(), /workspace_close_failed/); assert.equal(leader.active, true); assert.equal(closed, false);
await assert.rejects(() => workspace.close(), /workspace_close_failed/); assert.equal(leader.active, true); assert.equal(closed, false);assert.equal(digestClosed,false);
assert.equal(a.editor.dispatch({ type: 'element.data', target: { id: 'outside', incarnation: 1 }, patch: { title: 'late' } }).status, 'rejected');
assert.equal(workspace.getProjects().size, 1);
broken = false; await workspace.close(); assert.equal(leader.active, false); assert.equal(closed, true);
broken = false; await workspace.close(); assert.equal(leader.active, false); assert.equal(closed, true);assert.equal(digestClosed,true);
});
test('future curve formats remain readable and opaque while appearance edits preserve their path', () => {
+13
View File
@@ -0,0 +1,13 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {createBrowserDigests} from '../../src/lib/canvas-runtime/browser-digests.ts';
test('browser digest adapter rejects mismatched worker identity and terminates the worker',async()=>{
const previous=globalThis.Worker;let terminated=0;
globalThis.Worker=class{postMessage(value){queueMicrotask(()=>this.onmessage({data:{version:1,id:value.id+1,byteLength:value.bytes.size,sha256:'a'.repeat(64)}}));}terminate(){terminated++;}};
const pool=createBrowserDigests();try{await assert.rejects(pool.digest(new Blob(['bytes'])),/invalid_digest_response/);assert.equal(terminated,1);}finally{pool.close();if(previous===undefined)delete globalThis.Worker;else globalThis.Worker=previous;}
});
test('browser digest adapter exposes worker startup and runtime failures without returning a false hash',async()=>{
const previous=globalThis.Worker;let pool;
try{globalThis.Worker=class{constructor(){throw Error('denied');}};pool=createBrowserDigests();await assert.rejects(pool.digest(new Blob()),/digest_worker_unavailable/);pool.close();let terminated=0;globalThis.Worker=class{postMessage(){queueMicrotask(()=>this.onerror({preventDefault(){}}));}terminate(){terminated++;}};pool=createBrowserDigests();await assert.rejects(pool.digest(new Blob()),/digest_worker_failed/);assert.equal(terminated,1);}finally{pool?.close();if(previous===undefined)delete globalThis.Worker;else globalThis.Worker=previous;}
});
+13
View File
@@ -1,3 +1,4 @@
import {createBrowserDigests} from '../../src/lib/canvas-runtime/browser-digests.ts';
import {sealRecoveryImage} from '../../src/lib/canvas-runtime/recovery-encoding.ts';
import {planDuplicateAsync} from '../../src/lib/canvas-runtime/duplicate.ts';
import {replayHistoryEvidence,sameHistory} from '../../src/lib/canvas-runtime/history-replay.ts';
@@ -8,6 +9,16 @@ import { createFoundryRuns } from '../../src/lib/canvas-service/runs.ts';
import { createFoundryAssets } from '../../src/lib/canvas-service/assets.ts';
import { createFoundryPersistence } from '../../src/lib/canvas-service/foundry.ts';
async function digestWorkerCase(assert){
const NativeWorker=globalThis.Worker;let active=0,peak=0,created=0;const replies=[];
globalThis.Worker=class extends NativeWorker{constructor(...args){super(...args);created++;active++;peak=Math.max(peak,active);this.live=true;this.addEventListener('message',event=>replies.push(event.data));}terminate(){if(this.live){this.live=false;active--;}super.terminate();}};
const pool=createBrowserDigests();try{
const bytes=new Blob([new Uint8Array(4*1024*1024).fill(97)]),expected=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(value=>value.toString(16).padStart(2,'0')).join(''),cancel=new AbortController(),a=pool.digest(bytes),b=pool.digest(bytes),queued=pool.digest(bytes,cancel.signal),cancelled=queued.then(()=>{throw Error('queued digest ran');},error=>assert(error.code==='import_aborted','queued digest cancellation changed'));
cancel.abort();await cancelled;const results=await Promise.all([a,b]);assert(results.every(value=>value===expected),'worker digest differs from native SHA256');assert(peak===2&&created===2&&active===0,'worker pool exceeded bounds or leaked a worker');assert(replies.length===2&&replies.every(reply=>reply.version===1&&reply.byteLength===bytes.size),'worker protocol did not actually run');
const late=pool.digest(bytes),closed=late.then(()=>{throw Error('closed worker published');},error=>assert(error.code==='import_aborted','pool close did not cancel work'));await Promise.resolve();pool.close();await closed;assert(active===0,'pool close retained a worker');
}finally{pool.close();globalThis.Worker=NativeWorker;}
}
async function ownedImageStorageCase(name,assert){
const storage=createBrowserStorage({name:`${name}-owned-image`}),scope={accountId:'owner',workspaceId:'workspace',projectId:'project'};let leader;
try{
@@ -597,6 +608,8 @@ export async function run() {
results.push('project engine reopens real IndexedDB with one history unit and the original request; ACK resolves without duplicate edit');
await resourceRecoveryCase(name, assert);
results.push('staged file and image recover from real IndexedDB; lost upload ACK queries stable resource identity');
await digestWorkerCase(assert);
results.push('bounded real module workers hash chunked Blobs with native SHA256 parity, queued abort and close terminate work without a late result');
await ownedImageStorageCase(name,assert);
results.push('owned immutable recovery images survive IndexedDB checksums; mutable images and file descriptors are captured before asynchronous work');
await cooperativeCopyCase(assert);
+13
View File
@@ -0,0 +1,13 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {createDigestPool} from '../../src/lib/canvas-runtime/digest-pool.ts';
const hash='a'.repeat(64),tick=()=>new Promise(resolve=>setTimeout(resolve,0));
test('digest pool bounds execution and FIFO queue, rejects overload and drops queued cancellation',async()=>{
const started=[],release=[];const pool=createDigestPool({concurrency:2,maxQueued:2,execute:async bytes=>{started.push(await bytes.text());return new Promise(resolve=>release.push(()=>resolve(hash)));}}),cancel=new AbortController(),a=pool.digest(new Blob(['A'])),b=pool.digest(new Blob(['B'])),c=pool.digest(new Blob(['C']),cancel.signal),d=pool.digest(new Blob(['D']));const rejected=assert.rejects(c,/import_aborted/);await assert.rejects(pool.digest(new Blob(['overflow'])),/digest_capacity/);cancel.abort();await rejected;await tick();assert.deepEqual(started,['A','B']);release[0]();await a;await tick();assert.deepEqual(started,['A','B','D']);release[1]();release[2]();await Promise.all([b,d]);pool.close();await assert.rejects(pool.digest(new Blob()),/digest_pool_closed/);
});
test('active cancellation cannot free a compute slot before its executor stops; late output is ignored',async()=>{
let release;const started=[],controller=new AbortController(),pool=createDigestPool({concurrency:1,execute:async bytes=>{started.push(await bytes.text());if(started.length===1)await new Promise(resolve=>release=resolve);return hash;}}),first=pool.digest(new Blob(['first']),controller.signal),rejected=assert.rejects(first,/import_aborted/),second=pool.digest(new Blob(['second']));await tick();controller.abort();await rejected;await tick();assert.deepEqual(started,['first']);release();assert.equal(await second,hash);pool.close();
});
test('closing digest pool aborts active/queued tasks once and malformed executor output is rejected',async()=>{
const signals=[],pool=createDigestPool({concurrency:1,execute:(_bytes,signal)=>{signals.push(signal);return new Promise((_,reject)=>signal.addEventListener('abort',()=>reject(Error('stopped')),{once:true}));}}),a=pool.digest(new Blob()),b=pool.digest(new Blob()),failures=[assert.rejects(a,/import_aborted/),assert.rejects(b,/import_aborted/)];await tick();pool.close();pool.close();await Promise.all(failures);assert.equal(signals.length,1);assert.equal(signals[0].aborted,true);const invalid=createDigestPool({execute:async()=> 'bad'});await assert.rejects(invalid.digest(new Blob()),/invalid_digest_response/);invalid.close();
});
+9 -3
View File
@@ -187,7 +187,7 @@ test('a changed unparsed tail cannot pass native import recovery or alter its pr
test('late source digest cannot revive a cancelled job or publish into a held recovery root',async()=>{
for(const cancel of [true,false]){
const f=fixture(),engine=await f.open(),entered=deferred(),gate=deferred(),bytes=new class extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}([JSON.stringify(node('one'))]);
const f=fixture(),engine=await f.open(),entered=deferred(),gate=deferred(),bytes=new class extends Blob{slice(...args){return new class extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}([super.slice(...args)]);}}([JSON.stringify(node('one'))]);
try{const id=engine.imports.startNdjson(bytes,'canvas');await entered.promise;if(cancel){await engine.imports.cancel(id);gate.resolve();await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'cancelled');assert.equal(engine.getSnapshot().document.elements.size,0);assert.equal(engine.imports.getJobs().get(id).source.sha256,undefined);}else{const preparing=engine.prepareRecovery();gate.resolve();const session=await preparing,revision=engine.getSnapshot().rootRevision;await new Promise(resolve=>setTimeout(resolve,10));assert.equal(engine.getSnapshot().rootRevision,revision);assert.equal((await session.apply()).status,'applied');await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(f.server().elements.size,1);}assert.equal(f.errors.length,0);}finally{gate.resolve();await engine.close();}
}
});
@@ -219,7 +219,7 @@ test('late planned-source decoding respects cancellation and canonical recovery
const f=fixture();let engine=await f.open();const entered=deferred(),gate=deferred();try{
await seedCopy(engine);const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>snapshot.runtime.imports.get(id).committedSourceKeys.length===1);await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id),size=engine.getSnapshot().document.elements.size;await engine.close();
class DelayedChunk extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}
class DelayedSource extends Blob{slice(...args){return new DelayedChunk([super.slice(...args)]);}}
class DelayedSource extends Blob{slice(start,end,...rest){const part=super.slice(start,end,...rest);return end-start===16*1024?new DelayedChunk([part]):part;}}
f.files.set(before.source.fileId,new DelayedSource([f.files.get(before.source.fileId)]));engine=await f.open();const resuming=engine.imports.resume(id);await entered.promise;
if(cancel){await engine.imports.cancel(id);gate.resolve();await resuming;await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'cancelled');assert.equal(engine.getSnapshot().document.elements.size,size);}else{const preparing=engine.prepareRecovery();gate.resolve();await resuming;const session=await preparing,revision=engine.getSnapshot().rootRevision;await new Promise(resolve=>setTimeout(resolve,10));assert.equal(engine.getSnapshot().rootRevision,revision);assert.equal(engine.getSnapshot().document.elements.size,size);assert.equal((await session.apply()).status,'applied');await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');}assert.equal(f.errors.length,0);
}finally{gate.resolve();await engine.close();}
@@ -231,7 +231,13 @@ test('indexed copy displays a prefix while a later raw payload read is still blo
const id=await engine.imports.duplicate([{id:'F',incarnation:1}]);await until(engine,snapshot=>!!snapshot.runtime.imports.get(id).source.sha256);await engine.imports.pause(id);await engine.settle();const before=engine.imports.getJobs().get(id);await engine.close();const bytes=f.files.get(before.source.fileId),rows=(await bytes.text()).trimEnd().split('\n');let payload=0;
for(const text of rows){const row=JSON.parse(text);payload+=new TextEncoder().encode(text+'\n').byteLength;if(row.kind==='payload')break;}
class DelayedChunk extends Blob{async arrayBuffer(){entered.resolve();await gate.promise;return super.arrayBuffer();}}
class DelayedPayload extends Blob{slice(start,end,...rest){const part=super.slice(start,end,...rest);return start>=payload+64*1024?new DelayedChunk([part]):part;}}
class DelayedPayload extends Blob{slice(start,end,...rest){const part=super.slice(start,end,...rest);return start>=payload+64*1024&&(start-payload)%(64*1024)===0?new DelayedChunk([part]):part;}}
f.files.set(before.source.fileId,new DelayedPayload([bytes]));engine=await f.open();await engine.imports.resume(id);await entered.promise;const cut=engine.getSnapshot(),job=cut.runtime.imports.get(id);assert.equal(job.source.snapshotVersion,2);assert.equal(job.status,'running');assert.ok(job.committedSourceKeys.length>0&&job.committedSourceKeys.length<102);assert.ok(cut.document.elements.size>source.elements.size&&cut.document.elements.size<202,'raw tail read gated the entire copy');gate.resolve();await engine.settle();assert.equal(engine.imports.getJobs().get(id).status,'completed');assert.equal(engine.getSnapshot().document.elements.size,202);assert.equal(f.errors.length,0);
}finally{gate.resolve();await engine.close();}
});
test('closing a project aborts its injected digest without closing another project sharing the pool',async()=>{
const {createDigestPool}=await import('../../src/lib/canvas-runtime/digest-pool.ts'),entered=deferred();let first=true;const pool=createDigestPool({concurrency:1,execute:async(bytes,signal)=>{if(first){first=false;entered.resolve();await new Promise((_,reject)=>signal.addEventListener('abort',()=>reject(Error('stopped')),{once:true}));}return importSourceDigest(bytes,signal);}}),a=fixture(),b=fixture();let ea=await a.open({digests:pool}),eb=await b.open({digests:pool});
try{const aid=ea.imports.startNdjson(file([node('A')]),'canvas');await entered.promise;const bid=eb.imports.startNdjson(file([node('B')]),'canvas');await ea.close();await eb.settle();assert.equal(eb.imports.getJobs().get(bid).status,'completed');assert.equal(eb.getSnapshot().document.elements.size,1);assert.equal(a.errors.length,0);assert.equal(b.errors.length,0);ea=await a.open();await ea.settle();assert.equal(ea.imports.getJobs().get(aid).status,'completed');assert.equal(ea.getSnapshot().document.elements.size,1);}finally{await ea.close();await eb.close();pool.close();}
});
+6
View File
@@ -164,3 +164,9 @@ test('known successful upload coordination recovers stored identity and releases
assert.equal(f.uploads.length,1);assert.equal(f.writes.length,1);assert.equal(engine.getSnapshot().history,history);assert.equal(engine.getSnapshot().runtime.resources.get(resource.id).stored.id,resource.id);assert.equal(engine.getSnapshot().runtime.ledger.requests.get(request.id).body,request.body);
assert.equal(await f.files.get(resource.id).text(),'image bytes');assert.equal(decodeRecovery(f.image(),registry).runtime.resources.get(resource.id).stored.id,resource.id);await engine.close();
});
test('digest worker failure remains distinct from missing bytes and explicit retry resumes the original resource',async()=>{
const {importSourceDigest}=await import('../../src/lib/canvas-runtime/import-integrity.ts'),f=fixture(),errors=[];let calls=0;const engine=await f.open({digests:{async digest(bytes,signal){if(++calls===1)throw Object.assign(Error('worker failed'),{code:'digest_worker_failed'});return importSourceDigest(bytes,signal);}},onError:error=>errors.push(error)});
try{const ref=addImage(engine);await engine.settle();const resource=engine.getSnapshot().runtime.resources.get(ref.id);assert.deepEqual(engine.getSnapshot().runtime.ledger.operations.get(resource.operationId).blockers,['digest_worker_failed']);assert.equal(f.files.size,1);assert.equal(f.uploads.length,0);await new Promise(resolve=>setTimeout(resolve,10));assert.equal(calls,1,'failed worker retried in a busy loop');await engine.retry();await engine.settle();assert.equal(f.uploads.length,1);assert.equal(engine.getSnapshot().runtime.resources.get(ref.id).stored.id,ref.id);assert.equal(engine.getSnapshot().history.undo.length,1);assert.equal(errors.length,1);}finally{await engine.close();}
});
+20
View File
@@ -0,0 +1,20 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {createHash,randomBytes} from 'node:crypto';
import {createSha256} from '../../src/lib/canvas-runtime/sha256.ts';
import {importSourceDigest} from '../../src/lib/canvas-runtime/import-integrity.ts';
const expected=bytes=>createHash('sha256').update(bytes).digest('hex');
test('incremental SHA-256 matches known vectors, every padding boundary and varied chunk splits',()=>{
for(const bytes of [Buffer.alloc(0),Buffer.from('abc'),Buffer.from('abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq'),Buffer.alloc(1000000,97),...Array.from({length:260},(_,n)=>randomBytes(n))]){
for(const chunk of [1,7,63,64,65,8192]){const hash=createSha256();for(let offset=0;offset<bytes.length;offset+=chunk)hash.update(bytes.subarray(offset,offset+chunk));assert.equal(hash.hex(),expected(bytes),`${bytes.length}/${chunk}`);assert.equal(hash.hex(),expected(bytes));assert.throws(()=>hash.update(new Uint8Array()),/digest_finalized/);}
}
});
test('source digest reads bounded slices, yields and never materializes the entire Blob',async()=>{
const data=randomBytes(1024*1024+13),reads=[];let yields=0;
class SlicedBlob extends Blob{arrayBuffer(){throw Error('whole Blob read');}slice(start,end){reads.push(end-start);return super.slice(start,end);}}
assert.equal(await importSourceDigest(new SlicedBlob([data]),undefined,{chunkBytes:65536,yieldTask:async()=>{yields++;}}),expected(data));assert.equal(reads.length,17);assert.ok(reads.every(size=>size<=65536));assert.equal(yields,16);
});
test('source digest cancellation fences a late chunk and a scheduler yield',async()=>{
let release,entered;const gate=new Promise(resolve=>release=resolve),ready=new Promise(resolve=>entered=resolve),controller=new AbortController();class Chunk extends Blob{async arrayBuffer(){entered();await gate;return super.arrayBuffer();}}class Source extends Blob{slice(...args){return new Chunk([super.slice(...args)]);}}
const pending=importSourceDigest(new Source(['abc']),controller.signal);await ready;controller.abort();release();await assert.rejects(pending,/import_aborted/);const second=new AbortController();await assert.rejects(importSourceDigest(new Blob(['abcdef']),second.signal,{chunkBytes:2,yieldTask:async()=>second.abort()}),/import_aborted/);
});