Zayden-Jung/Endoleap
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Persist shared asset bytes and project pins with Root commits

Browse Source
This commit is contained in:
Codex Bot
2026-11-09 15:54:05 +08:00
parent 196ca4514e
commit 74c219f91d
14 changed files with 249 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/headless/asset-depot-browser-cases.mjs
+52
View File
@@ -0,0 +1,52 @@
import {openProjectEngine} from '../../src/lib/canvas-runtime/engine.ts';
import {createDocument,createRegistry,builtinComponents,applyChanges} from '../../src/lib/canvas-core/index.ts';
import {createBrowserStorage} from '../../src/lib/canvas-runtime/storage.ts';
export async function assetDepotStorageCase(assert){
const name=`depot-${crypto.randomUUID()}`,storage=createBrowserStorage({name}),scope={accountId:'a',workspaceId:'w',projectId:'p'},leaders=[];
const image=n=>({formatVersion:1,rootRevision:n,localRevision:n,payload:{n}}),bytes=new Blob(['shared original'],{type:'image/png'}),digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''),authority='https://source.test/w/assets';
const ref=id=>({version:1,authority,digest,asset:{id,url:`https://source.test/w/assets/${id}`,width:2,height:1,bytes:bytes.size,mime_type:'image/png'}}),cut=(pins=[],writes=[],complete=true)=>({version:1,authority,pins,writes,complete}),collect=(leader,now,extra={})=>storage.assetDepot.collect(leader,{authority,now,retentionMs:10,limit:128,...extra});
const rejects=async(fn,code)=>{try{await fn();}catch(e){assert(e.code===code,`expected ${code}, got ${e.code}`);return;}throw Error(`expected ${code}`);};
const open=async(projectId)=>{const leader=await storage.claim({...scope,projectId});leaders.push(leader);return leader;};
try{
const p=await open('p'),q=await open('q');
await storage.commit(p,{image:image(0),depot:cut([['one',['element:p']]], [{ref:ref('one'),bytes},{ref:ref('two'),bytes}])});
await storage.commit(q,{image:image(0),depot:cut([['two',['history:q']]])});
const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
const count=()=>new Promise((resolve,reject)=>{const tx=database.transaction('asset-bytes'),request=tx.objectStore('asset-bytes').count();request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
assert(await count()===1,'same digest was physically duplicated across identities');
await p.release();assert((await collect(q,0)).pinnedIds.length===2,'closed project pins were lost');
assert(await(await storage.assetDepot.read(q,authority,'one')).bytes.text()==='shared original','destination could not read shared bytes');
const otherAccount=await storage.claim({...scope,accountId:'other'});leaders.push(otherAccount);assert(await storage.assetDepot.read(otherAccount,authority,'one')===undefined,'account cache leak');
const otherWorkspace=await storage.claim({...scope,workspaceId:'other'});leaders.push(otherWorkspace);assert(await storage.assetDepot.read(otherWorkspace,authority,'one')===undefined,'workspace cache leak');
assert(await storage.assetDepot.read(q,'https://other.test','one')===undefined,'authority cache leak');
const stale=await open('stale');await storage.commit(stale,{image:image(0)});assert((await collect(q,100)).blockedProjects.includes('stale'),'unindexed project allowed deletion');
await storage.commit(stale,{image:image(0),depot:cut([],[],false)});assert(!(await collect(q,100)).complete,'opaque project allowed deletion');
await storage.commit(stale,{image:image(0),depot:cut()});await storage.commit(stale,{image:image(1)});assert(!(await collect(q,100)).complete,'stale manifest allowed deletion');
await storage.commit(stale,{image:image(1),depot:cut()});
const reopened=await open('p');await storage.commit(reopened,{image:image(1),depot:cut()});
assert((await collect(q,100)).markedIds.includes('one'),'unreferenced resource was not marked');
assert((await collect(q,110)).purgedIds.includes('one')&&await count()===1,'purging one identity deleted another identity bytes');
await storage.commit(q,{image:image(1),depot:cut()});await collect(q,110);await collect(q,120);assert(await count()===0,'expired unpinned shared bytes retained');
const purged=await storage.assetDepot.read(q,authority,'one');assert(purged.purgedAt===110&&!purged.bytes,'immutable identity lost or purged bytes exposed');
await rejects(()=>storage.commit(q,{image:image(2),depot:cut([], [{ref:{...ref('one'),digest:'0'.repeat(64)},bytes}])}),'shared_asset_bytes_changed');assert((await storage.recover(q)).rootRevision===1,'invalid bytes advanced Root');
await rejects(()=>storage.commit(q,{image:image(2),depot:cut([['one',['element:q']]], [{ref:{...ref('one'),asset:{...ref('one').asset,width:3}},bytes}])}),'shared_asset_identity_conflict');assert(await count()===0&&(await storage.recover(q)).rootRevision===1,'identity conflict partially committed bytes or Root');
// Force a later files-store conflict after the depot puts: the whole transaction must roll back.
await storage.commit(q,{image:image(2),files:[{id:'existing',bytes}],depot:cut()});
await rejects(()=>storage.commit(q,{image:image(3),files:[{id:'existing',bytes}],depot:cut([['one',['element:q']]], [{ref:ref('one'),bytes}])}),'resource_identity_conflict');assert(await count()===0&&(await storage.recover(q)).rootRevision===2,'late failure committed depot or Root');
await storage.commit(q,{image:image(3),depot:cut([['one',['element:q']]], [{ref:ref('one'),bytes}])});assert(await count()===1&&(await storage.assetDepot.read(q,authority,'one')).purgedAt===undefined,'authorized rehydration failed');
// A one-item sweep must provide a cursor and preserve each resource decision.
const first=await collect(q,130,{limit:1});assert(first.nextAfter==='one','bounded sweep cursor missing');assert((await collect(q,130,{after:first.nextAfter,limit:1})).nextAfter===undefined,'bounded sweep did not finish');
// Exercise the actual engine persistence entry point, rather than only hand-built batches.
const projectId='engine-project',registry=createRegistry(builtinComponents());let document=createDocument(projectId,[{id:'canvas',parent:null}],[]),version=1,counter=0,resource,uploads=0;
const receipts=new Map(),assets={authority:{scope:authority,async resolve(){throw Error('unexpected adoption');},async read(){throw Error('unexpected hydration');}},sourceURL:id=>`https://source.test/w/assets/${id}`,displayURL:id=>`/assets/${id}`,async upload(request,data){uploads++;const raw=JSON.parse(request.body);return{outcome:'applied',canonical:{id:raw.resource_id,url:assets.sourceURL(raw.resource_id),width:2,height:1,bytes:data.size,mime_type:data.type}};},async query(){return{outcome:'unresolved'};},accept:(_request,status)=>status.canonical};
const persistence={compile(intent,id,snapshot){return{payload:{intent},canvasIds:intent.canvasIds,expectedVersions:{canvas:snapshot.runtime.versions.get('canvas')}};},async submit(request){document=applyChanges(document,JSON.parse(request.body).intent.changes);receipts.set(request.id,{document,versions:new Map([['canvas',++version]])});return{outcome:'applied'};},async query(){return{outcome:'unresolved'};},canonical:request=>receipts.get(request.id)};
const engine=await openProjectEngine({scope:{...scope,projectId},registry,document,versions:new Map([['canvas',1]]),storage,assets,persistence,createId:()=>`engine-${++counter}`});
try{
const staged=engine.stageFiles([{name:'source.png',bytes}],([handle],tx)=>{resource=handle.id;tx.dispatch({type:'element.create',element:{id:'picture',incarnation:1,structure:'node',componentId:'image',dataVersion:1,ownerCanvasId:'canvas',geometry:{x:0,y:0,width:2,height:1,rotation:0},data:{title:'picture',text:'',assetId:handle.id,src:handle.src}}});});assert(staged.status!=='rejected','engine image staging failed');
let ready=false;for(let i=0;i<200;i++){if(engine.getSnapshot().runtime.resources.get(resource)?.stored){ready=true;break;}await new Promise(resolve=>setTimeout(resolve,5));}assert(ready,'engine upload did not settle');
}finally{await engine.close();}
const promoted=await storage.assetDepot.read(q,authority,resource);assert(promoted?.bytes&&await promoted.bytes.text()==='shared original'&&uploads===1,'engine did not promote acknowledged upload bytes');
assert((await collect(q,200)).pinnedIds.includes(resource),'closed engine did not retain durable document/history pins');
await q.release();await rejects(()=>collect(q,200),'storage_leader_lost');database.close();
}finally{for(const leader of leaders)await leader.release();storage.close();indexedDB.deleteDatabase(name);}
}
tests/headless/asset-depot.test.mjs
+41
View File
@@ -0,0 +1,41 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {createAssetDepot} from '../../src/lib/canvas-runtime/asset-depot.ts';
import {createFoundryTransport} from '../../src/lib/canvas-service/foundry.ts';
import {createFoundryAssets} from '../../src/lib/canvas-service/assets.ts';
const digest=async bytes=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join('');
async function fixture(){
const bytes=new Blob(['original'],{type:'image/png'}),scope={accountId:'account',workspaceId:'workspace'},leader={scope:{...scope,projectId:'p'},active:true},ref={version:1,authority:'authority',digest:await digest(bytes),asset:{id:'asset',url:'https://source.test/asset',width:2,height:1,bytes:bytes.size,mime_type:'image/png'}};
let cached,allowed=true,resolves=0,reads=0,current=true;
const depot=createAssetDepot({scope,port:{async read(){return cached;},async collect(){return{complete:true};}},current:()=>current,digest});
const authority={scope:ref.authority,async resolve(){resolves++;if(!allowed)throw Error('denied');return ref;},async read(){reads++;return bytes;}};
return{depot,scope,leader,ref,bytes,authority,setCached:v=>cached=v,deny:()=>allowed=false,retire:()=>current=false,counts:()=>({resolves,reads})};
}
test('new cross-project adoption reauthorizes even when immutable bytes are warm',async()=>{
const f=await fixture(),signal=new AbortController().signal;
assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).bytes,f.bytes);
f.setCached({ref:f.ref,bytes:f.bytes});assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).bytes,f.bytes);assert.deepEqual(f.counts(),{resolves:2,reads:1});
f.deny();await assert.rejects(f.depot.authorize(f.leader,f.authority,'asset',signal),/denied/);assert.deepEqual(f.counts(),{resolves:3,reads:1});f.depot.close();
});
test('depot captures scope, validates cached identity and bytes, and rejects revoked lifecycle',async()=>{
const f=await fixture(),signal=new AbortController().signal;f.scope.accountId='mutated';assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).ref.asset.id,'asset');
f.setCached({ref:{...f.ref,digest:'0'.repeat(64)},bytes:f.bytes});await assert.rejects(f.depot.authorize(f.leader,f.authority,'asset',signal),{code:'shared_asset_identity_conflict'});
f.setCached({ref:f.ref,bytes:new Blob(['tampered'])});await assert.rejects(f.depot.read(f.leader,f.ref),{code:'shared_asset_bytes_changed'});
f.retire();await assert.rejects(f.depot.read(f.leader,f.ref),{code:'identity_changed'});f.depot.close();
});
test('late authority result after close never enters the metadata cache',async()=>{
const f=await fixture();let done,started;const ready=new Promise(resolve=>started=resolve),wait=new Promise(resolve=>done=resolve);
const promise=f.depot.authorize(f.leader,{...f.authority,async resolve(){started();await wait;return f.ref;}},'asset',new AbortController().signal);await ready;f.depot.close();done();await assert.rejects(promise,{code:'identity_changed'});assert.equal(f.depot.getResources().size,0);
});
test('binary asset transport preserves original bytes and enforces identity and actual byte budgets',async()=>{
const accountId='11111111-1111-4111-8111-111111111111';let current=true,cancelled=false;
const transport=response=>createFoundryTransport({accountId,isCurrentIdentity:()=>current,fetch:async()=>response});
const result=await transport(new Response('original',{headers:{'content-type':'image/png'}}))({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'});assert.equal(await result.bytes.text(),'original');assert.equal(result.bytes.type,'image/png');
const oversized=new Response(new ReadableStream({start(c){c.enqueue(new Uint8Array(20*1024*1024+1));},cancel(){cancelled=true;}}));await assert.rejects(transport(oversized)({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'}),{code:'asset_byte_budget_exceeded'});assert.equal(cancelled,true);
const identityResponse=new Response(new ReadableStream({pull(c){current=false;c.enqueue(new Uint8Array([1]));}}));await assert.rejects(transport(identityResponse)({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'}),{code:'identity_changed'});
});
test('Foundry resource authority validates version, resource identity and canonical URL',async()=>{
const id='22222222-2222-4222-8222-222222222222',workspaceId='33333333-3333-4333-8333-333333333333';let version=1;const bytes=new Blob(['original']),sha256=await digest(bytes),calls=[];
const assets=createFoundryAssets({workspaceId,transport:async request=>{calls.push(request);return request.responseFormat==='asset-bytes-v1'?{status:200,body:null,bytes}:{status:200,body:{metadata_version:version,sha256,asset:{id,url:assets.sourceURL(id),width:2,height:1,bytes:bytes.size,mime_type:'image/png'}}};}});
const ref=await assets.authority.resolve(id,new AbortController().signal);assert.equal(ref.digest,sha256);assert.equal(await assets.authority.read(id,new AbortController().signal),bytes);assert.equal(calls[0].path.includes('/metadata/'),true);assert.equal(calls[1].responseFormat,'asset-bytes-v1');version=2;await assert.rejects(assets.authority.resolve(id,new AbortController().signal),{code:'asset_metadata_interpreter_unavailable'});
});
tests/headless/browser-storage-cases.mjs
+3 -1
View File
@@ -1,3 +1,4 @@
import {assetDepotStorageCase} from './asset-depot-browser-cases.mjs';
import {createBrowserImageDecoder} from '../../src/lib/canvas-runtime/browser-images.ts';
import {decodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts';
import {createBrowserDigests} from '../../src/lib/canvas-runtime/browser-digests.ts';
@@ -711,7 +712,7 @@ async function loadingRecoveryCase(name,assert,coverage=false,scoped=false,struc
assert(injected&&failure==='loading_full_cut_abort','completion did not reach forced checkpoint rollback');const partial=await adapter.recover(ownerForTest);assert(partial.rootRevision===before&&partial.formatVersion===format&&calls.length===0,'aborted full checkpoint released loading writes or lost the partial cut');
await engine.persist();await engine.settle();
assert(!engine.loading.getState()&&server.elements.get('A').data.text==='local before tail'&&engine.getSnapshot().document.elements.size===2,'full tail overwrote the local prefix');await engine.close();engine=await open();await engine.settle();assert(!engine.loading.getState()&&engine.getSnapshot().document.elements.size===2,'completed recovery reopened as partial');
const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(`${name}-loading`,2);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(`${name}-loading`);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
try{const transaction=database.transaction(['checkpoints','journal'],'readonly'),read=store=>new Promise((resolve,reject)=>{const request=transaction.objectStore(store).getAll();request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const cuts=(await Promise.all([read('checkpoints'),read('journal')])).flat();assert(cuts.every(cut=>(cut.image?.formatVersion??cut.header?.formatVersion)<7),'completed storage still requires a partial journal base');}finally{database.close();}
}finally{await engine?.close();storage.close();indexedDB.deleteDatabase(`${name}-loading`);}
}
@@ -737,6 +738,7 @@ export async function run() {
const rejects = async (callback, message) => { try { await callback(); } catch (error) { assert(error.message === message, `expected ${message}, received ${error.message}`); return; } throw new Error(`expected ${message}`); };
let leader, other;
try {
await assetDepotStorageCase(assert);results.push('workspace depot atomically shares digest bytes, preserves closed-project pins, blocks unknown/stale coverage and rolls back failed Root cuts');
leader = await storage.claim(scope);
assert(await storage.recover(leader) === null, 'empty scope must be distinguishable from a corrupt checkpoint');
await rejects(() => storage.claim(scope), 'storage_scope_in_use');
tests/headless/proxy.integration.mjs
+2
View File
@@ -47,6 +47,8 @@ try{
assert.ok(request);assert.equal(request.body.toString(),body);assert.equal(request.headers['idempotency-key'],key);assert.equal(request.headers['x-expected-account'],account);assert.equal(request.headers.authorization,'Bearer fixture-session');
const receiptPath=`workspaces/10000000-0000-4000-8000-000000000003/assets/operations/${key}`;
const assetReceipt=await fetch(`http://127.0.0.1:${port}/api/foundry/${receiptPath}`);assert.equal(assetReceipt.status,200);assert.deepEqual(await assetReceipt.json(),{id:'fixture'});
const metadata=await fetch(`http://127.0.0.1:${port}/api/foundry/${receiptPath.replace('/operations/','/metadata/')}`,{headers:{'X-Expected-Account':account,Cookie:'foundry_session=fixture-session'}});assert.equal(metadata.status,200);assert.deepEqual(await metadata.json(),{id:'fixture'});assert.match(metadata.headers.get('cache-control'),/no-store/);const metadataRequest=received.find(item=>item.path.includes('/assets/metadata/'));assert.equal(metadataRequest.headers['x-expected-account'],account);assert.equal(metadataRequest.headers.authorization,'Bearer fixture-session');
console.log('PASS asset metadata JSON uses the actual authenticated Next proxy without image-body validation or caching');
const multipart=new FormData();multipart.append('image',new Blob(['immutable file'],{type:'image/png'}),'fixture.png');
const assetPath=receiptPath.split('/operations/')[0],resourceId='10000000-0000-4000-8000-000000000006';
const uploaded=await fetch(`http://127.0.0.1:${port}/api/foundry/${assetPath}`,{method:'POST',body:multipart,headers:{'X-Resource-Id':resourceId,'Idempotency-Key':key,'X-Expected-Account':account}});await uploaded.text();assert.equal(uploaded.status,200);