From 902e30a65b07a97304ffc41fbbc7a9d1a91d08fd Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 16:31:31 +0800 Subject: [PATCH] Deduplicate stored upload files and retain adoption bytes until their Root is durable --- README.md | 4 +- headless/README.md | 39 ++++++++++++++ src/lib/canvas-runtime/asset-depot-storage.ts | 40 ++++++++++++++- src/lib/canvas-runtime/depot-cut.ts | 4 +- src/lib/canvas-runtime/engine.ts | 10 ++-- src/lib/canvas-runtime/storage.ts | 20 ++++---- tests/headless/asset-adoption.test.mjs | 12 +++++ tests/headless/asset-depot-browser-cases.mjs | 1 + tests/headless/browser-storage-cases.mjs | 4 ++ tests/headless/depot-files-browser-cases.mjs | 51 +++++++++++++++++++ 10 files changed, 167 insertions(+), 18 deletions(-) create mode 100644 tests/headless/depot-files-browser-cases.mjs diff --git a/README.md b/README.md index 3588247..916ca84 100644 --- a/README.md +++ b/README.md @@ -116,7 +116,9 @@ CANVAS_UI_TOOLS=/path/to/isolated-tools node tests/canvas-ui-check.mjs DOM 工具仅需在临时目录安装 esbuild/jsdom，见测试文件开头；不启动浏览器。生产构建与 `tests/api-smoke.mjs` 使用隔离副本和临时本地服务，避免影响现有开发服务器。实际结果及限制见 VALIDATION.md。 -Headless Canvas API 的拆包方案保留在本地忽略的 `docs/HEADLESS_CANVAS_API_DRAFT.md`，已按同步本地编辑与独立异步 runtime 重写，包含 element/component/feature、API 选择、八个工具栏位置、mode 作用域、上传恢复与 ACK 合并合同。当前没有创建或发布该库；outbox/IndexedDB 等仍是待实施方案。其他已被替代的元素/分区草案已清理。 +Headless Canvas API 的完整方案保留在本地忽略的 `docs/HEADLESS_CANVAS_API_DRAFT.md`。当前工作分支已接入纯 core/runtime、ledger/queue、IndexedDB 恢复、渐进导入/深复制、canonical Foundry 协议和私有 ESM 包预览；使用方式见 `headless/README.md`，尚未发布 npm registry，也不代表完整产品验收或后端部署完成。 + +Workspace AssetDepot 支持重新授权后的跨项目采用、持久绑定与共享字节。浏览器数据库 v4 将已确认上传的项目文件转换为共享字节指针，保留原 File MIME 和逻辑读取接口；绑定使用恢复格式13。共享回收要求所有项目的持久引用清单完整且与 Root 同步；关闭项目仍受保护。源文件转换、字节、引用清单和 Root 在同一个事务中保存，未确认上传继续保留原 Blob。 应用提示使用 Sonner，图标说明使用 Radix Tooltip；根布局统一安装 provider。画布通用操作错误与归属完成使用固定 ID 通知，保存冲突、失败上传和重试操作仍保留在相关内容处。消息和浮层依赖仅属于应用 UI，不作为未来 headless core 的依赖。 diff --git a/headless/README.md b/headless/README.md index 4577f43..1203019 100644 --- a/headless/README.md +++ b/headless/README.md @@ -209,3 +209,42 @@ NodeNext type check excludes DOM libraries. A separate peer consumer renders a registered component, verifies its command context, and checks typed NodeProps and EdgeProps with the declared peers. Browser tests still execute the compiled ESM storage and Worker artifacts independently of Next.js/TypeScript stripping. + + +Workspace asset depot and file storage: + +`createWorkspaceRuntime` owns one depot when `StoragePort.assetDepot` is provided. +`await engine.adoptResource(id, (resource, tx) => { ... }, {signal})` obtains fresh +server permission, verifies the original bytes, and runs a synchronous editor +transaction. The callback uses the existing `resource.id` / canonical `resource.src` +in a newly allocated element; it also receives server dimensions in `resource.asset`. +The project stores a `SharedAssetBinding`, not a second upload. Binding, document, +history, ledger, shared bytes and pins cross the same durable barrier. The returned +local `Result` does not imply durability or backend completion. Async or failed +callbacks leave no partial binding. The Foundry authority requires the matching +permission-checked asset metadata endpoint; cached metadata is not an authorization. + +Bindings use recovery envelope 13 (including partial loading v6); readers retain +formats 1–12 and reject unknown future formats. A reopened binding can display +shared local bytes, and undo/redo reuse the asset without uploading again. The +workspace closes projects before its depot; retiring the identity fences late work. + +The browser adapter uses IndexedDB version 4. After a proven stored-upload cut, a +project's `files` row holds an opaque pointer to digest-addressed `asset-bytes`. +Unconfirmed uploads and import source files keep their original Blob rows. Use +`readFile` / `scanFiles`; direct assumptions that every raw files-store row is a +Blob are invalid. Logical reads preserve the original local MIME, even if the +server detected another MIME. Existing v3 databases upgrade without dropping +checkpoints or original files and promote on a subsequent matching Root commit +(or an idempotent reindex). Older DB readers reject v4. Blob URLs and already +returned Blob objects live independently of a later persistent eviction; actual +disk reclamation remains the browser's responsibility. + +Resource identities retain immutable metadata after byte eviction. Complete, +revision-matched durable manifests from every project in the workspace protect +closed editors; missing/stale/opaque coverage prevents shared deletion. Distinct +resource IDs may own the same digest; the shared byte row disappears only when +its last live identity is retired. An older Root cannot release temporary bytes +that belong to a newer adoption revision. Collection is explicit and local; this +preview does not enable background deletion on Foundry or claim the full retention, +permission-race and product acceptance matrix is finished. diff --git a/src/lib/canvas-runtime/asset-depot-storage.ts b/src/lib/canvas-runtime/asset-depot-storage.ts index e17c00d..e3d40e8 100644 --- a/src/lib/canvas-runtime/asset-depot-storage.ts +++ b/src/lib/canvas-runtime/asset-depot-storage.ts @@ -1,6 +1,6 @@ import {equal,fault,freeze} from '../canvas-core/values.ts'; import {sharedAssetRef,validateSharedAsset,type AssetDepotPort,type DepotCut,type DepotEntry,type SharedAsset} from './asset-depot.ts'; -import type {StorageLeader,StorageScope} from './storage-port.ts'; +import type {StorageLeader,StorageScope,RecoveryImage} from './storage-port.ts'; export const depotStores=['asset-depot','asset-bytes','asset-pins']; type Fact={ref:SharedAsset;unreferencedAt?:number;purgedAt?:number}; @@ -30,6 +30,44 @@ export async function writeDepotCut(tx:IDBTransaction,scope:StorageScope,rootRev } tx.objectStore('asset-pins').put({version:1,rootRevision,complete:cut.complete,pins:cut.pins} satisfies Pins,[...base,scope.projectId]); } +/** DB v4 file rows may refer to immutable shared bytes after a proven upload ACK. */ +export type DepotFile={readonly fileVersion:1;readonly kind:'asset-depot';readonly authority:string;readonly resourceId:string;readonly digest:string;readonly byteLength:number;readonly mimeType:string}; +function depotFile(row:unknown,id:string):DepotFile{ + const file=row as DepotFile; + if(!file||file.fileVersion!==1)throw fault('file_interpreter_unavailable'); + authority(file.authority); + if(file.kind!=='asset-depot'||file.resourceId!==id||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest)||!Number.isSafeInteger(file.byteLength)||file.byteLength<1||file.byteLength>20*1024*1024||typeof file.mimeType!=='string')throw fault('corrupt_file_store'); + return file; +} +/** Undefined means bytes were explicitly evicted; identity metadata remains. */ +export async function resolveDepotFile(tx:IDBTransaction,scope:StorageScope,id:string,row:unknown):Promise{ + if(row===undefined||row instanceof Blob)return row; + const file=depotFile(row,id),base=prefix(scope,file.authority),fact=await value(tx.objectStore('asset-depot').get([...base,id])); + if(!fact)throw fault('corrupt_file_store');validateFact(fact,file.authority,id); + if(fact.ref.digest!==file.digest||fact.ref.asset.bytes!==file.byteLength)throw fault('corrupt_file_store'); + if(fact.purgedAt!==undefined)return; + const bytes=await value(tx.objectStore('asset-bytes').get([...base,file.digest])); + if(bytes===undefined)return; + if(!(bytes instanceof Blob)||bytes.size!==file.byteLength)throw fault('corrupt_file_store'); + // Preserve the original local File MIME even if the server detected another MIME. + return bytes.type===file.mimeType?bytes:bytes.slice(0,bytes.size,file.mimeType); +} +/** Convert only Root-proven stored uploads, after shared bytes and local files are written. */ +export async function promoteDepotFiles(tx:IDBTransaction,scope:StorageScope,key:string,image:RecoveryImage,cut:DepotCut){ + type Resource={id:string;digest?:string;byteLength:number;mimeType:string;stored?:SharedAsset['asset'];retirement?:{purgedAt?:number}}; + const payload=image.payload as {runtime?:{resources?:readonly (readonly [string,Resource])[]}},entries=payload?.runtime?.resources; + if(entries===undefined)return;if(!Array.isArray(entries))throw fault('corrupt_recovery_image'); + const resources=new Map(entries),files=tx.objectStore('files'),base=prefix(scope,cut.authority); + for(const write of cut.writes){const id=write.ref.asset.id,resource=resources.get(id);if(!resource?.stored||resource.retirement?.purgedAt!==undefined)continue; + if(resource.id!==id||resource.digest!==write.ref.digest||resource.byteLength!==write.ref.asset.bytes||typeof resource.mimeType!=='string'||!equal(resource.stored,write.ref.asset))throw fault('shared_asset_identity_conflict',[id]); + const local=await value(files.get([key,id]));if(local===undefined)continue; + if(!(local instanceof Blob)){const pointer=depotFile(local,id);if(pointer.authority!==cut.authority||pointer.digest!==write.ref.digest||pointer.byteLength!==resource.byteLength||pointer.mimeType!==resource.mimeType)throw fault('shared_asset_identity_conflict',[id]);continue;} + const bytes=await value(tx.objectStore('asset-bytes').get([...base,write.ref.digest])); + if(!bytes)continue; + if(!(bytes instanceof Blob)||bytes.size!==resource.byteLength||local.size!==resource.byteLength||local.type!==resource.mimeType)throw fault('shared_asset_bytes_changed',[id]); + files.put({fileVersion:1,kind:'asset-depot',authority:cut.authority,resourceId:id,digest:write.ref.digest,byteLength:resource.byteLength,mimeType:resource.mimeType} satisfies DepotFile,[key,id]); + } +} type Execute=(leader:StorageLeader,stores:string[],mode:IDBTransactionMode,run:(tx:IDBTransaction)=>Promise)=>Promise; export function createDepotStorage(execute:Execute):AssetDepotPort{return Object.freeze({ async read(leader,scopeAuthority,id){authority(scopeAuthority);if(typeof id!=='string'||!id)throw fault('invalid_resource_id');return execute(leader,depotStores,'readonly',async tx=>{const base=prefix(leader.scope,scopeAuthority),row=await value(tx.objectStore('asset-depot').get([...base,id]));if(!row)return;validateFact(row,scopeAuthority,id);const bytes=row.purgedAt===undefined?await value(tx.objectStore('asset-bytes').get([...base,row.ref.digest])):undefined;if(bytes!==undefined&&(!(bytes instanceof Blob)||bytes.size!==row.ref.asset.bytes))throw fault('corrupt_asset_depot');return freeze({...row,...(bytes?{bytes}:{})}) as DepotEntry;});}, diff --git a/src/lib/canvas-runtime/depot-cut.ts b/src/lib/canvas-runtime/depot-cut.ts index d1ee153..4a803c0 100644 --- a/src/lib/canvas-runtime/depot-cut.ts +++ b/src/lib/canvas-runtime/depot-cut.ts @@ -7,7 +7,7 @@ import type {StorageLeader,StoragePort} from './storage-port.ts'; import {resourceRetentionPins} from './resource-retention.ts'; /** Derive one project's shared-byte manifest from the exact Root being persisted. */ -export async function prepareDepotCut(input:{snapshot:EngineSnapshot;registry:Registry;storage:StoragePort;leader:StorageLeader;authority:string;files:ReadonlyMap;displays:ReadonlyMap;temporary:ReadonlyMap;adoptedFiles?:ReadonlyMap}):Promise{ +export async function prepareDepotCut(input:{snapshot:EngineSnapshot;registry:Registry;storage:StoragePort;leader:StorageLeader;authority:string;files:ReadonlyMap;displays:ReadonlyMap;temporary:ReadonlyMap;adoptedFiles?:ReadonlyMap}):Promise{ const {snapshot,storage,leader,authority}=input,pins=resourceRetentionPins(snapshot,input.registry,input.displays,input.temporary),writes:DepotWrite[]=[]; if(!storage.assetDepot)throw fault('asset_depot_unavailable'); for(const resource of snapshot.runtime.resources.values())if(resource.stored&&resource.digest){ @@ -23,7 +23,7 @@ export async function prepareDepotCut(input:{snapshot:EngineSnapshot;registry:Re } for(const [id,binding]of snapshot.runtime.assetBindings??[]){ if(binding.authority!==authority||binding.asset.id!==id||snapshot.runtime.resources.has(id))throw fault('shared_asset_identity_conflict',[id]); - const bytes=input.adoptedFiles?.get(id);writes.push({ref:sharedAssetRef(binding),...(bytes?{bytes}:{})}); + const pending=input.adoptedFiles?.get(id),bytes=pending&&pending.rootRevision<=snapshot.rootRevision?pending.bytes:undefined;writes.push({ref:sharedAssetRef(binding),...(bytes?{bytes}:{})}); } // A source/run envelope may refer to resources outside this project's upload table. // Its complete resource interpreter must be installed before global deletion is allowed. diff --git a/src/lib/canvas-runtime/engine.ts b/src/lib/canvas-runtime/engine.ts index b60b91c..d43e5d1 100644 --- a/src/lib/canvas-runtime/engine.ts +++ b/src/lib/canvas-runtime/engine.ts @@ -79,7 +79,7 @@ export async function openProjectEngine(options: EngineOptions) { const network = createNetworkGate(2); const depot=options.assetDepot??(options.storage.assetDepot?createAssetDepot({scope:options.scope,port:options.storage.assetDepot,current:()=>!closing&&!closed&&(!options.isCurrentIdentity||options.isCurrentIdentity()),digest:sourceDigest}):undefined); if(depot&&(depot.scope.accountId!==options.scope.accountId||depot.scope.workspaceId!==options.scope.workspaceId))throw fault('asset_depot_scope_mismatch'); - const adoptionTasks=new Map>(),adoptedFiles=new Map(); + const adoptionTasks=new Map>(),adoptedFiles=new Map(); let adopting:SharedAsset|undefined; const runClock: Clock = options.clock ?? { now: () => Date.now(), setTimer: (fn, delay) => setTimeout(fn, delay), clearTimer: handle => clearTimeout(handle as ReturnType) }; const displays = new Set<() => void>(); @@ -198,7 +198,7 @@ export async function openProjectEngine(options: EngineOptions) { durableRootRevision = revision; pending.delete(revision); for(const id of deleteFiles){durablyPurged.add(id);collectionReservations.delete(id);} for (const file of files) fileWrites.delete(file.id); - for(const id of snapshot.runtime.assetBindings?.keys()??[])adoptedFiles.delete(id); + for(const [id,entry]of adoptedFiles)if(entry.rootRevision<=snapshot.rootRevision&&depot?.writes.some(write=>write.ref.asset.id===id&&write.bytes===entry.bytes))adoptedFiles.delete(id); if (editor.getSnapshot().runtime.ledger.durableGeneration < snapshot.runtime.ledger.generation) mutate(ledger => ledger.markDurable(snapshot.runtime.ledger.generation)); } persistenceFailed = false; @@ -922,9 +922,9 @@ export async function openProjectEngine(options: EngineOptions) { assertWritable(false);signal.throwIfAborted();if(authorized.ref.asset.url!==options.assets!.sourceURL(id))throw fault('asset_authorization_mismatch'); if(staged.length||adopting)throw fault('reentrant_dispatch');const runtime=editor.getSnapshot().runtime; if(runtime.resources.has(id)||runtime.orphanFiles?.has(id)||importFiles(runtime.imports).has(id)||fileWrites.has(id)||collectionReservations.has(id))throw fault('resource_identity_conflict',[id]); - const previous=adoptedFiles.get(id);adopting=authorized.ref;adoptedFiles.set(id,authorized.bytes); + const previous=adoptedFiles.get(id);adopting=authorized.ref;adoptedFiles.set(id,{bytes:authorized.bytes,rootRevision:Number.MAX_SAFE_INTEGER}); let committed=false; - try{const result=editor.transaction(tx=>callback(freeze({id,src:authorized.ref.asset.url,asset:authorized.ref.asset}),tx),{label:'Add existing image'});committed=result.status==='applied'&&result.operationIds.length>0;return result;} + try{const result=editor.transaction(tx=>callback(freeze({id,src:authorized.ref.asset.url,asset:authorized.ref.asset}),tx),{label:'Add existing image'});committed=result.status==='applied'&&result.operationIds.length>0;if(committed)adoptedFiles.set(id,{bytes:authorized.bytes,rootRevision:editor.getSnapshot().rootRevision});return result;} finally{adopting=undefined;if(!committed){if(previous)adoptedFiles.set(id,previous);else adoptedFiles.delete(id);}} })(); adoptionTasks.set(controller,task);void task.finally(()=>adoptionTasks.delete(controller)).catch(()=>{});return task; @@ -955,7 +955,7 @@ export async function openProjectEngine(options: EngineOptions) { const binding=editor.getSnapshot().runtime.assetBindings?.get(id); const key=JSON.stringify(binding?['shared-asset',id,binding.authority,binding.digest]:retained?['asset',id,retained.operationId,retained.retirement?.purgedAt??null]:orphan?['orphan',id,orphan.digest]:source?['source',id,source.sha256??null]:['unmanaged',id]); lease=displayCache.acquire(displayOwner,key,async signal=>{ - let bytes=adoptedFiles.get(id)??fileWrites.get(id)??await options.storage.readFile(leader,id); + let bytes=adoptedFiles.get(id)?.bytes??fileWrites.get(id)??await options.storage.readFile(leader,id); if(!bytes&&binding&&depot)bytes=await depot.read(leader,binding,signal); assertDisplayIdentity();if(signal.aborted)throw fault('display_aborted'); if(orphan&&(!bytes||bytes.size!==orphan.byteLength||bytes.type!==orphan.mimeType||await sourceDigest(bytes,signal)!==orphan.digest))throw fault('file_inventory_changed',[id]); diff --git a/src/lib/canvas-runtime/storage.ts b/src/lib/canvas-runtime/storage.ts index 0ccab05..63a871f 100644 --- a/src/lib/canvas-runtime/storage.ts +++ b/src/lib/canvas-runtime/storage.ts @@ -1,4 +1,4 @@ -import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut} from './asset-depot-storage.ts'; +import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut,promoteDepotFiles,resolveDepotFile} from './asset-depot-storage.ts'; import { applyJournal, diffJournal, type JournalPatch } from './journal-delta.ts'; import { isSealedRecoveryImage } from './recovery-encoding.ts'; import type { Json } from '../canvas-core/types.ts'; @@ -46,7 +46,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: let closed = false; const handles = new Set(), lastCuts=new WeakMap(), writeTails=new WeakMap>(); const opening = new Promise((resolve, reject) => { - const request = factory.open(options.name ?? 'endoleap-headless', 3); + const request = factory.open(options.name ?? 'endoleap-headless', 4); request.onupgradeneeded = () => { for (const name of ['metadata', 'checkpoints', 'journal', 'files',...depotStores]) if (!request.result.objectStoreNames.contains(name)) request.result.createObjectStore(name); }; @@ -158,13 +158,14 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: const saved=previous===undefined?undefined:await requestValue(transaction.objectStore(metadata.checkpointRevision===previous?'checkpoints':'journal').get([key,previous])); if(previous!==undefined&&image.rootRevision!==previous+1){ if(image.rootRevision!==previous)throw fault('storage_revision_gap'); - if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot)await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);return{durableRootRevision:previous}; + if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot){await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);await promoteDepotFiles(transaction,leader.scope,key,image,depot);}return{durableRootRevision:previous}; } if(previous!==undefined&&(!saved||image.localRevision=7&&base?.image.formatVersion!==image.formatVersion)||image.rootRevision-(metadata.checkpointRevision??0)>=checkpointEvery){ transaction.objectStore('checkpoints').put(stored,[key,image.rootRevision]);if(metadata.checkpointRevision!==undefined)transaction.objectStore('checkpoints').delete([key,metadata.checkpointRevision]);transaction.objectStore('journal').delete(IDBKeyRange.bound([key,0],[key,image.rootRevision]));metadata.checkpointRevision=image.rootRevision; @@ -178,21 +179,22 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: scanFiles(leader:StorageLeader,input:{expectedRootRevision:number;after?:string;limit:number}) { const {expectedRootRevision,after,limit}=input; if(!Number.isSafeInteger(expectedRootRevision)||expectedRootRevision<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))return Promise.reject(fault('invalid_file_inventory')); - return execute(leader,['files'],'readonly',async(transaction,metadata,key)=>{ + return execute(leader,['files',...depotStores],'readonly',async(transaction,metadata,key)=>{ if(metadata.throughRevision!==expectedRootRevision)throw fault('stale_file_inventory'); const rows=await new Promise((resolve,reject)=>{ const files:StoredFileInfo[]=[],range=IDBKeyRange.bound(after===undefined?[key]:[key,after],[key,[]],after!==undefined,true),request=transaction.objectStore('files').openCursor(range); request.onerror=()=>reject(request.error); - request.onsuccess=()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key,bytes=cursor.value; - if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]||!(bytes instanceof Blob)){reject(fault('corrupt_file_store'));return;} - files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue(); - }; + request.onsuccess=()=>{void(async()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key; + if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]){throw fault('corrupt_file_store');} + const bytes=await resolveDepotFile(transaction,leader.scope,parts[1],cursor.value); + if(bytes)files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue(); + })().catch(reject);}; }); const more=rows.length>limit,files=more?rows.slice(0,limit):rows; return Object.freeze({files:Object.freeze(files),rootRevision:expectedRootRevision,...(more?{nextAfter:files.at(-1)!.id}:{})}); }); }, - readFile(leader: StorageLeader, id: string) { return execute(leader, ['files'], 'readonly', (transaction, _metadata, key) => requestValue(transaction.objectStore('files').get([key, id]))); }, + readFile(leader: StorageLeader, id: string) { return execute(leader, ['files',...depotStores], 'readonly', async(transaction, _metadata, key) => resolveDepotFile(transaction,leader.scope,id,await requestValue(transaction.objectStore('files').get([key, id])))); }, close() { closed = true; for (const handle of handles) void handle.release(); void opening.then(database => database.close()).catch(() => {}); }, }); } diff --git a/tests/headless/asset-adoption.test.mjs b/tests/headless/asset-adoption.test.mjs index 2b1539a..01c9770 100644 --- a/tests/headless/asset-adoption.test.mjs +++ b/tests/headless/asset-adoption.test.mjs @@ -55,3 +55,15 @@ test('WorkspaceRuntime owns one depot across projects and closes it after projec assert.deepEqual(runtime.assetDepot.scope,{accountId,workspaceId});assert.equal(Object.isFrozen(f.storage),false); const p=await runtime.open(createDocument('p',[{id:'canvas',parent:null}],[]),new Map([['canvas',1]]));await adopt(p);await p.settle();const q=await runtime.open(createDocument('q',[{id:'canvas',parent:null}],[]),new Map([['canvas',1]]));await adopt(q);await q.settle();assert.equal(runtime.assetDepot.getResources().size,1);assert.equal(f.calls.filter(v=>v==='read').length,1);await runtime.close();assert.equal(f.closed,true);assert.equal(runtime.assetDepot.getResources().size,0); }); + +test('an older Root cannot release newly adopted bytes while its unpinned manifest is committing',async()=>{ + const f=fixture(),engine=await f.open('p',{historyLimit:1});let release,started;const commit=f.storage.commit; + try{ + await adopt(engine);await engine.settle();engine.editor.dispatch({type:'element.remove',target:{id:'picture',incarnation:1}});await engine.settle();engine.editor.dispatch({type:'element.create',element:{...node({id:'unused',src:'unused'},'text'),componentId:'text',data:{title:'text',text:''}}});await engine.settle(); + const ready=new Promise(resolve=>started=resolve),gate=new Promise(resolve=>release=resolve);let once=true; + f.storage.commit=async(leader,batch)=>{if(once){once=false;assert.equal(batch.depot.pins.some(([id])=>id==='shared'),false);started();await gate;}return commit(leader,batch);}; + engine.editor.updateRuntime(previous=>({...previous}));await ready; + // GC is allowed here: the old Root has no working/history/ACK/job reference. + f.cache.clear();assert.equal((await adopt(engine,'fresh')).status,'applied');release();await engine.settle();assert.equal(await f.cache.values().next().value.bytes.text(),'original bytes');assert.equal(f.roots.get('p').payload.document.elements.some(element=>element.id==='fresh'),true); + }finally{release?.();f.storage.commit=commit;await engine.close();} +}); diff --git a/tests/headless/asset-depot-browser-cases.mjs b/tests/headless/asset-depot-browser-cases.mjs index 591d993..3b14531 100644 --- a/tests/headless/asset-depot-browser-cases.mjs +++ b/tests/headless/asset-depot-browser-cases.mjs @@ -45,6 +45,7 @@ export async function assetDepotStorageCase(assert){ const staged=engine.stageFiles([{name:'source.png',bytes}],([handle],tx)=>{resource=handle.id;tx.dispatch({type:'element.create',element:{id:'picture',incarnation:1,structure:'node',componentId:'image',dataVersion:1,ownerCanvasId:'canvas',geometry:{x:0,y:0,width:2,height:1,rotation:0},data:{title:'picture',text:'',assetId:handle.id,src:handle.src}}});});assert(staged.status!=='rejected','engine image staging failed'); let ready=false;for(let i=0;i<200;i++){if(engine.getSnapshot().runtime.resources.get(resource)?.stored){ready=true;break;}await new Promise(resolve=>setTimeout(resolve,5));}assert(ready,'engine upload did not settle'); }finally{await engine.close();} + const physical=await new Promise((resolve,reject)=>{const request=database.transaction('files').objectStore('files').get([JSON.stringify([scope.accountId,scope.workspaceId,projectId]),resource]);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(physical?.fileVersion===1&&!(physical instanceof Blob),'actual engine upload retained duplicate project bytes'); const promoted=await storage.assetDepot.read(q,authority,resource);assert(promoted?.bytes&&await promoted.bytes.text()==='shared original'&&uploads===1,'engine did not promote acknowledged upload bytes'); assert((await collect(q,200)).pinnedIds.includes(resource),'closed engine did not retain durable document/history pins'); await q.release();await rejects(()=>collect(q,200),'storage_leader_lost');database.close(); diff --git a/tests/headless/browser-storage-cases.mjs b/tests/headless/browser-storage-cases.mjs index 39bbb6f..a249359 100644 --- a/tests/headless/browser-storage-cases.mjs +++ b/tests/headless/browser-storage-cases.mjs @@ -1,3 +1,4 @@ +import {depotFilesCase,depotFilesUpgradeCase,depotAdoptionRaceCase} from './depot-files-browser-cases.mjs'; import {assetAdoptionStorageCase} from './asset-adoption-browser-cases.mjs'; import {assetDepotStorageCase} from './asset-depot-browser-cases.mjs'; import {createBrowserImageDecoder} from '../../src/lib/canvas-runtime/browser-images.ts'; @@ -739,6 +740,9 @@ export async function run() { const rejects = async (callback, message) => { try { await callback(); } catch (error) { assert(error.message === message, `expected ${message}, received ${error.message}`); return; } throw new Error(`expected ${message}`); }; let leader, other; try { + await depotAdoptionRaceCase(assert);results.push('real IndexedDB GC and a gated older Root cannot consume or release bytes belonging to a newer adoption cut'); + await depotFilesCase(assert);results.push('stored uploads use shared-byte file pointers with original MIME, atomic promotion rollback, logical inventory and independent identity eviction/rehydration'); + await depotFilesUpgradeCase(assert);results.push('DB v3 original files/checkpoints upgrade to v4 and lazily promote through an idempotent Root cut'); await assetAdoptionStorageCase(assert);results.push('cross-project adoption uses fresh authorization, survives workspace/IndexedDB reopening and undo/redo, and preserves closed destination pins without project-local byte copies'); await assetDepotStorageCase(assert);results.push('workspace depot atomically shares digest bytes, preserves closed-project pins, blocks unknown/stale coverage and rolls back failed Root cuts'); leader = await storage.claim(scope); diff --git a/tests/headless/depot-files-browser-cases.mjs b/tests/headless/depot-files-browser-cases.mjs new file mode 100644 index 0000000..8061161 --- /dev/null +++ b/tests/headless/depot-files-browser-cases.mjs @@ -0,0 +1,51 @@ +import {createBrowserStorage} from '../../src/lib/canvas-runtime/storage.ts'; +export async function depotFilesCase(assert){ + const name=`depot-files-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},authority='https://assets.test',a=new Blob(['shared bytes'],{type:'image/jpeg'}),b=new Blob(['shared bytes'],{type:'image/png'}),digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await a.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''),storage=createBrowserStorage({name});let p,q,raw; + const ref=id=>({version:1,authority,digest,asset:{id,url:`${authority}/${id}`,width:2,height:1,bytes:a.size,mime_type:'image/png'}}),resource=(id,mimeType,stored=true)=>({id,operationId:`upload-${id}`,digest,byteLength:a.size,mimeType,...(stored?{stored:ref(id).asset}:{})}),image=(n,resources)=>({formatVersion:1,rootRevision:n,localRevision:n,payload:{runtime:{resources}}}),cut=(pins=[],writes=[])=>({version:1,authority,complete:true,pins,writes}); + const key=JSON.stringify(['a','w','p']),rows=[['one',resource('one',a.type)],['two',resource('two',b.type)]],pins=[['one',['element:one']],['two',['element:two']]],writes=[{ref:ref('one'),bytes:a},{ref:ref('two'),bytes:b}]; + const openRaw=()=>new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const get=(store,key)=>new Promise((resolve,reject)=>{const tx=raw.transaction(store),request=tx.objectStore(store).get(key);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const count=store=>new Promise((resolve,reject)=>{const tx=raw.transaction(store),request=tx.objectStore(store).count();request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const put=(store,key,value)=>new Promise((resolve,reject)=>{const tx=raw.transaction(store,'readwrite');tx.objectStore(store).put(value,key);tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);}); + const rejects=async(fn,code)=>{try{await fn();}catch(error){assert(error.code===code,`expected ${code}, got ${error.code}`);return;}throw Error(`expected ${code}`);}; + const collect=now=>storage.assetDepot.collect(p,{authority,now,retentionMs:10,limit:128}); + try{ + p=await storage.claim(scope);q=await storage.claim({...scope,projectId:'q'});raw=await openRaw();assert(raw.version===4,'pointer storage did not upgrade to DB v4'); + await storage.commit(p,{image:image(0,[['one',resource('one',a.type,false)],['two',resource('two',b.type,false)]]),files:[{id:'one',bytes:a},{id:'two',bytes:b}],depot:cut(pins)}); + assert(await get('files',[key,'one']) instanceof Blob,'unconfirmed upload lost its original bytes'); + // The first pointer is converted before the second row fails. Everything must roll back. + await rejects(()=>storage.commit(p,{image:image(1,[rows[0],['two',resource('two','image/webp')]]),depot:cut(pins,writes)}),'shared_asset_bytes_changed');assert(await get('files',[key,'one']) instanceof Blob&&await count('asset-bytes')===0&&(await storage.recover(p)).rootRevision===0,'failed promotion partially converted files/bytes/Root'); + await storage.commit(p,{image:image(1,rows),depot:cut(pins,writes)});assert(await count('asset-bytes')===1,'same digest occupied multiple shared byte entries');for(const id of ['one','two']){const local=await get('files',[key,id]);assert(!(local instanceof Blob)&&local.fileVersion===1&&local.digest===digest,'stored upload still retained a project-local Blob');} + assert((await storage.readFile(p,'one')).type===a.type&&(await storage.readFile(p,'two')).type===b.type,'pointer read changed the original File MIME');assert(await(await storage.readFile(p,'one')).text()==='shared bytes','pointer did not resolve original bytes'); + const inventory=await storage.scanFiles(p,{expectedRootRevision:1,limit:128});assert(inventory.files.length===2&&inventory.files[0].mimeType===a.type&&inventory.files.every(row=>row.byteLength===a.size),'inventory did not expose logical file metadata'); + await storage.commit(q,{image:image(0,[]),depot:cut([['one',['history:q']]])});await storage.commit(p,{image:image(2,rows),depot:cut()});await collect(0);await collect(10);assert(await count('asset-bytes')===1,'purging one ResourceId deleted another ResourceId bytes');assert(await storage.readFile(p,'two')===undefined,'purged identity resolved through another live digest owner');assert((await storage.scanFiles(p,{expectedRootRevision:2,limit:128})).files.length===1,'inventory advertised an evicted pointer as readable bytes'); + await storage.commit(q,{image:image(1,[]),depot:cut()});await collect(10);await collect(20);assert(await count('asset-bytes')===0&&await storage.readFile(p,'one')===undefined,'last unreferenced digest was retained'); + await storage.commit(p,{image:image(3,rows),depot:cut([['one',['element:one']]],[writes[0]])});assert(await count('asset-bytes')===1&&await(await storage.readFile(p,'one')).text()==='shared bytes','same identity could not rehydrate a file pointer');assert(await storage.readFile(p,'two')===undefined,'rehydrating one identity revived another purged identity'); + const pointer=await get('files',[key,'one']);await put('files',[key,'one'],{...pointer,fileVersion:2});await rejects(()=>storage.readFile(p,'one'),'file_interpreter_unavailable');await rejects(()=>storage.scanFiles(p,{expectedRootRevision:3,limit:128}),'file_interpreter_unavailable');await put('files',[key,'one'],{...pointer,digest:'0'.repeat(64)});await rejects(()=>storage.readFile(p,'one'),'corrupt_file_store');await put('files',[key,'one'],pointer); + const old=await new Promise(resolve=>{const request=indexedDB.open(name,3);request.onerror=event=>{event.preventDefault();resolve(request.error.name);};request.onsuccess=()=>{request.result.close();resolve('accepted');};});assert(old==='VersionError','DB v3 reader accepted pointer file rows'); + }finally{raw?.close();await p?.release();await q?.release();storage.close();indexedDB.deleteDatabase(name);} +} + +export async function depotFilesUpgradeCase(assert){ + const name=`depot-upgrade-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},key=JSON.stringify(['a','w','p']),authority='https://assets.test',bytes=new Blob(['old bytes'],{type:'image/png'}),hex=buffer=>[...new Uint8Array(buffer)].map(n=>n.toString(16).padStart(2,'0')).join(''),digest=hex(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer())),asset={id:'old',url:`${authority}/old`,width:2,height:1,bytes:bytes.size,mime_type:bytes.type},image={formatVersion:1,rootRevision:0,localRevision:0,payload:{runtime:{resources:[['old',{id:'old',digest,byteLength:bytes.size,mimeType:bytes.type,stored:asset}]]}}},checksum=hex(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(image)))); + let raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name,3);request.onupgradeneeded=()=>{for(const store of ['metadata','checkpoints','journal','files','asset-depot','asset-bytes','asset-pins'])request.result.createObjectStore(store);};request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + await new Promise((resolve,reject)=>{const tx=raw.transaction(['metadata','checkpoints','files'],'readwrite');tx.objectStore('metadata').put({fence:1,throughRevision:0,checkpointRevision:0},key);tx.objectStore('checkpoints').put({image,checksum},[key,0]);tx.objectStore('files').put(bytes,[key,'old']);tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);});raw.close();const storage=createBrowserStorage({name});let leader; + try{leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===0,'v3 checkpoint lost during upgrade');await storage.commit(leader,{image,depot:{version:1,authority,complete:true,pins:[['old',['element:old']]],writes:[{ref:{version:1,authority,digest,asset},bytes}]}});assert(await(await storage.readFile(leader,'old')).text()==='old bytes','idempotent reindex lost old bytes');raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const row=await new Promise((resolve,reject)=>{const request=raw.transaction('files').objectStore('files').get([key,'old']);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(raw.version===4&&row.fileVersion===1&&!(row instanceof Blob),'old project was not lazily promoted');raw.close();}finally{await leader?.release();storage.close();indexedDB.deleteDatabase(name);} +} + +export async function depotAdoptionRaceCase(assert){ + const {openProjectEngine}=await import('../../src/lib/canvas-runtime/engine.ts'),{createDocument,createRegistry,builtinComponents,applyChanges}=await import('../../src/lib/canvas-core/index.ts'); + const name=`depot-race-${crypto.randomUUID()}`,base=createBrowserStorage({name}),scope={accountId:'a',workspaceId:'w',projectId:'source'},authority='https://assets.test',bytes=new Blob(['race bytes'],{type:'image/png'}),digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''),ref={version:1,authority,digest,asset:{id:'shared',url:`${authority}/shared`,width:2,height:1,bytes:bytes.size,mime_type:bytes.type}};let leader,engine,release,started,gate,blocked=false,version=1,hydrations=0;const receipts=new Map(); + let document=createDocument('target',[{id:'canvas',parent:null}],[]);const registry=createRegistry(builtinComponents()); + const storage={...base,async commit(held,batch){if(gate&&!blocked){blocked=true;assert(!batch.depot.pins.some(([id])=>id==='shared'),'older Root was not unpinned');started();await gate;}return base.commit(held,batch);}}; + const assets={authority:{scope:authority,async resolve(){return ref;},async read(){hydrations++;return bytes;}},sourceURL:id=>`${authority}/${id}`,displayURL:id=>`/assets/${id}`,async upload(){throw Error('unexpected upload');},async query(){throw Error('unexpected upload query');},accept(){throw Error('unexpected upload receipt');}},persistence={compile(intent,id,snapshot){return{payload:{intent},canvasIds:intent.canvasIds,expectedVersions:{canvas:snapshot.runtime.versions.get('canvas')}};},async submit(request){document=applyChanges(document,JSON.parse(request.body).intent.changes);receipts.set(request.id,{document,versions:new Map([['canvas',++version]])});return{outcome:'applied'};},async query(){return{outcome:'unresolved'};},canonical:request=>receipts.get(request.id)}; + const picture=(handle,id)=>({id,incarnation:1,structure:'node',componentId:'image',dataVersion:1,ownerCanvasId:'canvas',geometry:{x:0,y:0,width:2,height:1,rotation:0},data:{title:id,text:'',assetId:handle.id,src:handle.src}}),adopt=id=>engine.adoptResource('shared',(handle,tx)=>tx.dispatch({type:'element.create',element:picture(handle,id)})); + try{ + leader=await storage.claim(scope);await storage.commit(leader,{image:{formatVersion:1,rootRevision:0,localRevision:0,payload:null},depot:{version:1,authority,complete:true,pins:[],writes:[{ref,bytes}]}}); + engine=await openProjectEngine({scope:{...scope,projectId:'target'},registry,document,versions:new Map([['canvas',1]]),storage,assets,persistence,historyLimit:1});assert((await adopt('old')).status==='applied','initial adoption failed');await engine.settle();engine.editor.dispatch({type:'element.remove',target:{id:'old',incarnation:1}});await engine.settle();engine.editor.dispatch({type:'element.create',element:{...picture({id:'unused',src:'unused'},'text'),componentId:'text',data:{title:'text',text:''}}});await engine.settle(); + const ready=new Promise(resolve=>started=resolve);gate=new Promise(resolve=>release=resolve);engine.editor.updateRuntime(previous=>({...previous}));await ready; + // Old commit has not acquired IDB yet. A complete unpinned manifest permits GC. + await storage.assetDepot.collect(leader,{authority,now:0,retentionMs:0,limit:128});const purged=await storage.assetDepot.collect(leader,{authority,now:0,retentionMs:0,limit:128});assert(purged.purgedIds.includes('shared'),'GC could not retire the actually unreferenced identity');assert(!(await storage.assetDepot.read(leader,authority,'shared')).bytes,'GC left bytes before rehydration'); + assert((await adopt('new')).status==='applied'&&hydrations===1,'fresh adoption did not rehydrate original bytes');release();await engine.settle();const restored=await storage.assetDepot.read(leader,authority,'shared');assert(restored.bytes&&await restored.bytes.text()==='race bytes','older Root released the new adoption bytes before the matching durable cut');const stillPinned=await storage.assetDepot.collect(leader,{authority,now:1,retentionMs:0,limit:128});assert(stillPinned.pinnedIds.includes('shared'),'new Root did not atomically persist resource pins'); + }finally{release?.();await engine?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(name);} +}