From 74c219f91d4302df17536140b53f8d967b565330 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 15:54:05 +0800 Subject: [PATCH] Persist shared asset bytes and project pins with Root commits --- src/app/api/foundry/[...path]/route.ts | 5 +- src/lib/canvas-runtime/asset-depot-storage.ts | 53 +++++++++++++++++++ src/lib/canvas-runtime/asset-depot.ts | 35 ++++++++++++ src/lib/canvas-runtime/depot-cut.ts | 28 ++++++++++ src/lib/canvas-runtime/engine.ts | 4 +- src/lib/canvas-runtime/resources.ts | 1 + src/lib/canvas-runtime/storage-port.ts | 3 +- src/lib/canvas-runtime/storage.ts | 13 +++-- src/lib/canvas-service/assets.ts | 4 +- src/lib/canvas-service/foundry.ts | 17 ++++-- tests/headless/asset-depot-browser-cases.mjs | 52 ++++++++++++++++++ tests/headless/asset-depot.test.mjs | 41 ++++++++++++++ tests/headless/browser-storage-cases.mjs | 4 +- tests/headless/proxy.integration.mjs | 2 + 14 files changed, 249 insertions(+), 13 deletions(-) create mode 100644 src/lib/canvas-runtime/asset-depot-storage.ts create mode 100644 src/lib/canvas-runtime/asset-depot.ts create mode 100644 src/lib/canvas-runtime/depot-cut.ts create mode 100644 tests/headless/asset-depot-browser-cases.mjs create mode 100644 tests/headless/asset-depot.test.mjs diff --git a/src/app/api/foundry/[...path]/route.ts b/src/app/api/foundry/[...path]/route.ts index 7af1d3d..057afe0 100644 --- a/src/app/api/foundry/[...path]/route.ts +++ b/src/app/api/foundry/[...path]/route.ts @@ -5,7 +5,7 @@ import { NextRequest, NextResponse } from "next/server"; const allowed = /^(profile|auth\/(login|register|verify-email|exchange|me|logout)|workspaces(?:\/[a-zA-Z0-9_-]+(?:\/(?:materials(?:\/[a-zA-Z0-9_-]+(?:\/canvas)?)?|search))?)?)$/; const canvasRoute = /^workspaces\/[a-zA-Z0-9_-]+\/materials\/[a-zA-Z0-9_-]+\/canvas(?:\/(?:model(?:\/(?:migrate|operations(?:\/[a-zA-Z0-9_-]+)?))?|operations(?:\/[a-zA-Z0-9_-]+)?|connections|capabilities|subgraphs(?:\/(?:migrate|[a-zA-Z0-9_-]+\/(?:members|dissolve)))?|transactions\/[a-zA-Z0-9_-]+\/undo|ai\/(?:models|operations\/[a-zA-Z0-9_-]+|runs(?:\/[a-zA-Z0-9_-]+(?:\/(?:cancel|apply))?)?)))?$/; -const assetRoute = /^workspaces\/[a-zA-Z0-9_-]+\/assets(?:\/(?:operations\/[a-zA-Z0-9_-]+|[a-zA-Z0-9_-]+))?$/; +const assetRoute = /^workspaces\/[a-zA-Z0-9_-]+\/assets(?:\/(?:(?:operations|metadata)\/[a-zA-Z0-9_-]+|[a-zA-Z0-9_-]+))?$/; const bodyLimit = 1024 * 1024; // Route Handlers do not have the Pages Router bodyParser size limit. @@ -54,7 +54,8 @@ async function handler( const path = (await params).path.join("/"); if (!allowed.test(path) && !canvasRoute.test(path) && !assetRoute.test(path)) return json({ error: "not_found" }, 404); const assetReceipt = assetRoute.test(path) && /\/assets\/operations\//.test(path); - const asset = assetRoute.test(path) && !assetReceipt; + const assetMetadata = assetRoute.test(path) && /\/assets\/metadata\//.test(path); + const asset = assetRoute.test(path) && !assetReceipt && !assetMetadata; const canvas = canvasRoute.test(path); const canonical = canvas && /\/canvas\/model(?:\/|$)/.test(path); const frozenAI = canvas && /\/canvas\/ai\/runs(?:\/|$)/.test(path); diff --git a/src/lib/canvas-runtime/asset-depot-storage.ts b/src/lib/canvas-runtime/asset-depot-storage.ts new file mode 100644 index 0000000..e17c00d --- /dev/null +++ b/src/lib/canvas-runtime/asset-depot-storage.ts @@ -0,0 +1,53 @@ +import {equal,fault,freeze} from '../canvas-core/values.ts'; +import {sharedAssetRef,validateSharedAsset,type AssetDepotPort,type DepotCut,type DepotEntry,type SharedAsset} from './asset-depot.ts'; +import type {StorageLeader,StorageScope} from './storage-port.ts'; + +export const depotStores=['asset-depot','asset-bytes','asset-pins']; +type Fact={ref:SharedAsset;unreferencedAt?:number;purgedAt?:number}; +type Pins={version:1;rootRevision:number;complete:boolean;pins:readonly (readonly [string,readonly string[]])[]}; +const value=(request:IDBRequest)=>new Promise((resolve,reject)=>{request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); +const prefix=(scope:StorageScope,authority:string)=>[scope.accountId,scope.workspaceId,authority]; +const range=(scope:StorageScope,authority:string)=>IDBKeyRange.bound(prefix(scope,authority),[...prefix(scope,authority),[]],false,true); +function authority(value:string){if(typeof value!=='string'||!value||value.length>8192)throw fault('invalid_depot_cut');} +function validateFact(row:Fact,scopeAuthority:string,id:string){validateSharedAsset(row?.ref);if(row.ref.authority!==scopeAuthority||row.ref.asset.id!==id)throw fault('corrupt_asset_depot');for(const n of [row.unreferencedAt,row.purgedAt])if(n!==undefined&&(!Number.isSafeInteger(n)||n<0))throw fault('corrupt_asset_depot');if(row.purgedAt!==undefined&&(row.unreferencedAt===undefined||row.purgedAt(),pins=cut.pins.map(row=>{if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||seen.has(row[0])||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner)||new Set(row[1]).size!==row[1].length)throw fault('invalid_depot_cut');seen.add(row[0]);return Object.freeze([row[0],Object.freeze([...row[1]])] as const);}); + const written=new Set(),writes=cut.writes.map(row=>{validateSharedAsset(row.ref);if(row.ref.authority!==cut.authority||written.has(row.ref.asset.id)||row.bytes!==undefined&&!(row.bytes instanceof Blob))throw fault('invalid_depot_cut');written.add(row.ref.asset.id);return Object.freeze({ref:sharedAssetRef(row.ref),...(row.bytes?{bytes:row.bytes}:{})});}); + return Object.freeze({version:1,authority:cut.authority,complete:cut.complete,pins:Object.freeze(pins),writes:Object.freeze(writes)}); +} +export async function verifyDepotBytes(cut:DepotCut){ + for(const row of cut.writes)if(row.bytes){if(row.bytes.size!==row.ref.asset.bytes)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);const digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await row.bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join('');if(digest!==row.ref.digest)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);} +} +/** Must run in the same transaction as the project Root, including idempotent reindexing. */ +export async function writeDepotCut(tx:IDBTransaction,scope:StorageScope,rootRevision:number,cut:DepotCut){ + const base=prefix(scope,cut.authority),facts=tx.objectStore('asset-depot'),bytes=tx.objectStore('asset-bytes'); + for(const row of cut.writes){const key=[...base,row.ref.asset.id],previous=await value(facts.get(key));if(previous){validateFact(previous,cut.authority,row.ref.asset.id);if(!equal(previous.ref,row.ref))throw fault('shared_asset_identity_conflict',[row.ref.asset.id]);} + // Digest-addressed bytes are shared across resource IDs and projects in this authority. + if(row.bytes)bytes.put(row.bytes,[...base,row.ref.digest]); + facts.put({...previous,ref:row.ref,...(row.bytes?{unreferencedAt:undefined,purgedAt:undefined}:{})},key); + } + tx.objectStore('asset-pins').put({version:1,rootRevision,complete:cut.complete,pins:cut.pins} satisfies Pins,[...base,scope.projectId]); +} +type Execute=(leader:StorageLeader,stores:string[],mode:IDBTransactionMode,run:(tx:IDBTransaction)=>Promise)=>Promise; +export function createDepotStorage(execute:Execute):AssetDepotPort{return Object.freeze({ + async read(leader,scopeAuthority,id){authority(scopeAuthority);if(typeof id!=='string'||!id)throw fault('invalid_resource_id');return execute(leader,depotStores,'readonly',async tx=>{const base=prefix(leader.scope,scopeAuthority),row=await value(tx.objectStore('asset-depot').get([...base,id]));if(!row)return;validateFact(row,scopeAuthority,id);const bytes=row.purgedAt===undefined?await value(tx.objectStore('asset-bytes').get([...base,row.ref.digest])):undefined;if(bytes!==undefined&&(!(bytes instanceof Blob)||bytes.size!==row.ref.asset.bytes))throw fault('corrupt_asset_depot');return freeze({...row,...(bytes?{bytes}:{})}) as DepotEntry;});}, + async collect(leader,input){authority(input.authority);const {now,retentionMs,limit,after}=input;if(!Number.isSafeInteger(now)||now<0||!Number.isSafeInteger(retentionMs)||retentionMs<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))throw fault('invalid_depot_collection'); + return execute(leader,depotStores,'readwrite',async tx=>{ + const scope=leader.scope,metadata=tx.objectStore('metadata'),keys=await value(metadata.getAllKeys()),heads=await value<{throughRevision?:number}[]>(metadata.getAll()),base=prefix(scope,input.authority),manifests=tx.objectStore('asset-pins'),blockedProjects:string[]=[],pinned=new Set(); + for(let i=0;itypeof p!=='string'||!p))throw fault('corrupt_storage_scope');if(parts[0]!==scope.accountId||parts[1]!==scope.workspaceId||heads[i].throughRevision===undefined)continue; + const pins=await value(manifests.get([...base,parts[2]]));if(!pins||pins.version!==1||pins.rootRevision!==heads[i].throughRevision||pins.complete!==true){blockedProjects.push(parts[2]);continue;} + if(!Array.isArray(pins.pins))throw fault('corrupt_depot_pins');for(const row of pins.pins){if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner))throw fault('corrupt_depot_pins');pinned.add(row[0]);} + } + if(blockedProjects.length)return freeze({complete:false,blockedProjects,markedIds:[],purgedIds:[],pinnedIds:[]}); + const store=tx.objectStore('asset-depot'),factKeys=await value(store.getAllKeys(range(scope,input.authority))),rows=await value(store.getAll(range(scope,input.authority))),markedIds:string[]=[],purgedIds:string[]=[],pinnedIds:string[]=[]; + for(let i=0;i({row,key:factKeys[i] as string[]})).filter(({key})=>after===undefined||key[3]>after).slice(0,limit+1),page=candidates.slice(0,limit); + for(const {row,key}of page){const id=key[3];if(pinned.has(id)){pinnedIds.push(id);if(row.purgedAt===undefined&&row.unreferencedAt!==undefined){delete row.unreferencedAt;store.put(row,key);}continue;}if(row.purgedAt!==undefined)continue;if(row.unreferencedAt===undefined){row.unreferencedAt=now;markedIds.push(id);store.put(row,key);}else if(now>=row.unreferencedAt&&now-row.unreferencedAt>=retentionMs){row.purgedAt=now;purgedIds.push(id);store.put(row,key);}} + // Preserve a digest while any other live resource identity still owns its bytes. + for(const digest of new Set(page.filter(({row})=>row.purgedAt===now).map(({row})=>row.ref.digest)))if(!rows.some(row=>row.ref.digest===digest&&row.purgedAt===undefined))tx.objectStore('asset-bytes').delete([...base,digest]); + return freeze({complete:true,blockedProjects:[],markedIds,purgedIds,pinnedIds,...(candidates.length>limit?{nextAfter:page.at(-1)!.key[3]}:{})}); + }); + }, +});} diff --git a/src/lib/canvas-runtime/asset-depot.ts b/src/lib/canvas-runtime/asset-depot.ts new file mode 100644 index 0000000..4573836 --- /dev/null +++ b/src/lib/canvas-runtime/asset-depot.ts @@ -0,0 +1,35 @@ +import type {Id} from '../canvas-core/types.ts'; +import {equal,fault,freeze,readMap} from '../canvas-core/values.ts'; +import type {StoredAsset,LoadingAssetAdmission} from './resources.ts'; +import type {StorageLeader,StorageScope} from './storage-port.ts'; + +export type SharedAsset={readonly version:1;readonly authority:string;readonly digest:string;readonly asset:StoredAsset}; +export type SharedAssetBinding=SharedAsset&{readonly loadingAdmission?:(LoadingAssetAdmission&{readonly operationId:Id})}; +export type AssetAuthority={readonly scope:string;resolve(id:Id,signal:AbortSignal):Promise;read(id:Id,signal:AbortSignal):Promise}; +export type DepotWrite={readonly ref:SharedAsset;readonly bytes?:Blob}; +export type DepotCut={readonly version:1;readonly authority:string;readonly complete:boolean;readonly pins:readonly (readonly [Id,readonly string[]])[];readonly writes:readonly DepotWrite[]}; +export type DepotEntry={readonly ref:SharedAsset;readonly bytes?:Blob;readonly unreferencedAt?:number;readonly purgedAt?:number}; +export type DepotCollection={readonly complete:boolean;readonly blockedProjects:readonly Id[];readonly markedIds:readonly Id[];readonly purgedIds:readonly Id[];readonly pinnedIds:readonly Id[];readonly nextAfter?:Id}; +export type AssetDepotPort={ + read(leader:StorageLeader,authority:string,id:Id):Promise; + collect(leader:StorageLeader,input:{readonly authority:string;readonly now:number;readonly retentionMs:number;readonly limit:number;readonly after?:Id}):Promise; +}; +export function validateSharedAsset(value:SharedAsset):void{ + if(!value||value.version!==1||typeof value.authority!=='string'||!value.authority||value.authority.length>8192||typeof value.digest!=='string'||!/^([a-f0-9]{64})$/.test(value.digest))throw fault('invalid_shared_asset'); + const asset=value.asset;if(!asset||typeof asset.id!=='string'||!asset.id||typeof asset.url!=='string'||!asset.url||asset.url.length>8192||![asset.width,asset.height,asset.bytes].every(n=>Number.isSafeInteger(n)&&n>0)||asset.width*asset.height>40_000_000||asset.bytes>20*1024*1024||!['image/png','image/jpeg','image/gif','image/webp'].includes(asset.mime_type))throw fault('invalid_shared_asset'); +} +export function sharedAssetRef(value:SharedAsset):SharedAsset{validateSharedAsset(value);return freeze({version:1 as const,authority:value.authority,digest:value.digest,asset:{...value.asset}});} +const same=(a:SharedAsset,b:SharedAsset)=>equal(sharedAssetRef(a),sharedAssetRef(b)); +/** Immutable metadata cache only. Every new adoption resolves current permission first. */ +export function createAssetDepot(options:{scope:Pick;port:AssetDepotPort;current():boolean;digest(bytes:Blob,signal?:AbortSignal):Promise}){ + const scope=freeze({...options.scope});let closed=false;const records=new Map(),controllers=new Set(); + const check=(leader:StorageLeader)=>{if(closed||!options.current())throw fault('identity_changed');if(!leader.active||leader.scope.accountId!==scope.accountId||leader.scope.workspaceId!==scope.workspaceId)throw fault('asset_depot_scope_mismatch');}; + function remember(ref:SharedAsset){validateSharedAsset(ref);const key=JSON.stringify([ref.authority,ref.asset.id]),previous=records.get(key);if(previous&&!same(previous,ref))throw fault('shared_asset_identity_conflict',[ref.asset.id]);records.set(key,sharedAssetRef(ref));} + return Object.freeze({scope,getResources:()=>readMap(records), + async authorize(leader:StorageLeader,authority:AssetAuthority,id:Id,signal:AbortSignal){check(leader);const controller=new AbortController(),combined=AbortSignal.any([signal,controller.signal]);controllers.add(controller);try{const ref=await authority.resolve(id,combined);check(leader);combined.throwIfAborted();validateSharedAsset(ref);if(ref.authority!==authority.scope||ref.asset.id!==id)throw fault('asset_authorization_mismatch');const cached=await options.port.read(leader,authority.scope,id);check(leader);combined.throwIfAborted();if(cached&&!same(cached.ref,ref))throw fault('shared_asset_identity_conflict',[id]);let bytes=cached?.bytes;if(!bytes)bytes=await authority.read(id,combined);check(leader);combined.throwIfAborted();if(bytes.size!==ref.asset.bytes||await options.digest(bytes,combined)!==ref.digest)throw fault('shared_asset_bytes_changed',[id]);check(leader);combined.throwIfAborted();remember(ref);return Object.freeze({ref,bytes});}finally{controllers.delete(controller);}}, + async read(leader:StorageLeader,ref:SharedAsset,signal?:AbortSignal){check(leader);signal?.throwIfAborted();const row=await options.port.read(leader,ref.authority,ref.asset.id);check(leader);signal?.throwIfAborted();if(!row)return;if(!same(row.ref,ref))throw fault('shared_asset_identity_conflict',[ref.asset.id]);if(row.bytes&&(row.bytes.size!==ref.asset.bytes||await options.digest(row.bytes,signal)!==ref.digest))throw fault('shared_asset_bytes_changed',[ref.asset.id]);check(leader);signal?.throwIfAborted();remember(ref);return row.bytes;}, + async collect(leader:StorageLeader,input:Parameters[1]){check(leader);const result=await options.port.collect(leader,input);check(leader);return result;}, + close(){closed=true;for(const controller of controllers)controller.abort();controllers.clear();records.clear();}, + }); +} +export type AssetDepot=ReturnType; diff --git a/src/lib/canvas-runtime/depot-cut.ts b/src/lib/canvas-runtime/depot-cut.ts new file mode 100644 index 0000000..e8cf212 --- /dev/null +++ b/src/lib/canvas-runtime/depot-cut.ts @@ -0,0 +1,28 @@ +import type {Registry} from '../canvas-core/types.ts'; +import {fault} from '../canvas-core/values.ts'; +import type {EngineSnapshot} from './contracts.ts'; +import type {DepotCut,DepotWrite} from './asset-depot.ts'; +import {sharedAssetRef} from './asset-depot.ts'; +import type {StorageLeader,StoragePort} from './storage-port.ts'; +import {resourceRetentionPins} from './resource-retention.ts'; + +/** Derive one project's shared-byte manifest from the exact Root being persisted. */ +export async function prepareDepotCut(input:{snapshot:EngineSnapshot;registry:Registry;storage:StoragePort;leader:StorageLeader;authority:string;files:ReadonlyMap;displays:ReadonlyMap;temporary:ReadonlyMap}):Promise{ + const {snapshot,storage,leader,authority}=input,pins=resourceRetentionPins(snapshot,input.registry,input.displays,input.temporary),writes:DepotWrite[]=[]; + if(!storage.assetDepot)throw fault('asset_depot_unavailable'); + for(const resource of snapshot.runtime.resources.values())if(resource.stored&&resource.digest){ + const ref=sharedAssetRef({version:1,authority,digest:resource.digest,asset:resource.stored}); + if(ref.asset.id!==resource.id||ref.asset.bytes!==resource.byteLength)throw fault('shared_asset_identity_conflict',[resource.id]); + let bytes:Blob|undefined; + if(resource.retirement?.purgedAt===undefined&&pins.pins.has(resource.id)){ + const cached=await storage.assetDepot.read(leader,authority,resource.id); + // The store enforces immutable identity even when bytes are already present. + if(!cached?.bytes)bytes=input.files.get(resource.id)??await storage.readFile(leader,resource.id); + } + writes.push({ref,...(bytes?{bytes}:{})}); + } + // A source/run envelope may refer to resources outside this project's upload table. + // Its complete resource interpreter must be installed before global deletion is allowed. + const complete=pins.complete&&!snapshot.runtime.loading&&![...(snapshot.runtime.imports?.values()??[])].some(job=>!['completed','partial','cancelled'].includes(job.status))&&snapshot.runtime.runs.size===0; + return{version:1,authority,complete,pins:[...pins.pins].map(([id,owners])=>[id,[...owners]] as const),writes}; +} diff --git a/src/lib/canvas-runtime/engine.ts b/src/lib/canvas-runtime/engine.ts index 97ec694..2942717 100644 --- a/src/lib/canvas-runtime/engine.ts +++ b/src/lib/canvas-runtime/engine.ts @@ -1,3 +1,4 @@ +import {prepareDepotCut} from './depot-cut.ts'; import {loadingAssetURLs} from './loading-assets.ts'; import {loadingImportAdmission,validateLoadingCopySource} from './loading-import.ts'; import {loadingCanSend,mergeLoadingRead,loadingQueryCoverage,assertLoadingChanges,loadingCommand,validateLoadingRuntime,validatePrefixExtension,validateLoadingCompletion,type LoadingState} from './loading.ts'; @@ -169,7 +170,8 @@ export async function openProjectEngine(options: EngineOptions) { const sourceFiles=importFiles(snapshot.runtime.imports); const files = [...fileWrites].filter(([id]) => snapshot.runtime.resources.has(id)||sourceFiles.has(id)).map(([id, bytes]) => ({ id, bytes })); const deleteFiles=[...[...snapshot.runtime.resources.values()].filter(resource=>resource.retirement?.purgedAt!==undefined&&!durablyPurged.has(resource.id)).map(resource=>resource.id),...[...sourceFiles].filter(([id,entry])=>entry.file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id),...[...(snapshot.runtime.orphanFiles??[])].filter(([id,file])=>file.retirement?.purgedAt!==undefined&&!durablyPurged.has(id)).map(([id])=>id)]; - const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles }); + const depot=options.assets?.authority&&options.storage.assetDepot?await prepareDepotCut({snapshot,registry:options.registry,storage:options.storage,leader,authority:options.assets.authority.scope,files:fileWrites,displays:displayPins,temporary:admissionPins(snapshot)}):undefined; + const receipt = await options.storage.commit(leader, { image: encodeRecovery(snapshot), files,deleteFiles,...(depot?{depot}:{}) }); if (receipt.durableRootRevision !== revision) throw fault('invalid_durable_receipt'); durableRootRevision = revision; pending.delete(revision); for(const id of deleteFiles){durablyPurged.add(id);collectionReservations.delete(id);} diff --git a/src/lib/canvas-runtime/resources.ts b/src/lib/canvas-runtime/resources.ts index 94eb674..a6f65da 100644 --- a/src/lib/canvas-runtime/resources.ts +++ b/src/lib/canvas-runtime/resources.ts @@ -12,6 +12,7 @@ export type LoadingAssetAdmission = {readonly version:1;readonly sourceToken:str export type ResourceRecord = { readonly loadingAdmission?:LoadingAssetAdmission; readonly id: Id; readonly operationId: Id; readonly name: string; readonly mimeType: string; readonly byteLength: number; readonly sizing?: DefaultImageSizing; readonly image?:ImageDecodeState; readonly digest?: string; readonly retirement?: {readonly version:1;readonly unreferencedAt:number;readonly purgedAt?:number}; readonly stored?: StoredAsset }; export type DisplayHandle = { readonly url: string; release(): void }; export type AssetPort = { + readonly authority?:import('./asset-depot.ts').AssetAuthority; sourceURL(resourceId: Id): string; displayURL(resourceId: Id): string; upload(request: FrozenRequest, bytes: Blob, signal: AbortSignal): Promise; diff --git a/src/lib/canvas-runtime/storage-port.ts b/src/lib/canvas-runtime/storage-port.ts index aeae06a..d965e28 100644 --- a/src/lib/canvas-runtime/storage-port.ts +++ b/src/lib/canvas-runtime/storage-port.ts @@ -3,11 +3,12 @@ import type { Json } from '../canvas-core/types.ts'; /** Portable durable-store contract. Browser database types belong to the adapter. */ export type StorageScope = { readonly accountId: string; readonly workspaceId: string; readonly projectId: string }; export type RecoveryImage = { readonly formatVersion: 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json }; -export type DurableBatch = { readonly image: RecoveryImage; readonly deleteFiles?:readonly string[]; readonly files?: readonly { readonly id: string; readonly bytes: Blob }[] }; +export type DurableBatch = { readonly image: RecoveryImage; readonly deleteFiles?:readonly string[]; readonly depot?:import('./asset-depot.ts').DepotCut; readonly files?: readonly { readonly id: string; readonly bytes: Blob }[] }; export type StorageLeader = { readonly scope: StorageScope; readonly fence: number; readonly active: boolean; release(): Promise }; export type StoredFileInfo={readonly id:string;readonly byteLength:number;readonly mimeType:string}; export type FileInventoryPage={readonly files:readonly StoredFileInfo[];readonly rootRevision:number;readonly nextAfter?:string}; export type StoragePort = { + readonly assetDepot?:import('./asset-depot.ts').AssetDepotPort; scanFiles?(leader:StorageLeader,input:{readonly expectedRootRevision:number;readonly after?:string;readonly limit:number}):Promise; readonly fileDeletion?: 'atomic-v1'; claim(scope: StorageScope): Promise; diff --git a/src/lib/canvas-runtime/storage.ts b/src/lib/canvas-runtime/storage.ts index 458f9c4..2a27dd5 100644 --- a/src/lib/canvas-runtime/storage.ts +++ b/src/lib/canvas-runtime/storage.ts @@ -1,3 +1,4 @@ +import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut} from './asset-depot-storage.ts'; import { applyJournal, diffJournal, type JournalPatch } from './journal-delta.ts'; import { isSealedRecoveryImage } from './recovery-encoding.ts'; import type { Json } from '../canvas-core/types.ts'; @@ -45,9 +46,9 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: let closed = false; const handles = new Set(), lastCuts=new WeakMap(), writeTails=new WeakMap>(); const opening = new Promise((resolve, reject) => { - const request = factory.open(options.name ?? 'endoleap-headless', 2); + const request = factory.open(options.name ?? 'endoleap-headless', 3); request.onupgradeneeded = () => { - for (const name of ['metadata', 'checkpoints', 'journal', 'files']) if (!request.result.objectStoreNames.contains(name)) request.result.createObjectStore(name); + for (const name of ['metadata', 'checkpoints', 'journal', 'files',...depotStores]) if (!request.result.objectStoreNames.contains(name)) request.result.createObjectStore(name); }; request.onerror = () => reject(request.error); request.onblocked = () => reject(fault('storage_upgrade_blocked')); @@ -100,6 +101,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: } return Object.freeze({ fileDeletion:'atomic-v1' as const, + assetDepot:createDepotStorage((leader,stores,mode,run)=>execute(leader,stores,mode,tx=>run(tx))), async claim(scope: StorageScope) { const key = keyFor(scope); if (closed) throw fault('storage_closed'); @@ -128,6 +130,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: async recover(leader: StorageLeader) {const cut=await recoverCut(leader);return cut?clone(cut.image):null;}, async commit(leader: StorageLeader, batch: DurableBatch) { // Detach mutable callers and descriptors before the first await, including queued commits. + const depot=batch.depot?detachDepotCut(batch.depot):undefined; const image=isSealedRecoveryImage(batch.image)?batch.image:freeze(clone(batch.image)),files=(batch.files??[]).map(file=>({id:file.id,bytes:file.bytes})),deleteFiles=[...(batch.deleteFiles??[])];if(deleteFiles.length>128||new Set(deleteFiles).size!==deleteFiles.length||deleteFiles.some(id=>typeof id!=='string'||!id||files.some(file=>file.id===id)))throw fault('invalid_file_deletion');if(!isSealedRecoveryImage(image))validateImage(image); if(deleteFiles.length){ type FileProof={id?:string;version?:number;byteLength?:number;mimeType?:string;digest?:string;fileId?:string;retirement?:{version:number;unreferencedAt:number;purgedAt?:number}}; @@ -141,6 +144,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: } check(leader); const task=(writeTails.get(leader)??Promise.resolve()).catch(()=>{}).then(async()=>{ + if(depot)await verifyDepotBytes(depot); let base=lastCuts.get(leader)??await recoverCut(leader); if(base?.image.rootRevision===image.rootRevision)base=await recoverCut(leader); if(files.length&&base){const retired=purgedFileIds(base.image);if(files.some(file=>retired.has(file.id)))throw fault('resource_identity_conflict');} @@ -149,15 +153,16 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: const patches=diffJournal(base.image.payload,image.payload); if(patches){const delta:StoredDelta={journalVersion:1,baseRevision:base.image.rootRevision,baseChecksum:base.checksum,header:{formatVersion:image.formatVersion,rootRevision:image.rootRevision,localRevision:image.localRevision},patches,checksum:digest.checksum};if(new TextEncoder().encode(JSON.stringify(delta)).byteLength{ + const receipt=await execute(leader,['checkpoints','journal','files',...depotStores],'readwrite',async(transaction,metadata,key)=>{ const previous=metadata.throughRevision; const saved=previous===undefined?undefined:await requestValue(transaction.objectStore(metadata.checkpointRevision===previous?'checkpoints':'journal').get([key,previous])); if(previous!==undefined&&image.rootRevision!==previous+1){ if(image.rootRevision!==previous)throw fault('storage_revision_gap'); - if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');return{durableRootRevision:previous}; + if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot)await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);return{durableRootRevision:previous}; } if(previous!==undefined&&(!saved||image.localRevision `/api/foundry/${path}/${encodeURIComponent(id)}?variant=preview`, accept, + const authority=Object.freeze({scope:`${origin.origin}/projects/${encodeURIComponent(options.project??'endoleap')}/${path}`,async resolve(id:Id,signal:AbortSignal){if(!uuid.test(id))throw fault('invalid_resource_id');const response=await options.transport({method:'GET',path:`${path}/metadata/${encodeURIComponent(id)}`,signal}),raw=object(response.body);if(response.status!==200)throw fault(typeof raw.error==='string'?raw.error:'asset_authorization_failed');if(raw.metadata_version!==1)throw fault('asset_metadata_interpreter_unavailable');const value={version:1 as const,authority:authority.scope,digest:raw.sha256 as string,asset:raw.asset as StoredAsset};validateSharedAsset(value);if(value.asset.id!==id||value.asset.url!==sourceURL(id))throw fault('asset_authorization_mismatch');return freeze(value);},async read(id:Id,signal:AbortSignal){if(!uuid.test(id))throw fault('invalid_resource_id');const response=await options.transport({method:'GET',path:`${path}/${encodeURIComponent(id)}`,signal,responseFormat:'asset-bytes-v1'});if(response.status!==200||!response.bytes)throw fault('asset_bytes_unavailable');return response.bytes;}}); + return Object.freeze({ authority, sourceURL, displayURL: (id: Id) => `/api/foundry/${path}/${encodeURIComponent(id)}?variant=preview`, accept, upload: (request: FrozenRequest, bytes: Blob, signal: AbortSignal) => execute(request, bytes, signal), query: (request: FrozenRequest, signal: AbortSignal) => execute(request, undefined, signal), }); diff --git a/src/lib/canvas-service/foundry.ts b/src/lib/canvas-service/foundry.ts index 183a853..7df7d5c 100644 --- a/src/lib/canvas-service/foundry.ts +++ b/src/lib/canvas-service/foundry.ts @@ -7,8 +7,8 @@ import type { FrozenRequest } from '../canvas-runtime/ledger.ts'; import { projectBatch } from '../canvas-runtime/batch-projection.ts'; import type { OperationStatus } from '../canvas-runtime/scheduler.ts'; -export type ServiceRequest = { readonly responseFormat?:'model-stream-v1'; readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal }; -export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly stream?:AsyncIterable; readonly retryAfterMs?: number }; +export type ServiceRequest = { readonly responseFormat?:'model-stream-v1'|'asset-bytes-v1'; readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal }; +export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly bytes?:Blob; readonly stream?:AsyncIterable; readonly retryAfterMs?: number }; export type ServiceTransport = (request: ServiceRequest) => Promise; const uuid = /^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$/i; const record = (value: unknown): Record => { if (!value || Array.isArray(value) || typeof value !== 'object') throw fault('invalid_canonical_response'); return value as Record; }; @@ -29,9 +29,20 @@ export function createFoundryTransport(options: { accountId: Id; isCurrentIdenti if (!options.isCurrentIdentity()) throw fault('identity_changed'); if (input.expectedVersion !== undefined && (!Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1)) throw fault('invalid_server_version'); const response = await requestFetch(`${options.basePath ?? '/api/foundry/'}${input.path}`, { method: input.method, body: input.body, signal: input.signal, credentials: 'same-origin', cache: 'no-store', redirect: 'error', headers: { - Accept: input.responseFormat==='model-stream-v1'?modelStreamType:'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}), + Accept: input.responseFormat==='model-stream-v1'?modelStreamType:input.responseFormat==='asset-bytes-v1'?'image/*':'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}), } }); if (!options.isCurrentIdentity()) { await response.body?.cancel(); throw fault('identity_changed'); } + if(input.responseFormat==='asset-bytes-v1'&&response.status===200){ + const maximum=20*1024*1024,length=response.headers.get('content-length'); + if(length!==null&&(!/^\d+$/.test(length)||Number(length)>maximum)){await response.body?.cancel();throw fault('asset_byte_budget_exceeded');} + if(!response.body)throw fault('asset_bytes_unavailable'); + const reader=response.body.getReader(),chunks:Uint8Array[]= [];let size=0; + const check=()=>{input.signal?.throwIfAborted();if(!options.isCurrentIdentity())throw fault('identity_changed');}; + const abort=()=>{void reader.cancel().catch(()=>{});};input.signal?.addEventListener('abort',abort,{once:true}); + try{while(true){check();const next=await reader.read();check();if(next.done)break;size+=next.value.byteLength;if(size>maximum)throw fault('asset_byte_budget_exceeded');chunks.push(next.value);}} + finally{input.signal?.removeEventListener('abort',abort);await reader.cancel().catch(()=>{});reader.releaseLock();} + return{status:200,body:null,bytes:new Blob(chunks,{type:response.headers.get('content-type')?.split(';')[0].trim()??''})}; + } if(input.responseFormat==='model-stream-v1'&&response.status===200&&response.headers.get('content-type')?.split(';')[0].trim()==='application/vnd.foundry.canvas-model+ndjson'){ if(!/;\s*version=1(?:;|$)/.test(response.headers.get('content-type')??'')){await response.body?.cancel();throw fault('unsupported_model_stream');} if(!response.body)throw fault('incomplete_model_stream'); diff --git a/tests/headless/asset-depot-browser-cases.mjs b/tests/headless/asset-depot-browser-cases.mjs new file mode 100644 index 0000000..591d993 --- /dev/null +++ b/tests/headless/asset-depot-browser-cases.mjs @@ -0,0 +1,52 @@ +import {openProjectEngine} from '../../src/lib/canvas-runtime/engine.ts'; +import {createDocument,createRegistry,builtinComponents,applyChanges} from '../../src/lib/canvas-core/index.ts'; +import {createBrowserStorage} from '../../src/lib/canvas-runtime/storage.ts'; +export async function assetDepotStorageCase(assert){ + const name=`depot-${crypto.randomUUID()}`,storage=createBrowserStorage({name}),scope={accountId:'a',workspaceId:'w',projectId:'p'},leaders=[]; + const image=n=>({formatVersion:1,rootRevision:n,localRevision:n,payload:{n}}),bytes=new Blob(['shared original'],{type:'image/png'}),digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''),authority='https://source.test/w/assets'; + const ref=id=>({version:1,authority,digest,asset:{id,url:`https://source.test/w/assets/${id}`,width:2,height:1,bytes:bytes.size,mime_type:'image/png'}}),cut=(pins=[],writes=[],complete=true)=>({version:1,authority,pins,writes,complete}),collect=(leader,now,extra={})=>storage.assetDepot.collect(leader,{authority,now,retentionMs:10,limit:128,...extra}); + const rejects=async(fn,code)=>{try{await fn();}catch(e){assert(e.code===code,`expected ${code}, got ${e.code}`);return;}throw Error(`expected ${code}`);}; + const open=async(projectId)=>{const leader=await storage.claim({...scope,projectId});leaders.push(leader);return leader;}; + try{ + const p=await open('p'),q=await open('q'); + await storage.commit(p,{image:image(0),depot:cut([['one',['element:p']]], [{ref:ref('one'),bytes},{ref:ref('two'),bytes}])}); + await storage.commit(q,{image:image(0),depot:cut([['two',['history:q']]])}); + const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const count=()=>new Promise((resolve,reject)=>{const tx=database.transaction('asset-bytes'),request=tx.objectStore('asset-bytes').count();request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + assert(await count()===1,'same digest was physically duplicated across identities'); + await p.release();assert((await collect(q,0)).pinnedIds.length===2,'closed project pins were lost'); + assert(await(await storage.assetDepot.read(q,authority,'one')).bytes.text()==='shared original','destination could not read shared bytes'); + const otherAccount=await storage.claim({...scope,accountId:'other'});leaders.push(otherAccount);assert(await storage.assetDepot.read(otherAccount,authority,'one')===undefined,'account cache leak'); + const otherWorkspace=await storage.claim({...scope,workspaceId:'other'});leaders.push(otherWorkspace);assert(await storage.assetDepot.read(otherWorkspace,authority,'one')===undefined,'workspace cache leak'); + assert(await storage.assetDepot.read(q,'https://other.test','one')===undefined,'authority cache leak'); + const stale=await open('stale');await storage.commit(stale,{image:image(0)});assert((await collect(q,100)).blockedProjects.includes('stale'),'unindexed project allowed deletion'); + await storage.commit(stale,{image:image(0),depot:cut([],[],false)});assert(!(await collect(q,100)).complete,'opaque project allowed deletion'); + await storage.commit(stale,{image:image(0),depot:cut()});await storage.commit(stale,{image:image(1)});assert(!(await collect(q,100)).complete,'stale manifest allowed deletion'); + await storage.commit(stale,{image:image(1),depot:cut()}); + const reopened=await open('p');await storage.commit(reopened,{image:image(1),depot:cut()}); + assert((await collect(q,100)).markedIds.includes('one'),'unreferenced resource was not marked'); + assert((await collect(q,110)).purgedIds.includes('one')&&await count()===1,'purging one identity deleted another identity bytes'); + await storage.commit(q,{image:image(1),depot:cut()});await collect(q,110);await collect(q,120);assert(await count()===0,'expired unpinned shared bytes retained'); + const purged=await storage.assetDepot.read(q,authority,'one');assert(purged.purgedAt===110&&!purged.bytes,'immutable identity lost or purged bytes exposed'); + await rejects(()=>storage.commit(q,{image:image(2),depot:cut([], [{ref:{...ref('one'),digest:'0'.repeat(64)},bytes}])}),'shared_asset_bytes_changed');assert((await storage.recover(q)).rootRevision===1,'invalid bytes advanced Root'); + await rejects(()=>storage.commit(q,{image:image(2),depot:cut([['one',['element:q']]], [{ref:{...ref('one'),asset:{...ref('one').asset,width:3}},bytes}])}),'shared_asset_identity_conflict');assert(await count()===0&&(await storage.recover(q)).rootRevision===1,'identity conflict partially committed bytes or Root'); + // Force a later files-store conflict after the depot puts: the whole transaction must roll back. + await storage.commit(q,{image:image(2),files:[{id:'existing',bytes}],depot:cut()}); + await rejects(()=>storage.commit(q,{image:image(3),files:[{id:'existing',bytes}],depot:cut([['one',['element:q']]], [{ref:ref('one'),bytes}])}),'resource_identity_conflict');assert(await count()===0&&(await storage.recover(q)).rootRevision===2,'late failure committed depot or Root'); + await storage.commit(q,{image:image(3),depot:cut([['one',['element:q']]], [{ref:ref('one'),bytes}])});assert(await count()===1&&(await storage.assetDepot.read(q,authority,'one')).purgedAt===undefined,'authorized rehydration failed'); + // A one-item sweep must provide a cursor and preserve each resource decision. + const first=await collect(q,130,{limit:1});assert(first.nextAfter==='one','bounded sweep cursor missing');assert((await collect(q,130,{after:first.nextAfter,limit:1})).nextAfter===undefined,'bounded sweep did not finish'); + // Exercise the actual engine persistence entry point, rather than only hand-built batches. + const projectId='engine-project',registry=createRegistry(builtinComponents());let document=createDocument(projectId,[{id:'canvas',parent:null}],[]),version=1,counter=0,resource,uploads=0; + const receipts=new Map(),assets={authority:{scope:authority,async resolve(){throw Error('unexpected adoption');},async read(){throw Error('unexpected hydration');}},sourceURL:id=>`https://source.test/w/assets/${id}`,displayURL:id=>`/assets/${id}`,async upload(request,data){uploads++;const raw=JSON.parse(request.body);return{outcome:'applied',canonical:{id:raw.resource_id,url:assets.sourceURL(raw.resource_id),width:2,height:1,bytes:data.size,mime_type:data.type}};},async query(){return{outcome:'unresolved'};},accept:(_request,status)=>status.canonical}; + const persistence={compile(intent,id,snapshot){return{payload:{intent},canvasIds:intent.canvasIds,expectedVersions:{canvas:snapshot.runtime.versions.get('canvas')}};},async submit(request){document=applyChanges(document,JSON.parse(request.body).intent.changes);receipts.set(request.id,{document,versions:new Map([['canvas',++version]])});return{outcome:'applied'};},async query(){return{outcome:'unresolved'};},canonical:request=>receipts.get(request.id)}; + const engine=await openProjectEngine({scope:{...scope,projectId},registry,document,versions:new Map([['canvas',1]]),storage,assets,persistence,createId:()=>`engine-${++counter}`}); + try{ + const staged=engine.stageFiles([{name:'source.png',bytes}],([handle],tx)=>{resource=handle.id;tx.dispatch({type:'element.create',element:{id:'picture',incarnation:1,structure:'node',componentId:'image',dataVersion:1,ownerCanvasId:'canvas',geometry:{x:0,y:0,width:2,height:1,rotation:0},data:{title:'picture',text:'',assetId:handle.id,src:handle.src}}});});assert(staged.status!=='rejected','engine image staging failed'); + let ready=false;for(let i=0;i<200;i++){if(engine.getSnapshot().runtime.resources.get(resource)?.stored){ready=true;break;}await new Promise(resolve=>setTimeout(resolve,5));}assert(ready,'engine upload did not settle'); + }finally{await engine.close();} + const promoted=await storage.assetDepot.read(q,authority,resource);assert(promoted?.bytes&&await promoted.bytes.text()==='shared original'&&uploads===1,'engine did not promote acknowledged upload bytes'); + assert((await collect(q,200)).pinnedIds.includes(resource),'closed engine did not retain durable document/history pins'); + await q.release();await rejects(()=>collect(q,200),'storage_leader_lost');database.close(); + }finally{for(const leader of leaders)await leader.release();storage.close();indexedDB.deleteDatabase(name);} +} diff --git a/tests/headless/asset-depot.test.mjs b/tests/headless/asset-depot.test.mjs new file mode 100644 index 0000000..23eaf0f --- /dev/null +++ b/tests/headless/asset-depot.test.mjs @@ -0,0 +1,41 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {createAssetDepot} from '../../src/lib/canvas-runtime/asset-depot.ts'; +import {createFoundryTransport} from '../../src/lib/canvas-service/foundry.ts'; +import {createFoundryAssets} from '../../src/lib/canvas-service/assets.ts'; +const digest=async bytes=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''); +async function fixture(){ + const bytes=new Blob(['original'],{type:'image/png'}),scope={accountId:'account',workspaceId:'workspace'},leader={scope:{...scope,projectId:'p'},active:true},ref={version:1,authority:'authority',digest:await digest(bytes),asset:{id:'asset',url:'https://source.test/asset',width:2,height:1,bytes:bytes.size,mime_type:'image/png'}}; + let cached,allowed=true,resolves=0,reads=0,current=true; + const depot=createAssetDepot({scope,port:{async read(){return cached;},async collect(){return{complete:true};}},current:()=>current,digest}); + const authority={scope:ref.authority,async resolve(){resolves++;if(!allowed)throw Error('denied');return ref;},async read(){reads++;return bytes;}}; + return{depot,scope,leader,ref,bytes,authority,setCached:v=>cached=v,deny:()=>allowed=false,retire:()=>current=false,counts:()=>({resolves,reads})}; +} +test('new cross-project adoption reauthorizes even when immutable bytes are warm',async()=>{ + const f=await fixture(),signal=new AbortController().signal; + assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).bytes,f.bytes); + f.setCached({ref:f.ref,bytes:f.bytes});assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).bytes,f.bytes);assert.deepEqual(f.counts(),{resolves:2,reads:1}); + f.deny();await assert.rejects(f.depot.authorize(f.leader,f.authority,'asset',signal),/denied/);assert.deepEqual(f.counts(),{resolves:3,reads:1});f.depot.close(); +}); +test('depot captures scope, validates cached identity and bytes, and rejects revoked lifecycle',async()=>{ + const f=await fixture(),signal=new AbortController().signal;f.scope.accountId='mutated';assert.equal((await f.depot.authorize(f.leader,f.authority,'asset',signal)).ref.asset.id,'asset'); + f.setCached({ref:{...f.ref,digest:'0'.repeat(64)},bytes:f.bytes});await assert.rejects(f.depot.authorize(f.leader,f.authority,'asset',signal),{code:'shared_asset_identity_conflict'}); + f.setCached({ref:f.ref,bytes:new Blob(['tampered'])});await assert.rejects(f.depot.read(f.leader,f.ref),{code:'shared_asset_bytes_changed'}); + f.retire();await assert.rejects(f.depot.read(f.leader,f.ref),{code:'identity_changed'});f.depot.close(); +}); +test('late authority result after close never enters the metadata cache',async()=>{ + const f=await fixture();let done,started;const ready=new Promise(resolve=>started=resolve),wait=new Promise(resolve=>done=resolve); + const promise=f.depot.authorize(f.leader,{...f.authority,async resolve(){started();await wait;return f.ref;}},'asset',new AbortController().signal);await ready;f.depot.close();done();await assert.rejects(promise,{code:'identity_changed'});assert.equal(f.depot.getResources().size,0); +}); +test('binary asset transport preserves original bytes and enforces identity and actual byte budgets',async()=>{ + const accountId='11111111-1111-4111-8111-111111111111';let current=true,cancelled=false; + const transport=response=>createFoundryTransport({accountId,isCurrentIdentity:()=>current,fetch:async()=>response}); + const result=await transport(new Response('original',{headers:{'content-type':'image/png'}}))({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'});assert.equal(await result.bytes.text(),'original');assert.equal(result.bytes.type,'image/png'); + const oversized=new Response(new ReadableStream({start(c){c.enqueue(new Uint8Array(20*1024*1024+1));},cancel(){cancelled=true;}}));await assert.rejects(transport(oversized)({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'}),{code:'asset_byte_budget_exceeded'});assert.equal(cancelled,true); + const identityResponse=new Response(new ReadableStream({pull(c){current=false;c.enqueue(new Uint8Array([1]));}}));await assert.rejects(transport(identityResponse)({method:'GET',path:'asset',responseFormat:'asset-bytes-v1'}),{code:'identity_changed'}); +}); +test('Foundry resource authority validates version, resource identity and canonical URL',async()=>{ + const id='22222222-2222-4222-8222-222222222222',workspaceId='33333333-3333-4333-8333-333333333333';let version=1;const bytes=new Blob(['original']),sha256=await digest(bytes),calls=[]; + const assets=createFoundryAssets({workspaceId,transport:async request=>{calls.push(request);return request.responseFormat==='asset-bytes-v1'?{status:200,body:null,bytes}:{status:200,body:{metadata_version:version,sha256,asset:{id,url:assets.sourceURL(id),width:2,height:1,bytes:bytes.size,mime_type:'image/png'}}};}}); + const ref=await assets.authority.resolve(id,new AbortController().signal);assert.equal(ref.digest,sha256);assert.equal(await assets.authority.read(id,new AbortController().signal),bytes);assert.equal(calls[0].path.includes('/metadata/'),true);assert.equal(calls[1].responseFormat,'asset-bytes-v1');version=2;await assert.rejects(assets.authority.resolve(id,new AbortController().signal),{code:'asset_metadata_interpreter_unavailable'}); +}); diff --git a/tests/headless/browser-storage-cases.mjs b/tests/headless/browser-storage-cases.mjs index 2146a95..9c16d7d 100644 --- a/tests/headless/browser-storage-cases.mjs +++ b/tests/headless/browser-storage-cases.mjs @@ -1,3 +1,4 @@ +import {assetDepotStorageCase} from './asset-depot-browser-cases.mjs'; import {createBrowserImageDecoder} from '../../src/lib/canvas-runtime/browser-images.ts'; import {decodeRecovery} from '../../src/lib/canvas-runtime/recovery.ts'; import {createBrowserDigests} from '../../src/lib/canvas-runtime/browser-digests.ts'; @@ -711,7 +712,7 @@ async function loadingRecoveryCase(name,assert,coverage=false,scoped=false,struc assert(injected&&failure==='loading_full_cut_abort','completion did not reach forced checkpoint rollback');const partial=await adapter.recover(ownerForTest);assert(partial.rootRevision===before&&partial.formatVersion===format&&calls.length===0,'aborted full checkpoint released loading writes or lost the partial cut'); await engine.persist();await engine.settle(); assert(!engine.loading.getState()&&server.elements.get('A').data.text==='local before tail'&&engine.getSnapshot().document.elements.size===2,'full tail overwrote the local prefix');await engine.close();engine=await open();await engine.settle();assert(!engine.loading.getState()&&engine.getSnapshot().document.elements.size===2,'completed recovery reopened as partial'); - const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(`${name}-loading`,2);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const database=await new Promise((resolve,reject)=>{const request=indexedDB.open(`${name}-loading`);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); try{const transaction=database.transaction(['checkpoints','journal'],'readonly'),read=store=>new Promise((resolve,reject)=>{const request=transaction.objectStore(store).getAll();request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const cuts=(await Promise.all([read('checkpoints'),read('journal')])).flat();assert(cuts.every(cut=>(cut.image?.formatVersion??cut.header?.formatVersion)<7),'completed storage still requires a partial journal base');}finally{database.close();} }finally{await engine?.close();storage.close();indexedDB.deleteDatabase(`${name}-loading`);} } @@ -737,6 +738,7 @@ export async function run() { const rejects = async (callback, message) => { try { await callback(); } catch (error) { assert(error.message === message, `expected ${message}, received ${error.message}`); return; } throw new Error(`expected ${message}`); }; let leader, other; try { + await assetDepotStorageCase(assert);results.push('workspace depot atomically shares digest bytes, preserves closed-project pins, blocks unknown/stale coverage and rolls back failed Root cuts'); leader = await storage.claim(scope); assert(await storage.recover(leader) === null, 'empty scope must be distinguishable from a corrupt checkpoint'); await rejects(() => storage.claim(scope), 'storage_scope_in_use'); diff --git a/tests/headless/proxy.integration.mjs b/tests/headless/proxy.integration.mjs index 6d7b71a..b23b8e8 100644 --- a/tests/headless/proxy.integration.mjs +++ b/tests/headless/proxy.integration.mjs @@ -47,6 +47,8 @@ try{ assert.ok(request);assert.equal(request.body.toString(),body);assert.equal(request.headers['idempotency-key'],key);assert.equal(request.headers['x-expected-account'],account);assert.equal(request.headers.authorization,'Bearer fixture-session'); const receiptPath=`workspaces/10000000-0000-4000-8000-000000000003/assets/operations/${key}`; const assetReceipt=await fetch(`http://127.0.0.1:${port}/api/foundry/${receiptPath}`);assert.equal(assetReceipt.status,200);assert.deepEqual(await assetReceipt.json(),{id:'fixture'}); + const metadata=await fetch(`http://127.0.0.1:${port}/api/foundry/${receiptPath.replace('/operations/','/metadata/')}`,{headers:{'X-Expected-Account':account,Cookie:'foundry_session=fixture-session'}});assert.equal(metadata.status,200);assert.deepEqual(await metadata.json(),{id:'fixture'});assert.match(metadata.headers.get('cache-control'),/no-store/);const metadataRequest=received.find(item=>item.path.includes('/assets/metadata/'));assert.equal(metadataRequest.headers['x-expected-account'],account);assert.equal(metadataRequest.headers.authorization,'Bearer fixture-session'); + console.log('PASS asset metadata JSON uses the actual authenticated Next proxy without image-body validation or caching'); const multipart=new FormData();multipart.append('image',new Blob(['immutable file'],{type:'image/png'}),'fixture.png'); const assetPath=receiptPath.split('/operations/')[0],resourceId='10000000-0000-4000-8000-000000000006'; const uploaded=await fetch(`http://127.0.0.1:${port}/api/foundry/${assetPath}`,{method:'POST',body:multipart,headers:{'X-Resource-Id':resourceId,'Idempotency-Key':key,'X-Expected-Account':account}});await uploaded.text();assert.equal(uploaded.status,200);