92 lines
12 KiB
TypeScript
92 lines
12 KiB
TypeScript
import {equal,fault,freeze} from '../canvas-core/values.ts';
|
|
import {sharedAssetRef,validateSharedAsset,type AssetDepotPort,type DepotCut,type DepotEntry,type SharedAsset} from './asset-depot.ts';
|
|
import type {StorageLeader,StorageScope,RecoveryImage} from './storage-port.ts';
|
|
|
|
export const depotStores=['asset-depot','asset-bytes','asset-pins'];
|
|
type Fact={ref:SharedAsset;unreferencedAt?:number;purgedAt?:number};
|
|
type Pins={version:1;rootRevision:number;complete:boolean;pins:readonly (readonly [string,readonly string[]])[]};
|
|
const value=<T>(request:IDBRequest<T>)=>new Promise<T>((resolve,reject)=>{request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
|
|
const prefix=(scope:StorageScope,authority:string)=>[scope.accountId,scope.workspaceId,authority];
|
|
const range=(scope:StorageScope,authority:string)=>IDBKeyRange.bound(prefix(scope,authority),[...prefix(scope,authority),[]],false,true);
|
|
function authority(value:string){if(typeof value!=='string'||!value||value.length>8192)throw fault('invalid_depot_cut');}
|
|
function validateFact(row:Fact,scopeAuthority:string,id:string){validateSharedAsset(row?.ref);if(row.ref.authority!==scopeAuthority||row.ref.asset.id!==id)throw fault('corrupt_asset_depot');for(const n of [row.unreferencedAt,row.purgedAt])if(n!==undefined&&(!Number.isSafeInteger(n)||n<0))throw fault('corrupt_asset_depot');if(row.purgedAt!==undefined&&(row.unreferencedAt===undefined||row.purgedAt<row.unreferencedAt))throw fault('corrupt_asset_depot');}
|
|
/** Detach descriptors before awaiting crypto. Blob data is immutable and never JSON cloned. */
|
|
export function detachDepotCut(cut:DepotCut):DepotCut{
|
|
if(!cut||cut.version!==1||typeof cut.complete!=='boolean'||!Array.isArray(cut.pins)||!Array.isArray(cut.writes))throw fault('invalid_depot_cut');authority(cut.authority);
|
|
const seen=new Set<string>(),pins=cut.pins.map(row=>{if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||seen.has(row[0])||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner)||new Set(row[1]).size!==row[1].length)throw fault('invalid_depot_cut');seen.add(row[0]);return Object.freeze([row[0],Object.freeze([...row[1]])] as const);});
|
|
const written=new Set<string>(),writes=cut.writes.map(row=>{validateSharedAsset(row.ref);if(row.ref.authority!==cut.authority||written.has(row.ref.asset.id)||row.bytes!==undefined&&!(row.bytes instanceof Blob))throw fault('invalid_depot_cut');written.add(row.ref.asset.id);return Object.freeze({ref:sharedAssetRef(row.ref),...(row.bytes?{bytes:row.bytes}:{})});});
|
|
return Object.freeze({version:1,authority:cut.authority,complete:cut.complete,pins:Object.freeze(pins),writes:Object.freeze(writes)});
|
|
}
|
|
export async function verifyDepotBytes(cut:DepotCut){
|
|
for(const row of cut.writes)if(row.bytes){if(row.bytes.size!==row.ref.asset.bytes)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);const digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await row.bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join('');if(digest!==row.ref.digest)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);}
|
|
}
|
|
/** Must run in the same transaction as the project Root, including idempotent reindexing. */
|
|
export async function writeDepotCut(tx:IDBTransaction,scope:StorageScope,rootRevision:number,cut:DepotCut){
|
|
const base=prefix(scope,cut.authority),facts=tx.objectStore('asset-depot'),bytes=tx.objectStore('asset-bytes');
|
|
for(const row of cut.writes){const key=[...base,row.ref.asset.id],previous=await value<Fact|undefined>(facts.get(key));if(previous){validateFact(previous,cut.authority,row.ref.asset.id);if(!equal(previous.ref,row.ref))throw fault('shared_asset_identity_conflict',[row.ref.asset.id]);}
|
|
// Digest-addressed bytes are shared across resource IDs and projects in this authority.
|
|
if(row.bytes)bytes.put(row.bytes,[...base,row.ref.digest]);
|
|
facts.put({...previous,ref:row.ref,...(row.bytes?{unreferencedAt:undefined,purgedAt:undefined}:{})},key);
|
|
}
|
|
tx.objectStore('asset-pins').put({version:1,rootRevision,complete:cut.complete,pins:cut.pins} satisfies Pins,[...base,scope.projectId]);
|
|
}
|
|
/** DB v4 file rows may refer to immutable shared bytes after a proven upload ACK. */
|
|
export type DepotFile={readonly fileVersion:1;readonly kind:'asset-depot';readonly authority:string;readonly resourceId:string;readonly digest:string;readonly byteLength:number;readonly mimeType:string};
|
|
function depotFile(row:unknown,id:string):DepotFile{
|
|
const file=row as DepotFile;
|
|
if(!file||file.fileVersion!==1)throw fault('file_interpreter_unavailable');
|
|
authority(file.authority);
|
|
if(file.kind!=='asset-depot'||file.resourceId!==id||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest)||!Number.isSafeInteger(file.byteLength)||file.byteLength<1||file.byteLength>20*1024*1024||typeof file.mimeType!=='string')throw fault('corrupt_file_store');
|
|
return file;
|
|
}
|
|
/** Undefined means bytes were explicitly evicted; identity metadata remains. */
|
|
export async function resolveDepotFile(tx:IDBTransaction,scope:StorageScope,id:string,row:unknown):Promise<Blob|undefined>{
|
|
if(row===undefined||row instanceof Blob)return row;
|
|
const file=depotFile(row,id),base=prefix(scope,file.authority),fact=await value<Fact|undefined>(tx.objectStore('asset-depot').get([...base,id]));
|
|
if(!fact)throw fault('corrupt_file_store');validateFact(fact,file.authority,id);
|
|
if(fact.ref.digest!==file.digest||fact.ref.asset.bytes!==file.byteLength)throw fault('corrupt_file_store');
|
|
if(fact.purgedAt!==undefined)return;
|
|
const bytes=await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,file.digest]));
|
|
if(bytes===undefined)return;
|
|
if(!(bytes instanceof Blob)||bytes.size!==file.byteLength)throw fault('corrupt_file_store');
|
|
// Preserve the original local File MIME even if the server detected another MIME.
|
|
return bytes.type===file.mimeType?bytes:bytes.slice(0,bytes.size,file.mimeType);
|
|
}
|
|
/** Convert only Root-proven stored uploads, after shared bytes and local files are written. */
|
|
export async function promoteDepotFiles(tx:IDBTransaction,scope:StorageScope,key:string,image:RecoveryImage,cut:DepotCut){
|
|
type Resource={id:string;digest?:string;byteLength:number;mimeType:string;stored?:SharedAsset['asset'];retirement?:{purgedAt?:number}};
|
|
const payload=image.payload as {runtime?:{resources?:readonly (readonly [string,Resource])[]}},entries=payload?.runtime?.resources;
|
|
if(entries===undefined)return;if(!Array.isArray(entries))throw fault('corrupt_recovery_image');
|
|
const resources=new Map<string,Resource>(entries),files=tx.objectStore('files'),base=prefix(scope,cut.authority);
|
|
for(const write of cut.writes){const id=write.ref.asset.id,resource=resources.get(id);if(!resource?.stored||resource.retirement?.purgedAt!==undefined)continue;
|
|
if(resource.id!==id||resource.digest!==write.ref.digest||resource.byteLength!==write.ref.asset.bytes||typeof resource.mimeType!=='string'||!equal(resource.stored,write.ref.asset))throw fault('shared_asset_identity_conflict',[id]);
|
|
const local=await value<unknown>(files.get([key,id]));if(local===undefined)continue;
|
|
if(!(local instanceof Blob)){const pointer=depotFile(local,id);if(pointer.authority!==cut.authority||pointer.digest!==write.ref.digest||pointer.byteLength!==resource.byteLength||pointer.mimeType!==resource.mimeType)throw fault('shared_asset_identity_conflict',[id]);continue;}
|
|
const bytes=await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,write.ref.digest]));
|
|
if(!bytes)continue;
|
|
if(!(bytes instanceof Blob)||bytes.size!==resource.byteLength||local.size!==resource.byteLength||local.type!==resource.mimeType)throw fault('shared_asset_bytes_changed',[id]);
|
|
files.put({fileVersion:1,kind:'asset-depot',authority:cut.authority,resourceId:id,digest:write.ref.digest,byteLength:resource.byteLength,mimeType:resource.mimeType} satisfies DepotFile,[key,id]);
|
|
}
|
|
}
|
|
type Execute=<T>(leader:StorageLeader,stores:string[],mode:IDBTransactionMode,run:(tx:IDBTransaction)=>Promise<T>)=>Promise<T>;
|
|
export function createDepotStorage(execute:Execute):AssetDepotPort{return Object.freeze({
|
|
async read(leader,scopeAuthority,id){authority(scopeAuthority);if(typeof id!=='string'||!id)throw fault('invalid_resource_id');return execute(leader,depotStores,'readonly',async tx=>{const base=prefix(leader.scope,scopeAuthority),row=await value<Fact|undefined>(tx.objectStore('asset-depot').get([...base,id]));if(!row)return;validateFact(row,scopeAuthority,id);const bytes=row.purgedAt===undefined?await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,row.ref.digest])):undefined;if(bytes!==undefined&&(!(bytes instanceof Blob)||bytes.size!==row.ref.asset.bytes))throw fault('corrupt_asset_depot');return freeze({...row,...(bytes?{bytes}:{})}) as DepotEntry;});},
|
|
async collect(leader,input){authority(input.authority);const {now,retentionMs,limit,after}=input;if(!Number.isSafeInteger(now)||now<0||!Number.isSafeInteger(retentionMs)||retentionMs<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))throw fault('invalid_depot_collection');
|
|
return execute(leader,depotStores,'readwrite',async tx=>{
|
|
const scope=leader.scope,metadata=tx.objectStore('metadata'),keys=await value(metadata.getAllKeys()),heads=await value<{throughRevision?:number}[]>(metadata.getAll()),base=prefix(scope,input.authority),manifests=tx.objectStore('asset-pins'),blockedProjects:string[]=[],pinned=new Set<string>();
|
|
for(let i=0;i<keys.length;i++){let parts:unknown;try{parts=JSON.parse(String(keys[i]));}catch{throw fault('corrupt_storage_scope');}if(!Array.isArray(parts)||parts.length!==3||parts.some(p=>typeof p!=='string'||!p))throw fault('corrupt_storage_scope');if(parts[0]!==scope.accountId||parts[1]!==scope.workspaceId||heads[i].throughRevision===undefined)continue;
|
|
const pins=await value<Pins|undefined>(manifests.get([...base,parts[2]]));if(!pins||pins.version!==1||pins.rootRevision!==heads[i].throughRevision||pins.complete!==true){blockedProjects.push(parts[2]);continue;}
|
|
if(!Array.isArray(pins.pins))throw fault('corrupt_depot_pins');for(const row of pins.pins){if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner))throw fault('corrupt_depot_pins');pinned.add(row[0]);}
|
|
}
|
|
if(blockedProjects.length)return freeze({complete:false,blockedProjects,markedIds:[],purgedIds:[],pinnedIds:[]});
|
|
const store=tx.objectStore('asset-depot'),factKeys=await value(store.getAllKeys(range(scope,input.authority))),rows=await value<Fact[]>(store.getAll(range(scope,input.authority))),markedIds:string[]=[],purgedIds:string[]=[],pinnedIds:string[]=[];
|
|
for(let i=0;i<rows.length;i++){const key=factKeys[i];if(!Array.isArray(key)||key.length!==4||typeof key[3]!=='string')throw fault('corrupt_asset_depot');validateFact(rows[i],input.authority,key[3]);}
|
|
const candidates=rows.map((row,i)=>({row,key:factKeys[i] as string[]})).filter(({key})=>after===undefined||key[3]>after).slice(0,limit+1),page=candidates.slice(0,limit);
|
|
for(const {row,key}of page){const id=key[3];if(pinned.has(id)){pinnedIds.push(id);if(row.purgedAt===undefined&&row.unreferencedAt!==undefined){delete row.unreferencedAt;store.put(row,key);}continue;}if(row.purgedAt!==undefined)continue;if(row.unreferencedAt===undefined){row.unreferencedAt=now;markedIds.push(id);store.put(row,key);}else if(now>=row.unreferencedAt&&now-row.unreferencedAt>=retentionMs){row.purgedAt=now;purgedIds.push(id);store.put(row,key);}}
|
|
// Preserve a digest while any other live resource identity still owns its bytes.
|
|
for(const digest of new Set(page.filter(({row})=>row.purgedAt===now).map(({row})=>row.ref.digest)))if(!rows.some(row=>row.ref.digest===digest&&row.purgedAt===undefined))tx.objectStore('asset-bytes').delete([...base,digest]);
|
|
return freeze({complete:true,blockedProjects:[],markedIds,purgedIds,pinnedIds,...(candidates.length>limit?{nextAfter:page.at(-1)!.key[3]}:{})});
|
|
});
|
|
},
|
|
});}
|