Persist shared asset bytes and project pins with Root commits
This commit is contained in:
@@ -0,0 +1,53 @@
|
||||
import {equal,fault,freeze} from '../canvas-core/values.ts';
|
||||
import {sharedAssetRef,validateSharedAsset,type AssetDepotPort,type DepotCut,type DepotEntry,type SharedAsset} from './asset-depot.ts';
|
||||
import type {StorageLeader,StorageScope} from './storage-port.ts';
|
||||
|
||||
export const depotStores=['asset-depot','asset-bytes','asset-pins'];
|
||||
type Fact={ref:SharedAsset;unreferencedAt?:number;purgedAt?:number};
|
||||
type Pins={version:1;rootRevision:number;complete:boolean;pins:readonly (readonly [string,readonly string[]])[]};
|
||||
const value=<T>(request:IDBRequest<T>)=>new Promise<T>((resolve,reject)=>{request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
|
||||
const prefix=(scope:StorageScope,authority:string)=>[scope.accountId,scope.workspaceId,authority];
|
||||
const range=(scope:StorageScope,authority:string)=>IDBKeyRange.bound(prefix(scope,authority),[...prefix(scope,authority),[]],false,true);
|
||||
function authority(value:string){if(typeof value!=='string'||!value||value.length>8192)throw fault('invalid_depot_cut');}
|
||||
function validateFact(row:Fact,scopeAuthority:string,id:string){validateSharedAsset(row?.ref);if(row.ref.authority!==scopeAuthority||row.ref.asset.id!==id)throw fault('corrupt_asset_depot');for(const n of [row.unreferencedAt,row.purgedAt])if(n!==undefined&&(!Number.isSafeInteger(n)||n<0))throw fault('corrupt_asset_depot');if(row.purgedAt!==undefined&&(row.unreferencedAt===undefined||row.purgedAt<row.unreferencedAt))throw fault('corrupt_asset_depot');}
|
||||
/** Detach descriptors before awaiting crypto. Blob data is immutable and never JSON cloned. */
|
||||
export function detachDepotCut(cut:DepotCut):DepotCut{
|
||||
if(!cut||cut.version!==1||typeof cut.complete!=='boolean'||!Array.isArray(cut.pins)||!Array.isArray(cut.writes))throw fault('invalid_depot_cut');authority(cut.authority);
|
||||
const seen=new Set<string>(),pins=cut.pins.map(row=>{if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||seen.has(row[0])||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner)||new Set(row[1]).size!==row[1].length)throw fault('invalid_depot_cut');seen.add(row[0]);return Object.freeze([row[0],Object.freeze([...row[1]])] as const);});
|
||||
const written=new Set<string>(),writes=cut.writes.map(row=>{validateSharedAsset(row.ref);if(row.ref.authority!==cut.authority||written.has(row.ref.asset.id)||row.bytes!==undefined&&!(row.bytes instanceof Blob))throw fault('invalid_depot_cut');written.add(row.ref.asset.id);return Object.freeze({ref:sharedAssetRef(row.ref),...(row.bytes?{bytes:row.bytes}:{})});});
|
||||
return Object.freeze({version:1,authority:cut.authority,complete:cut.complete,pins:Object.freeze(pins),writes:Object.freeze(writes)});
|
||||
}
|
||||
export async function verifyDepotBytes(cut:DepotCut){
|
||||
for(const row of cut.writes)if(row.bytes){if(row.bytes.size!==row.ref.asset.bytes)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);const digest=[...new Uint8Array(await crypto.subtle.digest('SHA-256',await row.bytes.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join('');if(digest!==row.ref.digest)throw fault('shared_asset_bytes_changed',[row.ref.asset.id]);}
|
||||
}
|
||||
/** Must run in the same transaction as the project Root, including idempotent reindexing. */
|
||||
export async function writeDepotCut(tx:IDBTransaction,scope:StorageScope,rootRevision:number,cut:DepotCut){
|
||||
const base=prefix(scope,cut.authority),facts=tx.objectStore('asset-depot'),bytes=tx.objectStore('asset-bytes');
|
||||
for(const row of cut.writes){const key=[...base,row.ref.asset.id],previous=await value<Fact|undefined>(facts.get(key));if(previous){validateFact(previous,cut.authority,row.ref.asset.id);if(!equal(previous.ref,row.ref))throw fault('shared_asset_identity_conflict',[row.ref.asset.id]);}
|
||||
// Digest-addressed bytes are shared across resource IDs and projects in this authority.
|
||||
if(row.bytes)bytes.put(row.bytes,[...base,row.ref.digest]);
|
||||
facts.put({...previous,ref:row.ref,...(row.bytes?{unreferencedAt:undefined,purgedAt:undefined}:{})},key);
|
||||
}
|
||||
tx.objectStore('asset-pins').put({version:1,rootRevision,complete:cut.complete,pins:cut.pins} satisfies Pins,[...base,scope.projectId]);
|
||||
}
|
||||
type Execute=<T>(leader:StorageLeader,stores:string[],mode:IDBTransactionMode,run:(tx:IDBTransaction)=>Promise<T>)=>Promise<T>;
|
||||
export function createDepotStorage(execute:Execute):AssetDepotPort{return Object.freeze({
|
||||
async read(leader,scopeAuthority,id){authority(scopeAuthority);if(typeof id!=='string'||!id)throw fault('invalid_resource_id');return execute(leader,depotStores,'readonly',async tx=>{const base=prefix(leader.scope,scopeAuthority),row=await value<Fact|undefined>(tx.objectStore('asset-depot').get([...base,id]));if(!row)return;validateFact(row,scopeAuthority,id);const bytes=row.purgedAt===undefined?await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,row.ref.digest])):undefined;if(bytes!==undefined&&(!(bytes instanceof Blob)||bytes.size!==row.ref.asset.bytes))throw fault('corrupt_asset_depot');return freeze({...row,...(bytes?{bytes}:{})}) as DepotEntry;});},
|
||||
async collect(leader,input){authority(input.authority);const {now,retentionMs,limit,after}=input;if(!Number.isSafeInteger(now)||now<0||!Number.isSafeInteger(retentionMs)||retentionMs<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))throw fault('invalid_depot_collection');
|
||||
return execute(leader,depotStores,'readwrite',async tx=>{
|
||||
const scope=leader.scope,metadata=tx.objectStore('metadata'),keys=await value(metadata.getAllKeys()),heads=await value<{throughRevision?:number}[]>(metadata.getAll()),base=prefix(scope,input.authority),manifests=tx.objectStore('asset-pins'),blockedProjects:string[]=[],pinned=new Set<string>();
|
||||
for(let i=0;i<keys.length;i++){let parts:unknown;try{parts=JSON.parse(String(keys[i]));}catch{throw fault('corrupt_storage_scope');}if(!Array.isArray(parts)||parts.length!==3||parts.some(p=>typeof p!=='string'||!p))throw fault('corrupt_storage_scope');if(parts[0]!==scope.accountId||parts[1]!==scope.workspaceId||heads[i].throughRevision===undefined)continue;
|
||||
const pins=await value<Pins|undefined>(manifests.get([...base,parts[2]]));if(!pins||pins.version!==1||pins.rootRevision!==heads[i].throughRevision||pins.complete!==true){blockedProjects.push(parts[2]);continue;}
|
||||
if(!Array.isArray(pins.pins))throw fault('corrupt_depot_pins');for(const row of pins.pins){if(!Array.isArray(row)||row.length!==2||typeof row[0]!=='string'||!row[0]||!Array.isArray(row[1])||!row[1].length||row[1].some(owner=>typeof owner!=='string'||!owner))throw fault('corrupt_depot_pins');pinned.add(row[0]);}
|
||||
}
|
||||
if(blockedProjects.length)return freeze({complete:false,blockedProjects,markedIds:[],purgedIds:[],pinnedIds:[]});
|
||||
const store=tx.objectStore('asset-depot'),factKeys=await value(store.getAllKeys(range(scope,input.authority))),rows=await value<Fact[]>(store.getAll(range(scope,input.authority))),markedIds:string[]=[],purgedIds:string[]=[],pinnedIds:string[]=[];
|
||||
for(let i=0;i<rows.length;i++){const key=factKeys[i];if(!Array.isArray(key)||key.length!==4||typeof key[3]!=='string')throw fault('corrupt_asset_depot');validateFact(rows[i],input.authority,key[3]);}
|
||||
const candidates=rows.map((row,i)=>({row,key:factKeys[i] as string[]})).filter(({key})=>after===undefined||key[3]>after).slice(0,limit+1),page=candidates.slice(0,limit);
|
||||
for(const {row,key}of page){const id=key[3];if(pinned.has(id)){pinnedIds.push(id);if(row.purgedAt===undefined&&row.unreferencedAt!==undefined){delete row.unreferencedAt;store.put(row,key);}continue;}if(row.purgedAt!==undefined)continue;if(row.unreferencedAt===undefined){row.unreferencedAt=now;markedIds.push(id);store.put(row,key);}else if(now>=row.unreferencedAt&&now-row.unreferencedAt>=retentionMs){row.purgedAt=now;purgedIds.push(id);store.put(row,key);}}
|
||||
// Preserve a digest while any other live resource identity still owns its bytes.
|
||||
for(const digest of new Set(page.filter(({row})=>row.purgedAt===now).map(({row})=>row.ref.digest)))if(!rows.some(row=>row.ref.digest===digest&&row.purgedAt===undefined))tx.objectStore('asset-bytes').delete([...base,digest]);
|
||||
return freeze({complete:true,blockedProjects:[],markedIds,purgedIds,pinnedIds,...(candidates.length>limit?{nextAfter:page.at(-1)!.key[3]}:{})});
|
||||
});
|
||||
},
|
||||
});}
|
||||
Reference in New Issue
Block a user