Deduplicate stored upload files and retain adoption bytes until their Root is durable

This commit is contained in:
2026-11-09 16:31:31 +08:00
parent a626e6c300
commit 902e30a65b
10 changed files with 167 additions and 18 deletions
+39 -1
View File
@@ -1,6 +1,6 @@
import {equal,fault,freeze} from '../canvas-core/values.ts';
import {sharedAssetRef,validateSharedAsset,type AssetDepotPort,type DepotCut,type DepotEntry,type SharedAsset} from './asset-depot.ts';
import type {StorageLeader,StorageScope} from './storage-port.ts';
import type {StorageLeader,StorageScope,RecoveryImage} from './storage-port.ts';
export const depotStores=['asset-depot','asset-bytes','asset-pins'];
type Fact={ref:SharedAsset;unreferencedAt?:number;purgedAt?:number};
@@ -30,6 +30,44 @@ export async function writeDepotCut(tx:IDBTransaction,scope:StorageScope,rootRev
}
tx.objectStore('asset-pins').put({version:1,rootRevision,complete:cut.complete,pins:cut.pins} satisfies Pins,[...base,scope.projectId]);
}
/** DB v4 file rows may refer to immutable shared bytes after a proven upload ACK. */
export type DepotFile={readonly fileVersion:1;readonly kind:'asset-depot';readonly authority:string;readonly resourceId:string;readonly digest:string;readonly byteLength:number;readonly mimeType:string};
function depotFile(row:unknown,id:string):DepotFile{
const file=row as DepotFile;
if(!file||file.fileVersion!==1)throw fault('file_interpreter_unavailable');
authority(file.authority);
if(file.kind!=='asset-depot'||file.resourceId!==id||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest)||!Number.isSafeInteger(file.byteLength)||file.byteLength<1||file.byteLength>20*1024*1024||typeof file.mimeType!=='string')throw fault('corrupt_file_store');
return file;
}
/** Undefined means bytes were explicitly evicted; identity metadata remains. */
export async function resolveDepotFile(tx:IDBTransaction,scope:StorageScope,id:string,row:unknown):Promise<Blob|undefined>{
if(row===undefined||row instanceof Blob)return row;
const file=depotFile(row,id),base=prefix(scope,file.authority),fact=await value<Fact|undefined>(tx.objectStore('asset-depot').get([...base,id]));
if(!fact)throw fault('corrupt_file_store');validateFact(fact,file.authority,id);
if(fact.ref.digest!==file.digest||fact.ref.asset.bytes!==file.byteLength)throw fault('corrupt_file_store');
if(fact.purgedAt!==undefined)return;
const bytes=await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,file.digest]));
if(bytes===undefined)return;
if(!(bytes instanceof Blob)||bytes.size!==file.byteLength)throw fault('corrupt_file_store');
// Preserve the original local File MIME even if the server detected another MIME.
return bytes.type===file.mimeType?bytes:bytes.slice(0,bytes.size,file.mimeType);
}
/** Convert only Root-proven stored uploads, after shared bytes and local files are written. */
export async function promoteDepotFiles(tx:IDBTransaction,scope:StorageScope,key:string,image:RecoveryImage,cut:DepotCut){
type Resource={id:string;digest?:string;byteLength:number;mimeType:string;stored?:SharedAsset['asset'];retirement?:{purgedAt?:number}};
const payload=image.payload as {runtime?:{resources?:readonly (readonly [string,Resource])[]}},entries=payload?.runtime?.resources;
if(entries===undefined)return;if(!Array.isArray(entries))throw fault('corrupt_recovery_image');
const resources=new Map<string,Resource>(entries),files=tx.objectStore('files'),base=prefix(scope,cut.authority);
for(const write of cut.writes){const id=write.ref.asset.id,resource=resources.get(id);if(!resource?.stored||resource.retirement?.purgedAt!==undefined)continue;
if(resource.id!==id||resource.digest!==write.ref.digest||resource.byteLength!==write.ref.asset.bytes||typeof resource.mimeType!=='string'||!equal(resource.stored,write.ref.asset))throw fault('shared_asset_identity_conflict',[id]);
const local=await value<unknown>(files.get([key,id]));if(local===undefined)continue;
if(!(local instanceof Blob)){const pointer=depotFile(local,id);if(pointer.authority!==cut.authority||pointer.digest!==write.ref.digest||pointer.byteLength!==resource.byteLength||pointer.mimeType!==resource.mimeType)throw fault('shared_asset_identity_conflict',[id]);continue;}
const bytes=await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,write.ref.digest]));
if(!bytes)continue;
if(!(bytes instanceof Blob)||bytes.size!==resource.byteLength||local.size!==resource.byteLength||local.type!==resource.mimeType)throw fault('shared_asset_bytes_changed',[id]);
files.put({fileVersion:1,kind:'asset-depot',authority:cut.authority,resourceId:id,digest:write.ref.digest,byteLength:resource.byteLength,mimeType:resource.mimeType} satisfies DepotFile,[key,id]);
}
}
type Execute=<T>(leader:StorageLeader,stores:string[],mode:IDBTransactionMode,run:(tx:IDBTransaction)=>Promise<T>)=>Promise<T>;
export function createDepotStorage(execute:Execute):AssetDepotPort{return Object.freeze({
async read(leader,scopeAuthority,id){authority(scopeAuthority);if(typeof id!=='string'||!id)throw fault('invalid_resource_id');return execute(leader,depotStores,'readonly',async tx=>{const base=prefix(leader.scope,scopeAuthority),row=await value<Fact|undefined>(tx.objectStore('asset-depot').get([...base,id]));if(!row)return;validateFact(row,scopeAuthority,id);const bytes=row.purgedAt===undefined?await value<Blob|undefined>(tx.objectStore('asset-bytes').get([...base,row.ref.digest])):undefined;if(bytes!==undefined&&(!(bytes instanceof Blob)||bytes.size!==row.ref.asset.bytes))throw fault('corrupt_asset_depot');return freeze({...row,...(bytes?{bytes}:{})}) as DepotEntry;});},