Zayden-Jung/Endoleap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Deduplicate stored upload files and retain adoption bytes until their Root is durable

Browse Source
This commit is contained in:
Codex Bot
2026-11-09 16:31:31 +08:00
parent a626e6c300
commit 902e30a65b
10 changed files with 167 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lib/canvas-runtime/storage.ts
+11 -9
View File
@@ -1,4 +1,4 @@
import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut} from './asset-depot-storage.ts';
import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut,promoteDepotFiles,resolveDepotFile} from './asset-depot-storage.ts';
import { applyJournal, diffJournal, type JournalPatch } from './journal-delta.ts';
import { isSealedRecoveryImage } from './recovery-encoding.ts';
import type { Json } from '../canvas-core/types.ts';
@@ -46,7 +46,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
let closed = false;
const handles = new Set<StorageLeader>(), lastCuts=new WeakMap<StorageLeader,StoredImage>(), writeTails=new WeakMap<StorageLeader,Promise<unknown>>();
const opening = new Promise<IDBDatabase>((resolve, reject) => {
const request = factory.open(options.name ?? 'endoleap-headless', 3);
const request = factory.open(options.name ?? 'endoleap-headless', 4);
request.onupgradeneeded = () => {
for (const name of ['metadata', 'checkpoints', 'journal', 'files',...depotStores]) if (!request.result.objectStoreNames.contains(name)) request.result.createObjectStore(name);
};
@@ -158,13 +158,14 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
const saved=previous===undefined?undefined:await requestValue<StoredCut>(transaction.objectStore(metadata.checkpointRevision===previous?'checkpoints':'journal').get([key,previous]));
if(previous!==undefined&&image.rootRevision!==previous+1){
if(image.rootRevision!==previous)throw fault('storage_revision_gap');
if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot)await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);return{durableRootRevision:previous};
if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot){await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);await promoteDepotFiles(transaction,leader.scope,key,image,depot);}return{durableRootRevision:previous};
}
if(previous!==undefined&&(!saved||image.localRevision<cutHeader(saved).localRevision))throw fault('storage_revision_conflict');
if(!('image' in candidate)&&(candidate.baseRevision!==previous||candidate.baseChecksum!==saved?.checksum))throw fault('recovery_delta_base_mismatch');
if(depot)await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);
for(const file of files){if(!file.id||!(file.bytes instanceof Blob))throw fault('invalid_staged_file');if(await requestValue(transaction.objectStore('files').getKey([key,file.id])))throw fault('resource_identity_conflict',[file.id]);transaction.objectStore('files').put(file.bytes,[key,file.id]);}
for(const id of deleteFiles)transaction.objectStore('files').delete([key,id]);
if(depot)await promoteDepotFiles(transaction,leader.scope,key,image,depot);
// A format transition must not depend on a newer partial checkpoint in an older reader.
if(previous===undefined||((base?.image.formatVersion??0)>=7&&base?.image.formatVersion!==image.formatVersion)||image.rootRevision-(metadata.checkpointRevision??0)>=checkpointEvery){
transaction.objectStore('checkpoints').put(stored,[key,image.rootRevision]);if(metadata.checkpointRevision!==undefined)transaction.objectStore('checkpoints').delete([key,metadata.checkpointRevision]);transaction.objectStore('journal').delete(IDBKeyRange.bound([key,0],[key,image.rootRevision]));metadata.checkpointRevision=image.rootRevision;
@@ -178,21 +179,22 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
scanFiles(leader:StorageLeader,input:{expectedRootRevision:number;after?:string;limit:number}) {
const {expectedRootRevision,after,limit}=input;
if(!Number.isSafeInteger(expectedRootRevision)||expectedRootRevision<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))return Promise.reject(fault('invalid_file_inventory'));
return execute(leader,['files'],'readonly',async(transaction,metadata,key)=>{
return execute(leader,['files',...depotStores],'readonly',async(transaction,metadata,key)=>{
if(metadata.throughRevision!==expectedRootRevision)throw fault('stale_file_inventory');
const rows=await new Promise<StoredFileInfo[]>((resolve,reject)=>{
const files:StoredFileInfo[]=[],range=IDBKeyRange.bound(after===undefined?[key]:[key,after],[key,[]],after!==undefined,true),request=transaction.objectStore('files').openCursor(range);
request.onerror=()=>reject(request.error);
request.onsuccess=()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key,bytes=cursor.value;
if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]||!(bytes instanceof Blob)){reject(fault('corrupt_file_store'));return;}
files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue();
};
request.onsuccess=()=>{void(async()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key;
if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]){throw fault('corrupt_file_store');}
const bytes=await resolveDepotFile(transaction,leader.scope,parts[1],cursor.value);
if(bytes)files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue();
})().catch(reject);};
});
const more=rows.length>limit,files=more?rows.slice(0,limit):rows;
return Object.freeze({files:Object.freeze(files),rootRevision:expectedRootRevision,...(more?{nextAfter:files.at(-1)!.id}:{})});
});
},
readFile(leader: StorageLeader, id: string) { return execute(leader, ['files'], 'readonly', (transaction, _metadata, key) => requestValue<Blob | undefined>(transaction.objectStore('files').get([key, id]))); },
readFile(leader: StorageLeader, id: string) { return execute(leader, ['files',...depotStores], 'readonly', async(transaction, _metadata, key) => resolveDepotFile(transaction,leader.scope,id,await requestValue<unknown>(transaction.objectStore('files').get([key, id])))); },
close() { closed = true; for (const handle of handles) void handle.release(); void opening.then(database => database.close()).catch(() => {}); },
});
}