Inventory scoped files and atomically collect verified legacy orphans

This commit is contained in:
2026-11-08 23:21:27 +08:00
parent 4b594f85eb
commit 3ccb8dbc5c
9 changed files with 232 additions and 23 deletions
+33 -5
View File
@@ -4,10 +4,13 @@ import type { Json } from '../canvas-core/types.ts';
import { clone, equal, fault, freeze, validJson } from '../canvas-core/values.ts';
export type StorageScope = { readonly accountId: string; readonly workspaceId: string; readonly projectId: string };
export type RecoveryImage = { readonly formatVersion: 1 | 2 | 3 | 4; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json };
export type RecoveryImage = { readonly formatVersion: 1 | 2 | 3 | 4 | 5; readonly rootRevision: number; readonly localRevision: number; readonly payload: Json };
export type DurableBatch = { readonly image: RecoveryImage; readonly deleteFiles?:readonly string[]; readonly files?: readonly { readonly id: string; readonly bytes: Blob }[] };
export type StorageLeader = { readonly scope: StorageScope; readonly fence: number; readonly active: boolean; release(): Promise<void> };
export type StoredFileInfo={readonly id:string;readonly byteLength:number;readonly mimeType:string};
export type FileInventoryPage={readonly files:readonly StoredFileInfo[];readonly rootRevision:number;readonly nextAfter?:string};
export type StoragePort = {
scanFiles?(leader:StorageLeader,input:{readonly expectedRootRevision:number;readonly after?:string;readonly limit:number}):Promise<FileInventoryPage>;
readonly fileDeletion?: 'atomic-v1';
claim(scope: StorageScope): Promise<StorageLeader>;
recover(leader: StorageLeader): Promise<RecoveryImage | null>;
@@ -33,8 +36,14 @@ const keyFor = (scope: StorageScope) => {
if (![scope.accountId, scope.workspaceId, scope.projectId].every(value => typeof value === 'string' && value.length > 0)) throw fault('invalid_storage_scope');
return JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]);
};
function purgedFileIds(image:RecoveryImage):Set<string>{
const result=new Set<string>(),payload=image.payload as {runtime?:{resources?:[string,{retirement?:{purgedAt?:number}}][];orphanFiles?:[string,{retirement?:{purgedAt?:number}}][];imports?:[string,{source?:{fileId:string;retirement?:{purgedAt?:number};previousFiles?:{fileId:string;retirement?:{purgedAt?:number}}[]}}][]}};
for(const rows of [payload?.runtime?.resources,payload?.runtime?.orphanFiles])if(Array.isArray(rows))for(const row of rows)if(Array.isArray(row)&&typeof row[0]==='string'&&row[1]?.retirement?.purgedAt!==undefined)result.add(row[0]);
if(Array.isArray(payload?.runtime?.imports))for(const [,job]of payload.runtime.imports){if(!job?.source)continue;for(const file of [job.source,...(job.source.previousFiles??[])])if(file.retirement?.purgedAt!==undefined)result.add(file.fileId);}
return result;
}
function validateImage(image: RecoveryImage) {
if (![1,2,3,4].includes(image.formatVersion)) throw fault('unsupported_recovery_version');
if (![1,2,3,4,5].includes(image.formatVersion)) throw fault('unsupported_recovery_version');
if (!Number.isSafeInteger(image.rootRevision) || image.rootRevision < 0 || !Number.isSafeInteger(image.localRevision) || image.localRevision < 0 || image.localRevision > image.rootRevision || !validJson(image.payload)) throw fault('corrupt_recovery_image');
}
const requestValue = <T>(request: IDBRequest<T>) => new Promise<T>((resolve, reject) => { request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); });
@@ -134,18 +143,20 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
// Detach mutable callers and descriptors before the first await, including queued commits.
const image=isSealedRecoveryImage(batch.image)?batch.image:freeze(clone(batch.image)),files=(batch.files??[]).map(file=>({id:file.id,bytes:file.bytes})),deleteFiles=[...(batch.deleteFiles??[])];if(deleteFiles.length>128||new Set(deleteFiles).size!==deleteFiles.length||deleteFiles.some(id=>typeof id!=='string'||!id||files.some(file=>file.id===id)))throw fault('invalid_file_deletion');if(!isSealedRecoveryImage(image))validateImage(image);
if(deleteFiles.length){
type FileProof={fileId?:string;retirement?:{version:number;unreferencedAt:number;purgedAt?:number}};
const payload=image.payload as {runtime?:{resources?:[string,FileProof][];imports?:[string,{status:string;source?:FileProof&{previousFiles?:FileProof[]}}][]}};
type FileProof={id?:string;version?:number;byteLength?:number;mimeType?:string;digest?:string;fileId?:string;retirement?:{version:number;unreferencedAt:number;purgedAt?:number}};
const payload=image.payload as {runtime?:{orphanFiles?:[string,FileProof][];resources?:[string,FileProof][];imports?:[string,{status:string;source?:FileProof&{previousFiles?:FileProof[]}}][]}};
const proofs=new Map<string,FileProof>(),owned=new Set<string>();
const own=(id:string,file:FileProof,eligible:boolean)=>{if(typeof id!=='string'||!id||owned.has(id))throw fault('invalid_file_deletion');owned.add(id);if(eligible)proofs.set(id,file);};
for(const [id,resource]of payload?.runtime?.resources??[])own(id,resource,true);
if(image.formatVersion===4)for(const [,job]of payload?.runtime?.imports??[]){if(job.source){own(job.source.fileId!,job.source,['completed','partial','cancelled'].includes(job.status));for(const file of job.source.previousFiles??[])own(file.fileId!,file,true);}}
if(image.formatVersion>=4)for(const [,job]of payload?.runtime?.imports??[]){if(job.source){own(job.source.fileId!,job.source,['completed','partial','cancelled'].includes(job.status));for(const file of job.source.previousFiles??[])own(file.fileId!,file,true);}}
if(image.formatVersion===5)for(const [id,file]of payload?.runtime?.orphanFiles??[]){if(file.version!==1||file.id!==id||!Number.isSafeInteger(file.byteLength)||file.byteLength!<0||typeof file.mimeType!=='string'||typeof file.digest!=='string'||!/^[a-f0-9]{64}$/.test(file.digest))throw fault('invalid_file_deletion');own(id,file,true);}
if(image.formatVersion<3||deleteFiles.some(id=>{const marker=proofs.get(id)?.retirement;return !marker||marker.version!==1||!Number.isSafeInteger(marker.unreferencedAt)||marker.unreferencedAt<0||!Number.isSafeInteger(marker.purgedAt)||marker.purgedAt!<marker.unreferencedAt;}))throw fault('invalid_file_deletion');
}
check(leader);
const task=(writeTails.get(leader)??Promise.resolve()).catch(()=>{}).then(async()=>{
let base=lastCuts.get(leader)??await recoverCut(leader);
if(base?.image.rootRevision===image.rootRevision)base=await recoverCut(leader);
if(files.length&&base){const retired=purgedFileIds(base.image);if(files.some(file=>retired.has(file.id)))throw fault('resource_identity_conflict');}
const digest=await imageDigest(image),stored:StoredImage={image,checksum:digest.checksum};let candidate:StoredCut=stored;
if(base&&image.rootRevision===base.image.rootRevision+1){
const patches=diffJournal(base.image.payload,image.payload);
@@ -171,6 +182,23 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
});
writeTails.set(leader,task);return task;
},
scanFiles(leader:StorageLeader,input:{expectedRootRevision:number;after?:string;limit:number}) {
const {expectedRootRevision,after,limit}=input;
if(!Number.isSafeInteger(expectedRootRevision)||expectedRootRevision<0||!Number.isSafeInteger(limit)||limit<1||limit>128||after!==undefined&&(typeof after!=='string'||!after))return Promise.reject(fault('invalid_file_inventory'));
return execute(leader,['files'],'readonly',async(transaction,metadata,key)=>{
if(metadata.throughRevision!==expectedRootRevision)throw fault('stale_file_inventory');
const rows=await new Promise<StoredFileInfo[]>((resolve,reject)=>{
const files:StoredFileInfo[]=[],range=IDBKeyRange.bound(after===undefined?[key]:[key,after],[key,[]],after!==undefined,true),request=transaction.objectStore('files').openCursor(range);
request.onerror=()=>reject(request.error);
request.onsuccess=()=>{const cursor=request.result;if(!cursor){resolve(files);return;}const parts=cursor.key,bytes=cursor.value;
if(!Array.isArray(parts)||parts.length!==2||parts[0]!==key||typeof parts[1]!=='string'||!parts[1]||!(bytes instanceof Blob)){reject(fault('corrupt_file_store'));return;}
files.push(Object.freeze({id:parts[1],byteLength:bytes.size,mimeType:bytes.type}));if(files.length>limit)resolve(files);else cursor.continue();
};
});
const more=rows.length>limit,files=more?rows.slice(0,limit):rows;
return Object.freeze({files:Object.freeze(files),rootRevision:expectedRootRevision,...(more?{nextAfter:files.at(-1)!.id}:{})});
});
},
readFile(leader: StorageLeader, id: string) { return execute(leader, ['files'], 'readonly', (transaction, _metadata, key) => requestValue<Blob | undefined>(transaction.objectStore('files').get([key, id]))); },
close() { closed = true; for (const handle of handles) void handle.release(); void opening.then(database => database.close()).catch(() => {}); },
});