Chain delta journal checksums without rehashing every full recovery image
This commit is contained in:
@@ -2,22 +2,24 @@ import type { Json } from '../canvas-core/types.ts';
|
||||
import { fault, freeze, validJson } from '../canvas-core/values.ts';
|
||||
import type { RecoveryImage } from './storage-port.ts';
|
||||
|
||||
type Encoded = {value:Json|undefined;height:number;reusable:boolean};
|
||||
const immutable=new WeakMap<object,Encoded>(),sealed=new WeakSet<object>();
|
||||
type Encoded = {value:Json|undefined;height:number;reusable:boolean;bytes:number};
|
||||
const immutable=new WeakMap<object,Encoded>(),sealed=new WeakSet<object>(),encodedSizes=new WeakMap<object,number>(),imageSizes=new WeakMap<object,number>();
|
||||
function utf8Length(text:string){let bytes=0;for(let i=0;i<text.length;i++){const c=text.charCodeAt(i);if(c<128)bytes++;else if(c<2048)bytes+=2;else if(c>=0xd800&&c<=0xdbff&&i+1<text.length&&text.charCodeAt(i+1)>=0xdc00&&text.charCodeAt(i+1)<=0xdfff){bytes+=4;i++;}else bytes+=3;}return bytes;}
|
||||
const scalarBytes=(value:unknown)=>utf8Length(JSON.stringify(value));
|
||||
|
||||
/** JSON value semantics, with reuse only for transitively frozen plain data properties. */
|
||||
export function encodeRecoveryValue(input:unknown):Json {
|
||||
const visiting=new Set<object>();
|
||||
function visit(value:unknown,depth:number,key:string):Encoded {
|
||||
if(value===null||typeof value==='string'||typeof value==='boolean'){if(depth>64)throw fault('invalid_recovery_payload');return{value,height:0,reusable:true};}
|
||||
if(typeof value==='number'){if(depth>64)throw fault('invalid_recovery_payload');return{value:Number.isFinite(value)?(Object.is(value,-0)?0:value):null,height:0,reusable:true};}
|
||||
if(value===undefined||typeof value==='symbol')return{value:undefined,height:0,reusable:true};
|
||||
if(value===null||typeof value==='string'||typeof value==='boolean'){if(depth>64)throw fault('invalid_recovery_payload');return{value,height:0,reusable:true,bytes:scalarBytes(value)};}
|
||||
if(typeof value==='number'){if(depth>64)throw fault('invalid_recovery_payload');const normalized=Number.isFinite(value)?(Object.is(value,-0)?0:value):null;return{value:normalized,height:0,reusable:true,bytes:scalarBytes(normalized)};}
|
||||
if(value===undefined||typeof value==='symbol')return{value:undefined,height:0,reusable:true,bytes:0};
|
||||
const fallback=()=>{
|
||||
// Preserve the previous JSON codec for toJSON, boxed values and custom prototypes.
|
||||
// These can depend on hidden mutable state and are deliberately never cached.
|
||||
const text=JSON.stringify({[key]:value}),encoded=JSON.parse(text)[key] as Json|undefined;
|
||||
if(encoded!==undefined&&!validJson(encoded,depth))throw fault('invalid_recovery_payload');
|
||||
return{value:freeze(encoded),height:0,reusable:false};
|
||||
return{value:freeze(encoded),height:0,reusable:false,bytes:encoded===undefined?0:scalarBytes(encoded)};
|
||||
};
|
||||
if(typeof value==='bigint'||typeof value==='function')return fallback();
|
||||
if(typeof value!=='object')throw fault('invalid_recovery_payload');
|
||||
@@ -29,23 +31,23 @@ export function encodeRecoveryValue(input:unknown):Json {
|
||||
if(depth>64)throw fault('invalid_recovery_payload');
|
||||
if(visiting.has(object))throw fault('invalid_recovery_payload');visiting.add(object);
|
||||
const toJSON=Object.getOwnPropertyDescriptor(object,'toJSON');
|
||||
let reusable=Object.isFrozen(object)&&(!toJSON||Object.hasOwn(toJSON,'value')),height=0;
|
||||
let reusable=Object.isFrozen(object)&&(!toJSON||Object.hasOwn(toJSON,'value')),height=0,bytes=2;
|
||||
try {
|
||||
if(Array.isArray(object)){
|
||||
const result:Json[]=[];
|
||||
for(let index=0;index<object.length;index++){
|
||||
const descriptor=Object.getOwnPropertyDescriptor(object,String(index));if(!descriptor||!Object.hasOwn(descriptor,'value'))reusable=false;
|
||||
const child=visit(object[index],depth+1,String(index));if(child.value===undefined&&depth+1>64)throw fault('invalid_recovery_payload');result.push(child.value??null);height=Math.max(height,1+child.height);reusable&&=child.reusable;
|
||||
const child=visit(object[index],depth+1,String(index));if(child.value===undefined&&depth+1>64)throw fault('invalid_recovery_payload');result.push(child.value??null);bytes+=(index?1:0)+(child.value===undefined?4:child.bytes);height=Math.max(height,1+child.height);reusable&&=child.reusable;
|
||||
}
|
||||
const encoded={value:Object.freeze(result) as unknown as Json,height,reusable};if(reusable)immutable.set(object,encoded);return encoded;
|
||||
const encoded={value:Object.freeze(result) as unknown as Json,height,reusable,bytes};encodedSizes.set(result,bytes);if(reusable)immutable.set(object,encoded);return encoded;
|
||||
}
|
||||
const result:Record<string,Json>={};
|
||||
const result:Record<string,Json>={};let properties=0;
|
||||
for(const name of Object.keys(object)){
|
||||
const descriptor=Object.getOwnPropertyDescriptor(object,name);if(!descriptor||!Object.hasOwn(descriptor,'value'))reusable=false;
|
||||
const child=visit((object as Record<string,unknown>)[name],depth+1,name);reusable&&=child.reusable;
|
||||
if(child.value!==undefined){Object.defineProperty(result,name,{value:child.value,enumerable:true});height=Math.max(height,1+child.height);}
|
||||
if(child.value!==undefined){bytes+=(properties++?1:0)+scalarBytes(name)+1+child.bytes;Object.defineProperty(result,name,{value:child.value,enumerable:true});height=Math.max(height,1+child.height);}
|
||||
}
|
||||
const encoded={value:Object.freeze(result) as Json,height,reusable};if(reusable)immutable.set(object,encoded);return encoded;
|
||||
const encoded={value:Object.freeze(result) as Json,height,reusable,bytes};encodedSizes.set(result,bytes);if(reusable)immutable.set(object,encoded);return encoded;
|
||||
}finally{visiting.delete(object);}
|
||||
}
|
||||
const result=visit(input,0,'');if(result.value===undefined)throw fault('invalid_recovery_payload');return result.value;
|
||||
@@ -54,6 +56,9 @@ export function encodeRecoveryValue(input:unknown):Json {
|
||||
/** Only codec-owned, validated immutable images qualify for the storage copy fast path. */
|
||||
export function sealRecoveryImage(formatVersion:1|2|3|4|5|6|7|8|9|10|11|12|13|14|15|16|17,rootRevision:number,localRevision:number,payload:unknown):RecoveryImage {
|
||||
if(![1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17].includes(formatVersion)||!Number.isSafeInteger(rootRevision)||rootRevision<0||!Number.isSafeInteger(localRevision)||localRevision<0||localRevision>rootRevision)throw fault('corrupt_recovery_image');
|
||||
const image=Object.freeze({formatVersion,rootRevision,localRevision,payload:encodeRecoveryValue(payload)});sealed.add(image);return image;
|
||||
const encoded=encodeRecoveryValue(payload),image=Object.freeze({formatVersion,rootRevision,localRevision,payload:encoded});sealed.add(image);imageSizes.set(image,JSON.stringify({...image,payload:null}).length-4+(encoded&&typeof encoded==='object'?encodedSizes.get(encoded)??scalarBytes(encoded):scalarBytes(encoded)));return image;
|
||||
}
|
||||
export function isSealedRecoveryImage(image:RecoveryImage):boolean{return sealed.has(image);}
|
||||
|
||||
/** Exact UTF-8 JSON size, available only for codec-owned immutable images. */
|
||||
export function sealedRecoveryByteLength(image:RecoveryImage){return imageSizes.get(image);}
|
||||
|
||||
@@ -1,25 +1,26 @@
|
||||
import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut,promoteDepotFiles,resolveDepotFile} from './asset-depot-storage.ts';
|
||||
import { applyJournal, diffJournal, type JournalPatch } from './journal-delta.ts';
|
||||
import { isSealedRecoveryImage } from './recovery-encoding.ts';
|
||||
import { isSealedRecoveryImage, sealedRecoveryByteLength } from './recovery-encoding.ts';
|
||||
import type { Json } from '../canvas-core/types.ts';
|
||||
import { clone, equal, fault, freeze, validJson } from '../canvas-core/values.ts';
|
||||
import { clone, fault, freeze, validJson } from '../canvas-core/values.ts';
|
||||
|
||||
import type {StorageScope,RecoveryImage,DurableBatch,StorageLeader,StoredFileInfo,FileInventoryPage,StoragePort} from './storage-port.ts';
|
||||
export type {StorageScope,RecoveryImage,DurableBatch,StorageLeader,StoredFileInfo,FileInventoryPage,StoragePort} from './storage-port.ts';
|
||||
type Metadata = { fence: number; throughRevision?: number; checkpointRevision?: number };
|
||||
type StoredImage = { readonly image: RecoveryImage; readonly checksum: string };
|
||||
type StoredDelta = {journalVersion:1;baseRevision:number;baseChecksum:string;header:Omit<RecoveryImage,'payload'>;patches:JournalPatch[];checksum:string};
|
||||
type StoredDelta = {journalVersion:1|2;baseRevision:number;baseChecksum:string;header:Omit<RecoveryImage,'payload'>;patches:JournalPatch[];checksum:string};
|
||||
type StoredCut = StoredImage | StoredDelta;
|
||||
function cutHeader(cut:StoredCut):Omit<RecoveryImage,'payload'>{
|
||||
if(!cut||typeof cut!=='object')throw fault('corrupt_recovery_image');
|
||||
if('image' in cut){validateImage(cut.image);return cut.image;}
|
||||
if(cut.journalVersion!==1)throw fault('recovery_delta_interpreter_unavailable');
|
||||
if(![1,2].includes(cut.journalVersion))throw fault('recovery_delta_interpreter_unavailable');
|
||||
if(!cut.header)throw fault('corrupt_recovery_delta');validateImage({...cut.header,payload:null});return cut.header;
|
||||
}
|
||||
async function imageDigest(image: RecoveryImage) {
|
||||
async function imageDigest(image: unknown) {
|
||||
const bytes = new TextEncoder().encode(JSON.stringify(image));
|
||||
return {checksum:[...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))].map(value => value.toString(16).padStart(2, '0')).join(''),byteLength:bytes.byteLength};
|
||||
}
|
||||
function deltaBody(cut:StoredDelta){const {journalVersion,baseRevision,baseChecksum,header,patches}=cut;return{journalVersion,baseRevision,baseChecksum,header,patches};}
|
||||
const keyFor = (scope: StorageScope) => {
|
||||
if (![scope.accountId, scope.workspaceId, scope.projectId].every(value => typeof value === 'string' && value.length > 0)) throw fault('invalid_storage_scope');
|
||||
return JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]);
|
||||
@@ -44,7 +45,7 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
|
||||
const checkpointEvery = options.checkpointEvery ?? 32;
|
||||
if (!Number.isSafeInteger(checkpointEvery) || checkpointEvery < 1) throw fault('invalid_checkpoint_interval');
|
||||
let closed = false;
|
||||
const handles = new Set<StorageLeader>(), lastCuts=new WeakMap<StorageLeader,StoredImage>(), writeTails=new WeakMap<StorageLeader,Promise<unknown>>();
|
||||
const handles = new Set<StorageLeader>(), lastCuts=new WeakMap<StorageLeader,StoredImage>(), writeTails=new WeakMap<StorageLeader,Promise<unknown>>(),lastCheckpoints=new WeakMap<StorageLeader,number>();
|
||||
const opening = new Promise<IDBDatabase>((resolve, reject) => {
|
||||
const request = factory.open(options.name ?? 'endoleap-headless', 4);
|
||||
request.onupgradeneeded = () => {
|
||||
@@ -79,25 +80,25 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
|
||||
} catch (error) { try { transaction.abort(); } catch { /* Already terminal. */ } await done.catch(() => {}); throw error; }
|
||||
}
|
||||
async function recoverCut(leader:StorageLeader):Promise<StoredImage|null>{
|
||||
const cuts=await execute(leader,['checkpoints','journal'],'readonly',async(transaction,metadata,key)=>{
|
||||
if(metadata.throughRevision===undefined)return [];
|
||||
const captured=await execute(leader,['checkpoints','journal'],'readonly',async(transaction,metadata,key)=>{
|
||||
if(metadata.throughRevision===undefined)return{cuts:[] as StoredCut[],checkpointRevision:undefined};
|
||||
if(metadata.checkpointRevision===undefined)throw fault('recovery_checkpoint_missing');
|
||||
const checkpoint=await requestValue<StoredImage|undefined>(transaction.objectStore('checkpoints').get([key,metadata.checkpointRevision]));
|
||||
if(!checkpoint?.image)throw fault('recovery_checkpoint_missing');
|
||||
let header=cutHeader(checkpoint);if(header.rootRevision!==metadata.checkpointRevision)throw fault('corrupt_recovery_image');
|
||||
const journal=metadata.checkpointRevision===metadata.throughRevision?[]:await requestValue<StoredCut[]>(transaction.objectStore('journal').getAll(IDBKeyRange.bound([key,metadata.checkpointRevision+1],[key,metadata.throughRevision])));
|
||||
for(const row of journal){const next=cutHeader(row);if(next.rootRevision!==header.rootRevision+1||next.localRevision<header.localRevision)throw fault('recovery_journal_gap');header=next;}
|
||||
if(header.rootRevision!==metadata.throughRevision)throw fault('recovery_journal_gap');return [checkpoint,...journal];
|
||||
if(header.rootRevision!==metadata.throughRevision)throw fault('recovery_journal_gap');return{cuts:[checkpoint,...journal],checkpointRevision:metadata.checkpointRevision};
|
||||
});
|
||||
let previous:StoredImage|null=null;
|
||||
// Hash and apply after the readonly transaction; awaiting crypto inside IDB would close it.
|
||||
for(const cut of cuts){
|
||||
for(const cut of captured.cuts){
|
||||
let image:RecoveryImage;
|
||||
if('image' in cut)image=cut.image;
|
||||
else{if(!previous||cut.baseRevision!==previous.image.rootRevision||cut.baseChecksum!==previous.checksum)throw fault('recovery_delta_base_mismatch');image={...cut.header,payload:applyJournal(previous.image.payload,cut.patches)};validateImage(image);}
|
||||
if(cut.checksum!==(await imageDigest(image)).checksum)throw fault('recovery_checksum_mismatch');previous=freeze({image,checksum:cut.checksum});
|
||||
if(cut.checksum!==(await imageDigest('image' in cut||cut.journalVersion===1?image:deltaBody(cut))).checksum)throw fault('recovery_checksum_mismatch');previous=freeze({image,checksum:cut.checksum});
|
||||
}
|
||||
check(leader);if(previous&&(!lastCuts.has(leader)||lastCuts.get(leader)!.image.rootRevision<=previous.image.rootRevision))lastCuts.set(leader,previous);return previous;
|
||||
check(leader);if(previous&&(!lastCuts.has(leader)||lastCuts.get(leader)!.image.rootRevision<=previous.image.rootRevision)){lastCuts.set(leader,previous);lastCheckpoints.set(leader,captured.checkpointRevision!);}return previous;
|
||||
}
|
||||
return Object.freeze({
|
||||
fileDeletion:'atomic-v1' as const,
|
||||
@@ -148,31 +149,43 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?:
|
||||
let base=lastCuts.get(leader)??await recoverCut(leader);
|
||||
if(base?.image.rootRevision===image.rootRevision)base=await recoverCut(leader);
|
||||
if(files.length&&base){const retired=purgedFileIds(base.image);if(files.some(file=>retired.has(file.id)))throw fault('resource_identity_conflict');}
|
||||
const digest=await imageDigest(image),stored:StoredImage={image,checksum:digest.checksum};let candidate:StoredCut=stored;
|
||||
if(base&&image.rootRevision===base.image.rootRevision+1){
|
||||
const baseCheckpoint=lastCheckpoints.get(leader),checkpoint=!base||((base.image.formatVersion>=7)&&base.image.formatVersion!==image.formatVersion)||image.rootRevision-(baseCheckpoint??0)>=checkpointEvery;
|
||||
let stored:StoredImage|undefined,candidate:StoredCut|undefined;
|
||||
if(base?.image.rootRevision===image.rootRevision){stored={image,checksum:base.checksum};candidate=stored;}
|
||||
else if(base&&!checkpoint&&image.rootRevision===base.image.rootRevision+1&&JSON.stringify(Object.keys(image))===JSON.stringify(['formatVersion','rootRevision','localRevision','payload'])){
|
||||
const patches=diffJournal(base.image.payload,image.payload);
|
||||
if(patches){const delta:StoredDelta={journalVersion:1,baseRevision:base.image.rootRevision,baseChecksum:base.checksum,header:{formatVersion:image.formatVersion,rootRevision:image.rootRevision,localRevision:image.localRevision},patches,checksum:digest.checksum};if(new TextEncoder().encode(JSON.stringify(delta)).byteLength<digest.byteLength)candidate=delta;}
|
||||
if(patches){
|
||||
const unsigned={journalVersion:2 as const,baseRevision:base.image.rootRevision,baseChecksum:base.checksum,header:{formatVersion:image.formatVersion,rootRevision:image.rootRevision,localRevision:image.localRevision},patches},digest=await imageDigest(unsigned),delta:StoredDelta={...unsigned,checksum:digest.checksum};
|
||||
// Adding the ASCII checksum member costs 78 UTF-8 bytes. Owned codec
|
||||
// images carry an exact size; mutable callers keep the defensive path.
|
||||
const fullBytes=sealedRecoveryByteLength(image)??new TextEncoder().encode(JSON.stringify(image)).byteLength;
|
||||
if(digest.byteLength+78<fullBytes){stored={image,checksum:delta.checksum};candidate=delta;}
|
||||
}
|
||||
}
|
||||
if(!stored||!candidate){stored={image,checksum:(await imageDigest(image)).checksum};candidate=stored;}
|
||||
const accepted=stored,cut=candidate;
|
||||
const receipt=await execute(leader,['checkpoints','journal','files',...depotStores],'readwrite',async(transaction,metadata,key)=>{
|
||||
const previous=metadata.throughRevision;
|
||||
if(previous===undefined&&base)throw fault('recovery_delta_base_mismatch');
|
||||
const saved=previous===undefined?undefined:await requestValue<StoredCut>(transaction.objectStore(metadata.checkpointRevision===previous?'checkpoints':'journal').get([key,previous]));
|
||||
if(previous!==undefined&&image.rootRevision!==previous+1){
|
||||
if(image.rootRevision!==previous)throw fault('storage_revision_gap');
|
||||
if(!saved||saved.checksum!==stored.checksum||!base||!equal(base.image,image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot){await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);await promoteDepotFiles(transaction,leader.scope,key,image,depot);}return{durableRootRevision:previous};
|
||||
if(!saved||saved.checksum!==accepted.checksum||!base||JSON.stringify(base.image)!==JSON.stringify(image)||files.length)throw fault('storage_revision_conflict');for(const id of deleteFiles)if(await requestValue(transaction.objectStore('files').getKey([key,id])))throw fault('storage_revision_conflict');if(depot){await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);await promoteDepotFiles(transaction,leader.scope,key,image,depot);}return{durableRootRevision:previous,checkpointRevision:metadata.checkpointRevision!};
|
||||
}
|
||||
if(previous!==undefined&&(!saved||image.localRevision<cutHeader(saved).localRevision))throw fault('storage_revision_conflict');
|
||||
if(!('image' in candidate)&&(candidate.baseRevision!==previous||candidate.baseChecksum!==saved?.checksum))throw fault('recovery_delta_base_mismatch');
|
||||
if(previous!==undefined&&(metadata.checkpointRevision!==baseCheckpoint||base?.image.rootRevision!==previous||base.checksum!==saved?.checksum))throw fault('recovery_delta_base_mismatch');
|
||||
if(!('image' in cut)&&(cut.baseRevision!==previous||cut.baseChecksum!==saved?.checksum))throw fault('recovery_delta_base_mismatch');
|
||||
if(depot)await writeDepotCut(transaction,leader.scope,image.rootRevision,depot);
|
||||
for(const file of files){if(!file.id||!(file.bytes instanceof Blob))throw fault('invalid_staged_file');if(await requestValue(transaction.objectStore('files').getKey([key,file.id])))throw fault('resource_identity_conflict',[file.id]);transaction.objectStore('files').put(file.bytes,[key,file.id]);}
|
||||
for(const id of deleteFiles)transaction.objectStore('files').delete([key,id]);
|
||||
if(depot)await promoteDepotFiles(transaction,leader.scope,key,image,depot);
|
||||
// A format transition must not depend on a newer partial checkpoint in an older reader.
|
||||
if(previous===undefined||((base?.image.formatVersion??0)>=7&&base?.image.formatVersion!==image.formatVersion)||image.rootRevision-(metadata.checkpointRevision??0)>=checkpointEvery){
|
||||
transaction.objectStore('checkpoints').put(stored,[key,image.rootRevision]);if(metadata.checkpointRevision!==undefined)transaction.objectStore('checkpoints').delete([key,metadata.checkpointRevision]);transaction.objectStore('journal').delete(IDBKeyRange.bound([key,0],[key,image.rootRevision]));metadata.checkpointRevision=image.rootRevision;
|
||||
}else transaction.objectStore('journal').put(candidate,[key,image.rootRevision]);
|
||||
transaction.objectStore('metadata').put({...metadata,throughRevision:image.rootRevision},key);return{durableRootRevision:image.rootRevision};
|
||||
if(checkpoint){
|
||||
transaction.objectStore('checkpoints').put(accepted,[key,image.rootRevision]);if(metadata.checkpointRevision!==undefined)transaction.objectStore('checkpoints').delete([key,metadata.checkpointRevision]);transaction.objectStore('journal').delete(IDBKeyRange.bound([key,0],[key,image.rootRevision]));metadata.checkpointRevision=image.rootRevision;
|
||||
}else transaction.objectStore('journal').put(cut,[key,image.rootRevision]);
|
||||
transaction.objectStore('metadata').put({...metadata,throughRevision:image.rootRevision},key);return{durableRootRevision:image.rootRevision,checkpointRevision:metadata.checkpointRevision!};
|
||||
});
|
||||
lastCuts.set(leader,freeze(stored));return receipt;
|
||||
lastCuts.set(leader,freeze(accepted));lastCheckpoints.set(leader,receipt.checkpointRevision);return{durableRootRevision:receipt.durableRootRevision};
|
||||
});
|
||||
writeTails.set(leader,task);return task;
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user