Persist versioned Blob recovery cuts with cooperative JSON serialization

This commit is contained in:
2026-11-10 19:01:25 +08:00
parent b051509dee
commit 8958d835b9
8 changed files with 155 additions and 26 deletions
@@ -0,0 +1,30 @@
import {createBrowserStorage} from '../../src/lib/canvas-runtime/storage.ts';
import {sealRecoveryImage} from '../../src/lib/canvas-runtime/recovery-encoding.ts';
export async function blobCheckpointCase(assert){
const name=`blob-checkpoint-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},key=JSON.stringify(['a','w','p']),image=revision=>({formatVersion:1,rootRevision:revision,localRevision:revision,payload:{rows:Array.from({length:250},(_,i)=>({id:i,text:`版本 ${revision} 👀 ${'x'.repeat(100)}`})),request:{id:'frozen',body:'{"exact":"original bytes"}'}}});
const read=req=>new Promise((resolve,reject)=>{req.onsuccess=()=>resolve(req.result);req.onerror=()=>reject(req.error);}),done=tx=>new Promise((resolve,reject)=>{tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);}),hash=async blob=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',await blob.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join('');
let raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name,4);request.onupgradeneeded=()=>{for(const store of ['metadata','checkpoints','journal','files'])request.result.createObjectStore(store);};request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
const initial=image(0),checksum=await hash(new Blob([JSON.stringify(initial)]));let tx=raw.transaction(['metadata','checkpoints'],'readwrite'),terminal=done(tx);tx.objectStore('metadata').put({fence:0,throughRevision:0,checkpointRevision:0},key);tx.objectStore('checkpoints').put({image:initial,checksum},[key,0]);await terminal;raw.close();let storage=createBrowserStorage({name,checkpointEvery:1}),leader;
try{
leader=await storage.claim(scope);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(initial),'DB v4 legacy checkpoint changed during upgrade');const first=image(1);await storage.commit(leader,{image:sealRecoveryImage(1,1,1,first.payload),files:[{id:'kept',bytes:new Blob(['kept bytes'])}]});
raw=await read(indexedDB.open(name));assert(raw.version===5,'Blob cuts did not fence old DB clients');const saved=await read(raw.transaction('checkpoints').objectStore('checkpoints').get([key,1]));assert(saved.checkpointVersion===1&&saved.bytes instanceof Blob&&!saved.image,'full cut is still a cloned document object');assert(await saved.bytes.text()===JSON.stringify(first),'Blob checkpoint changed serialized bytes');assert(saved.checksum===await hash(saved.bytes),'Blob checksum did not cover exact source bytes');
const old=await new Promise(resolve=>{const request=indexedDB.open(name,4);request.onerror=event=>{event.preventDefault();resolve(request.error.name);};request.onsuccess=()=>{request.result.close();resolve('accepted');};});assert(old==='VersionError','DB v4 reader accepted Blob checkpoint rows');
const write=async value=>{const tx=raw.transaction('checkpoints','readwrite'),terminal=done(tx);tx.objectStore('checkpoints').put(value,[key,1]);await terminal;};
for(const [kind,expected]of [['bytes','recovery_checksum_mismatch'],['header','recovery_checksum_mismatch'],['future','recovery_checkpoint_interpreter_unavailable'],['utf8','corrupt_recovery_image']]){
const damaged=structuredClone(saved);if(kind==='bytes')damaged.bytes=new Blob([JSON.stringify(image(9))]);else if(kind==='header')damaged.header.formatVersion=2;else if(kind==='future')damaged.checkpointVersion=2;else{damaged.bytes=new Blob([new Uint8Array([123,34,120,34,58,34,255,34,125])]);damaged.checksum=await hash(damaged.bytes);}
await write(damaged);let code;try{await storage.recover(leader);}catch(error){code=error.code;}assert(code===expected,`${kind} cut was accepted: ${code}`);if(kind==='future'){code=undefined;try{await storage.commit(leader,{image:image(2)});}catch(error){code=error.code;}assert(code===expected,'writer replaced a future checkpoint');}await write(saved);
}
const nativePut=IDBObjectStore.prototype.put;let injected=false,error;
IDBObjectStore.prototype.put=function(value,k){const request=nativePut.call(this,value,k);if(this.name==='checkpoints'&&value?.checkpointVersion===1&&value.header.rootRevision===2){injected=true;throw Error('abort_blob_checkpoint');}return request;};
try{await storage.commit(leader,{image:image(2),files:[{id:'orphan',bytes:new Blob(['must rollback'])}]});}catch(cause){error=cause.message;}finally{IDBObjectStore.prototype.put=nativePut;}
assert(injected&&error==='abort_blob_checkpoint','checkpoint fault was not injected after put');assert((await storage.recover(leader)).rootRevision===1&&await storage.readFile(leader,'orphan')===undefined,'aborted Blob cut leaked head or file bytes');
await leader.release();storage.close();storage=createBrowserStorage({name,checkpointEvery:1});leader=await storage.claim(scope);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(first),'fresh adapter lost Blob content');assert(await(await storage.readFile(leader,'kept')).text()==='kept bytes','Blob reopening lost atomic file');
const large=sealRecoveryImage(1,2,2,{rows:Array.from({length:16000},(_,i)=>({id:i,text:'x'.repeat(100)}))});let failure;const pending=storage.commit(leader,{image:large,files:[{id:'late',bytes:new Blob(['late'])}]}).catch(error=>{failure=error.code;});await Promise.resolve();await leader.release();await pending;assert(failure==='storage_leader_lost','released owner published or changed its cancellation error');leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===1&&await storage.readFile(leader,'late')===undefined,'released owner advanced Blob checkpoint');
}finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(name);}
}
export async function blockedUpgradeCase(assert){
const name=`blocked-upgrade-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},read=req=>new Promise((resolve,reject)=>{req.onsuccess=()=>resolve(req.result);req.onerror=()=>reject(req.error);});let raw=await read(indexedDB.open(name,4)),storage=createBrowserStorage({name}),leader;
try{let code;try{await storage.claim(scope);}catch(error){code=error.code;}assert(code==='storage_upgrade_blocked','active old connection did not reject the upgrade');code=undefined;try{await storage.claim(scope);}catch(error){code=error.code;}assert(code==='storage_upgrade_blocked','blocked adapter lost the original failure reason');raw.close();raw=await read(indexedDB.open(name));assert(raw.version===4,'abandoned upgrade changed database version after rejection');raw.close();storage.close();storage=createBrowserStorage({name});leader=await storage.claim(scope);assert(await storage.recover(leader)===null,'fresh adapter could not open after blocked request was retired');raw=await read(indexedDB.open(name));assert(raw.version===5,'fresh upgrade failed after abandoned open');}
finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(name);}
}
+10 -7
View File
@@ -1,3 +1,4 @@
import {blobCheckpointCase,blockedUpgradeCase} from './blob-checkpoint-browser-cases.mjs';
import {depotFilesCase,depotFilesUpgradeCase,depotAdoptionRaceCase} from './depot-files-browser-cases.mjs';
import {assetAdoptionStorageCase} from './asset-adoption-browser-cases.mjs';
import {assetDepotStorageCase} from './asset-depot-browser-cases.mjs';
@@ -500,7 +501,7 @@ async function deltaJournalCase(name,assert){
const checkpointHashes=[];crypto.subtle.digest=function(algorithm,bytes){checkpointHashes.push(bytes.byteLength);return digest.call(this,algorithm,bytes);};try{await storage.commit(leader,{image:image(3)});}finally{crypto.subtle.digest=digest;}assert(checkpointHashes.length===1&&checkpointHashes[0]===new TextEncoder().encode(JSON.stringify(image(3))).byteLength,'checkpoint did not hash the full image');assert((await storage.recover(leader)).rootRevision===3,'checkpoint after deltas failed');assert(await read(raw.transaction('journal').objectStore('journal').count())===0,'checkpoint did not compact prior deltas');
const detached=await storage.recover(leader);detached.payload.rows[0].text='caller mutation';const fourth=image(4),fifth=image(5),one=storage.commit(leader,{image:fourth}),two=storage.commit(leader,{image:fifth});fifth.payload.rows[0].text='late mutation';await Promise.all([one,two]);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(image(5)),'queued commits or recovered callers mutated the cached base');await storage.commit(leader,{image:image(5)});
const reordered=image(5);reordered.payload={ledger:reordered.payload.ledger,rows:reordered.payload.rows};code=undefined;try{await storage.commit(leader,{image:reordered});}catch(error){code=error.code;}assert(code==='storage_revision_conflict','same-cut retry accepted different serialized property order');
await storage.commit(leader,{image:image(6)});const extended={marker:'retained header',...image(7)};await storage.commit(leader,{image:extended});assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(extended),'noncanonical header lost fields or property order');assert((await read(raw.transaction('journal').objectStore('journal').get([key,7]))).image,'noncanonical header was encoded as a lossy delta');
await storage.commit(leader,{image:image(6)});const extended={marker:'retained header',...image(7)};await storage.commit(leader,{image:extended});assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(extended),'noncanonical header lost fields or property order');const fullRow=await read(raw.transaction('journal').objectStore('journal').get([key,7]));assert(fullRow.checkpointVersion===1&&fullRow.bytes instanceof Blob,'noncanonical header was encoded as a lossy delta');
const metadata=await read(raw.transaction('metadata').objectStore('metadata').get(key));let metadataTx=raw.transaction('metadata','readwrite'),metadataDone=done(metadataTx);const missingHead={...metadata};delete missingHead.throughRevision;metadataTx.objectStore('metadata').put(missingHead,key);await metadataDone;code=undefined;try{await storage.commit(leader,{image:image(8)});}catch(error){code=error.code;}assert(code==='recovery_delta_base_mismatch','cached baseline overwrote missing durable head');metadataTx=raw.transaction('metadata','readwrite');metadataDone=done(metadataTx);metadataTx.objectStore('metadata').put(metadata,key);await metadataDone;assert((await storage.recover(leader)).rootRevision===7,'head validation failure advanced cached state');
const oldError=await new Promise(resolve=>{const request=indexedDB.open(databaseName,1);request.onerror=event=>{event.preventDefault();resolve(request.error.name);};request.onsuccess=()=>{request.result.close();resolve('accepted');};});assert(oldError==='VersionError','old database reader accepted the delta format');
}finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(databaseName);}
@@ -717,7 +718,7 @@ async function loadingRecoveryCase(name,assert,coverage=false,scoped=false,struc
try{engine=await open();const target={id:'A',incarnation:1};assert(engine.editor.dispatch({type:'element.data',target,patch:{text:'local before tail'}}).status==='applied','loading field rejected');await engine.settle();assert(calls.length===0&&engine.getSnapshot().runtime.ledger.requests.size===0,'loading prefix froze or sent a request');await engine.close();
const owner=await storage.claim(scope),image=await storage.recover(owner);assert(image.formatVersion===format,'loading state did not require its versioned recovery envelope');decodeRecovery(image,registry);await owner.release();engine=await open();assert(engine.getSnapshot().document.elements.get('A').data.text==='local before tail','loading edit lost after IndexedDB reopen');assert(engine.editor.history.undo().status==='applied'&&engine.editor.history.redo().status==='applied','loading history did not reopen');await engine.settle();assert(calls.length===0,'history traversal escaped the loading hold');
const before=engine.getDurability().durableRootRevision,nativePut=IDBObjectStore.prototype.put;let injected=false,failure;
IDBObjectStore.prototype.put=function(value,key){const request=nativePut.call(this,value,key);if(!injected&&this.name==='checkpoints'&&value?.image?.formatVersion!==format){injected=true;throw Error('loading_full_cut_abort');}return request;};
IDBObjectStore.prototype.put=function(value,key){const request=nativePut.call(this,value,key);if(!injected&&this.name==='checkpoints'&&(value?.image?.formatVersion??value?.header?.formatVersion)!==format){injected=true;throw Error('loading_full_cut_abort');}return request;};
try{await engine.loading.complete(full,versions,{sourceToken:loading.sourceToken});}catch(error){failure=error.message;}finally{IDBObjectStore.prototype.put=nativePut;}
assert(injected&&failure==='loading_full_cut_abort','completion did not reach forced checkpoint rollback');const partial=await adapter.recover(ownerForTest);assert(partial.rootRevision===before&&partial.formatVersion===format&&calls.length===0,'aborted full checkpoint released loading writes or lost the partial cut');
await engine.persist();await engine.settle();
@@ -750,7 +751,9 @@ export async function run() {
try {
await depotAdoptionRaceCase(assert);results.push('real IndexedDB GC and a gated older Root cannot consume or release bytes belonging to a newer adoption cut');
await depotFilesCase(assert);results.push('stored uploads use shared-byte file pointers with original MIME, atomic promotion rollback, logical inventory and independent identity eviction/rehydration');
await depotFilesUpgradeCase(assert);results.push('DB v3 original files/checkpoints upgrade to v4 and lazily promote through an idempotent Root cut');
await blockedUpgradeCase(assert);results.push('blocked upgrades abort abandoned opens without upgrading after failure or leaking a connection');
await blobCheckpointCase(assert);results.push('DB v4 legacy checkpoints upgrade to v5 Blob cuts with exact bytes, stale-reader rejection and unchanged request bodies');results.push('Blob cut corruption, future format and atomic-file rollback preserve the previous durable Root');
await depotFilesUpgradeCase(assert);results.push('DB v3 original files/checkpoints upgrade to v5 and lazily promote through an idempotent Root cut');
await assetAdoptionStorageCase(assert);results.push('cross-project adoption uses fresh authorization, survives workspace/IndexedDB reopening and undo/redo, and preserves closed destination pins without project-local byte copies');
await assetDepotStorageCase(assert);results.push('workspace depot atomically shares digest bytes, preserves closed-project pins, blocks unknown/stale coverage and rolls back failed Root cuts');
leader = await storage.claim(scope);
@@ -792,10 +795,10 @@ export async function run() {
await other.commit(next, { image: image(4) });
const database = await new Promise((resolve, reject) => { const request = indexedDB.open(name); request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); });
const storageKey = [JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]), 4];
const tamper = database.transaction('journal', 'readwrite'), read = tamper.objectStore('journal').get(storageKey);
let intact;
read.onsuccess = () => { intact = read.result; const corrupted = structuredClone(intact); corrupted.image.payload.document.title = 'altered'; tamper.objectStore('journal').put(corrupted, storageKey); };
await new Promise((resolve, reject) => { tamper.oncomplete = resolve; tamper.onabort = () => reject(tamper.error); });
const intact=await new Promise((resolve,reject)=>{const request=database.transaction('journal').objectStore('journal').get(storageKey);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}),corrupted=structuredClone(intact);
if(corrupted.image)corrupted.image.payload.document.title='altered';else{const image=JSON.parse(await corrupted.bytes.text());image.payload.document.title='altered';corrupted.bytes=new Blob([JSON.stringify(image)]);}
const tamper=database.transaction('journal','readwrite');tamper.objectStore('journal').put(corrupted,storageKey);
await new Promise((resolve,reject)=>{tamper.oncomplete=resolve;tamper.onabort=()=>reject(tamper.error);});
await rejects(() => other.recover(next), 'recovery_checksum_mismatch');
results.push('well-formed data tampering is detected by the stored checksum');
const transaction = database.transaction('journal', 'readwrite');
+2 -2
View File
@@ -10,7 +10,7 @@ export async function depotFilesCase(assert){
const rejects=async(fn,code)=>{try{await fn();}catch(error){assert(error.code===code,`expected ${code}, got ${error.code}`);return;}throw Error(`expected ${code}`);};
const collect=now=>storage.assetDepot.collect(p,{authority,now,retentionMs:10,limit:128});
try{
p=await storage.claim(scope);q=await storage.claim({...scope,projectId:'q'});raw=await openRaw();assert(raw.version===4,'pointer storage did not upgrade to DB v4');
p=await storage.claim(scope);q=await storage.claim({...scope,projectId:'q'});raw=await openRaw();assert(raw.version===5,'pointer storage did not upgrade to DB v5');
await storage.commit(p,{image:image(0,[['one',resource('one',a.type,false)],['two',resource('two',b.type,false)]]),files:[{id:'one',bytes:a},{id:'two',bytes:b}],depot:cut(pins)});
assert(await get('files',[key,'one']) instanceof Blob,'unconfirmed upload lost its original bytes');
// The first pointer is converted before the second row fails. Everything must roll back.
@@ -30,7 +30,7 @@ export async function depotFilesUpgradeCase(assert){
const name=`depot-upgrade-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},key=JSON.stringify(['a','w','p']),authority='https://assets.test',bytes=new Blob(['old bytes'],{type:'image/png'}),hex=buffer=>[...new Uint8Array(buffer)].map(n=>n.toString(16).padStart(2,'0')).join(''),digest=hex(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer())),asset={id:'old',url:`${authority}/old`,width:2,height:1,bytes:bytes.size,mime_type:bytes.type},image={formatVersion:1,rootRevision:0,localRevision:0,payload:{runtime:{resources:[['old',{id:'old',digest,byteLength:bytes.size,mimeType:bytes.type,stored:asset}]]}}},checksum=hex(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(image))));
let raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name,3);request.onupgradeneeded=()=>{for(const store of ['metadata','checkpoints','journal','files','asset-depot','asset-bytes','asset-pins'])request.result.createObjectStore(store);};request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});
await new Promise((resolve,reject)=>{const tx=raw.transaction(['metadata','checkpoints','files'],'readwrite');tx.objectStore('metadata').put({fence:1,throughRevision:0,checkpointRevision:0},key);tx.objectStore('checkpoints').put({image,checksum},[key,0]);tx.objectStore('files').put(bytes,[key,'old']);tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);});raw.close();const storage=createBrowserStorage({name});let leader;
try{leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===0,'v3 checkpoint lost during upgrade');await storage.commit(leader,{image,depot:{version:1,authority,complete:true,pins:[['old',['element:old']]],writes:[{ref:{version:1,authority,digest,asset},bytes}]}});assert(await(await storage.readFile(leader,'old')).text()==='old bytes','idempotent reindex lost old bytes');raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const row=await new Promise((resolve,reject)=>{const request=raw.transaction('files').objectStore('files').get([key,'old']);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(raw.version===4&&row.fileVersion===1&&!(row instanceof Blob),'old project was not lazily promoted');raw.close();}finally{await leader?.release();storage.close();indexedDB.deleteDatabase(name);}
try{leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===0,'v3 checkpoint lost during upgrade');await storage.commit(leader,{image,depot:{version:1,authority,complete:true,pins:[['old',['element:old']]],writes:[{ref:{version:1,authority,digest,asset},bytes}]}});assert(await(await storage.readFile(leader,'old')).text()==='old bytes','idempotent reindex lost old bytes');raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const row=await new Promise((resolve,reject)=>{const request=raw.transaction('files').objectStore('files').get([key,'old']);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(raw.version===5&&row.fileVersion===1&&!(row instanceof Blob),'old project was not lazily promoted');raw.close();}finally{await leader?.release();storage.close();indexedDB.deleteDatabase(name);}
}
export async function depotAdoptionRaceCase(assert){
+17
View File
@@ -0,0 +1,17 @@
import {test} from 'node:test';
import assert from 'node:assert/strict';
import {sealRecoveryImage,sealedRecoveryByteLength} from '../../src/lib/canvas-runtime/recovery-encoding.ts';
import {serializeRecoveryImage} from '../../src/lib/canvas-runtime/recovery-json.ts';
const options=()=>{let clock=0,yields=0;return{value:{now:()=>clock+=10,yieldTask:async()=>{yields++;}},get yields(){return yields;}};};
test('chunked recovery JSON preserves native byte order, Unicode boundaries, escaping and scalar normalization',async()=>{
const values=['a'.repeat(4095)+'👀'+'\n\t"\\'.repeat(4000),'a'.repeat(4095)+'\ud800x\udc00','\u2028\u2029中文'.repeat(5000)],properties=Object.fromEntries(Array.from({length:1800},(_,i)=>[String(1800-i),i%3?values[i%values.length].slice(0,20):null]));Object.defineProperty(properties,'__proto__',{value:{keep:true},enumerable:true});
const image=sealRecoveryImage(17,8,5,{values,properties,scalars:[NaN,Infinity,-0,null,true,false],sparse:Array(3)}),scheduling=options(),blob=await serializeRecoveryImage(image,scheduling.value);assert.equal(await blob.text(),JSON.stringify(image));assert.equal(blob.size,new TextEncoder().encode(JSON.stringify(image)).byteLength);assert.equal(blob.size,sealedRecoveryByteLength(image));assert.ok(scheduling.yields>0);assert.deepEqual(JSON.parse(await blob.text()),JSON.parse(JSON.stringify(image)));
});
test('small owned subtree reuse does not reread caller getters or infer ownership from external freezing',async()=>{
let reads=0;const caller=Object.freeze({get text(){reads++;return'captured';}}),image=sealRecoveryImage(1,0,0,{rows:Array.from({length:1000},(_,i)=>({id:i,value:caller}))}),before=reads,a=await serializeRecoveryImage(image,options().value),b=await serializeRecoveryImage(image,options().value);assert.equal(reads,before);assert.equal(await a.text(),JSON.stringify(image));assert.equal(await b.text(),await a.text());
const external={formatVersion:1,rootRevision:0,localRevision:0,payload:{nested:'external'}};const pending=serializeRecoveryImage(external);external.payload.nested='late';assert.equal(JSON.parse(await(await pending).text()).payload.nested,'external');
});
test('coarse clocks yield for large images and a lost owner or host failure cannot return partial bytes',async()=>{
const image=sealRecoveryImage(1,1,1,{rows:Array.from({length:14000},(_,i)=>({id:i,text:'x'.repeat(100)}))});let yields=0;const blob=await serializeRecoveryImage(image,{now:()=>0,yieldTask:async()=>{yields++;}});assert.ok(yields>0);assert.equal(await blob.text(),JSON.stringify(image));
const failure=Error('owner lost');let gone=false,clock=0;await assert.rejects(serializeRecoveryImage(image,{now:()=>clock+=10,yieldTask:async()=>{gone=true;},check(){if(gone)throw failure;}}),error=>error===failure);clock=0;await assert.rejects(serializeRecoveryImage(image,{now:()=>clock+=10,yieldTask:async()=>{throw failure;}}),error=>error===failure);
});