diff --git a/headless/README.md b/headless/README.md index c39d37d..e070eb6 100644 --- a/headless/README.md +++ b/headless/README.md @@ -230,7 +230,7 @@ formats 1–12 and reject unknown future formats. A reopened binding can display shared local bytes, and undo/redo reuse the asset without uploading again. The workspace closes projects before its depot; retiring the identity fences late work. -The browser adapter uses IndexedDB version 4. After a proven stored-upload cut, a +The browser adapter uses IndexedDB version 5 (files retain their version-1 pointer format). After a proven stored-upload cut, a project's `files` row holds an opaque pointer to digest-addressed `asset-bytes`. Unconfirmed uploads and import source files keep their original Blob rows. Use `readFile` / `scanFiles`; direct assumptions that every raw files-store row is a @@ -362,7 +362,8 @@ same bytes. Legal prefixes survive malformed array/footer tails with an explicit partial result. Headers following the elements array require indexed preparation to establish ownership first. Legacy parser 4 jobs keep their indexed interpreter. New document jobs use recovery v16; legacy document jobs keep v15 (readers accept -1–16, reject 17), without changing DB v4 or the wire protocol. Resource-bearing +1–16, reject 17). That parser change retained DB v4 and the wire protocol; the +subsequent checkpoint representation upgrade uses DB v5 separately. Resource-bearing records use fresh authorization when the host provides a depot and authority; missing capabilities or denied resources produce explicit partial results. Complete service acceptance remains pending. Source SHA verification still reads @@ -957,3 +958,31 @@ resource authorization limits, durable send barriers and source ordering remain in force. Hosts that set a lower runner ceiling retain that ordinary-batch bound. An indivisible SCC can exceed the soft unit limit under its existing hard byte and change-count limits. + +Full browser recovery cuts now use a versioned envelope: +`{checkpointVersion:1, header:{formatVersion,rootRevision,localRevision}, bytes:Blob, checksum}`. +The Blob contains exact UTF-8 `JSON.stringify(image)` bytes, not another document +model. It can represent a checkpoint or a full replacement journal cut. Delta +journals retain versions 1/2; legacy object cuts remain readable. DB v5 fences DB +v4 clients before these rows are written. Recovery envelope/wire versions do not +change. Rolling application code back requires retaining the v5 storage reader; +an old v4 client cannot open the upgraded database. + +For codec-owned sealed images, serialization splits large containers/strings and +uses existing codec byte facts to stringify small owned subtrees natively. It +preserves key order, escaping and surrogate pairs, yielding on the shared soft +time/work budget before starting an IDB write transaction. External images retain +the defensive capture/native JSON path. Each yield rechecks the storage owner. +The complete Blob and checksum are prepared first; head, cut, files and depot +changes still commit in one transaction. No Root is skipped. Reopening validates +headers/sequence, hashes exact Blob bytes, decodes strict UTF-8, checks the decoded +header and then applies later deltas. Future checkpoint versions reject reads +and writes; malformed/tampered data is never treated as an empty document. Legacy +recovery, JSON parse, recovery-image encoding, same-cut retry comparison and +individual native operations still have synchronous work; this is not a strict +6ms bound for all storage paths or a disk-space reduction guarantee. + +A blocked schema upgrade retires its pending open. If the old connection closes +later, that abandoned request aborts in `onupgradeneeded` rather than silently +upgrading after rejection. A fresh adapter can retry once the blocking client has +closed; closing an adapter before its upgrade event also aborts the pending open. diff --git a/src/lib/canvas-runtime/recovery-encoding.ts b/src/lib/canvas-runtime/recovery-encoding.ts index 7027c1f..6a031a7 100644 --- a/src/lib/canvas-runtime/recovery-encoding.ts +++ b/src/lib/canvas-runtime/recovery-encoding.ts @@ -62,3 +62,6 @@ export function isSealedRecoveryImage(image:RecoveryImage):boolean{return sealed /** Exact UTF-8 JSON size, available only for codec-owned immutable images. */ export function sealedRecoveryByteLength(image:RecoveryImage){return imageSizes.get(image);} + +/** Internal codec ownership/size fact, never inferred from external Object.freeze. */ +export function encodedRecoveryByteLength(value:object){return encodedSizes.get(value);} diff --git a/src/lib/canvas-runtime/recovery-json.ts b/src/lib/canvas-runtime/recovery-json.ts new file mode 100644 index 0000000..b57a060 --- /dev/null +++ b/src/lib/canvas-runtime/recovery-json.ts @@ -0,0 +1,32 @@ +import type {RecoveryImage} from './storage-port.ts'; +import {isSealedRecoveryImage,sealedRecoveryByteLength,encodedRecoveryByteLength} from './recovery-encoding.ts'; +import {createImportBudget,type ImportScheduling} from './import-budget.ts'; +import {fault} from '../canvas-core/values.ts'; + +function* stringTokens(value:string):Generator{ + if(value.length<=4096){yield JSON.stringify(value);return;} + yield '"'; + for(let start=0;start=0xd800&&a<=0xdbff&&b>=0xdc00&&b<=0xdfff)end--;yield JSON.stringify(value.slice(start,end)).slice(1,-1);start=end;} + yield '"'; +} +/** Only traverse codec-owned normalized JSON. Small owned subtrees use native stringify. */ +function* tokens(value:unknown):Generator{ + if(value&&typeof value==='object'){ + const size=encodedRecoveryByteLength(value);if(size!==undefined&&size<=16384){yield JSON.stringify(value);return;} + if(Array.isArray(value)){yield '[';for(let i=0;i)[key]);}yield '}';return; + } + if(typeof value==='string')yield* stringTokens(value);else yield JSON.stringify(value); +} +/** Preserve exact JSON.stringify bytes; scheduling never exposes a partial recovery image. */ +export async function serializeRecoveryImage(image:RecoveryImage,options:ImportScheduling&{check?:()=>void}={}):Promise{ + options.check?.(); + if(!isSealedRecoveryImage(image)){const bytes=new Blob([JSON.stringify(image)],{type:'application/json'});options.check?.();return bytes;} + const budget=createImportBudget(new AbortController().signal,options),parts:BlobPart[]=[];let pending:string[]=[],length=0; + for(const token of tokens(image)){ + pending.push(token);length+=token.length; + if(length>=8192){parts.push(pending.join(''));pending=[];length=0;options.check?.();await budget.step(128);options.check?.();} + } + if(pending.length)parts.push(pending.join(''));options.check?.(); + const bytes=new Blob(parts,{type:'application/json'});if(bytes.size!==sealedRecoveryByteLength(image))throw fault('recovery_encoding_size_mismatch');return bytes; +} diff --git a/src/lib/canvas-runtime/storage.ts b/src/lib/canvas-runtime/storage.ts index 2a598c8..17c570c 100644 --- a/src/lib/canvas-runtime/storage.ts +++ b/src/lib/canvas-runtime/storage.ts @@ -1,3 +1,4 @@ +import {serializeRecoveryImage} from './recovery-json.ts'; import {depotStores,createDepotStorage,detachDepotCut,verifyDepotBytes,writeDepotCut,promoteDepotFiles,resolveDepotFile} from './asset-depot-storage.ts'; import { applyJournal, diffJournal, type JournalPatch } from './journal-delta.ts'; import { isSealedRecoveryImage, sealedRecoveryByteLength } from './recovery-encoding.ts'; @@ -9,10 +10,13 @@ export type {StorageScope,RecoveryImage,DurableBatch,StorageLeader,StoredFileInf type Metadata = { fence: number; throughRevision?: number; checkpointRevision?: number }; type StoredImage = { readonly image: RecoveryImage; readonly checksum: string }; type StoredDelta = {journalVersion:1|2;baseRevision:number;baseChecksum:string;header:Omit;patches:JournalPatch[];checksum:string}; -type StoredCut = StoredImage | StoredDelta; +type StoredBlobImage={readonly checkpointVersion:1;readonly header:Omit;readonly bytes:Blob;readonly checksum:string}; +type StoredCut = StoredImage | StoredDelta | StoredBlobImage; +const isDelta=(cut:StoredCut):cut is StoredDelta=>!('image' in cut)&&!('checkpointVersion' in cut); function cutHeader(cut:StoredCut):Omit{ if(!cut||typeof cut!=='object')throw fault('corrupt_recovery_image'); if('image' in cut){validateImage(cut.image);return cut.image;} + if('checkpointVersion' in cut){if(cut.checkpointVersion!==1)throw fault('recovery_checkpoint_interpreter_unavailable');if(!(cut.bytes instanceof Blob)||!cut.header)throw fault('corrupt_recovery_image');validateImage({...cut.header,payload:null});return cut.header;} if(![1,2].includes(cut.journalVersion))throw fault('recovery_delta_interpreter_unavailable'); if(!cut.header)throw fault('corrupt_recovery_delta');validateImage({...cut.header,payload:null});return cut.header; } @@ -20,6 +24,8 @@ async function imageDigest(image: unknown) { const bytes = new TextEncoder().encode(JSON.stringify(image)); return {checksum:[...new Uint8Array(await crypto.subtle.digest('SHA-256', bytes))].map(value => value.toString(16).padStart(2, '0')).join(''),byteLength:bytes.byteLength}; } +async function bytesDigest(bytes:ArrayBuffer){return [...new Uint8Array(await crypto.subtle.digest('SHA-256',bytes))].map(value=>value.toString(16).padStart(2,'0')).join('');} +async function blobDigest(bytes:Blob){return bytesDigest(await bytes.arrayBuffer());} function deltaBody(cut:StoredDelta){const {journalVersion,baseRevision,baseChecksum,header,patches}=cut;return{journalVersion,baseRevision,baseChecksum,header,patches};} const keyFor = (scope: StorageScope) => { if (![scope.accountId, scope.workspaceId, scope.projectId].every(value => typeof value === 'string' && value.length > 0)) throw fault('invalid_storage_scope'); @@ -44,17 +50,18 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: if (!factory || !locks) throw fault('persistent_storage_unavailable'); const checkpointEvery = options.checkpointEvery ?? 32; if (!Number.isSafeInteger(checkpointEvery) || checkpointEvery < 1) throw fault('invalid_checkpoint_interval'); - let closed = false; + let closed = false, openingFailure:Error|undefined; const handles = new Set(), lastCuts=new WeakMap(), writeTails=new WeakMap>(),lastCheckpoints=new WeakMap(); const opening = new Promise((resolve, reject) => { - const request = factory.open(options.name ?? 'endoleap-headless', 4); + const request = factory.open(options.name ?? 'endoleap-headless', 5); request.onupgradeneeded = () => { + if(closed||openingFailure){request.transaction?.abort();return;} for (const name of ['metadata', 'checkpoints', 'journal', 'files',...depotStores]) if (!request.result.objectStoreNames.contains(name)) request.result.createObjectStore(name); }; - request.onerror = () => reject(request.error); - request.onblocked = () => reject(fault('storage_upgrade_blocked')); + request.onerror = () => reject(closed?fault('storage_closed'):openingFailure??request.error); + request.onblocked = () => {openingFailure=fault('storage_upgrade_blocked');reject(openingFailure);}; request.onsuccess = () => { - if (closed) { request.result.close(); reject(fault('storage_closed')); return; } + if (closed||openingFailure) { request.result.close(); reject(closed?fault('storage_closed'):openingFailure); return; } request.result.onversionchange = () => { closed = true; request.result.close(); for (const handle of handles) void handle.release(); }; resolve(request.result); }; @@ -83,9 +90,9 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: const captured=await execute(leader,['checkpoints','journal'],'readonly',async(transaction,metadata,key)=>{ if(metadata.throughRevision===undefined)return{cuts:[] as StoredCut[],checkpointRevision:undefined}; if(metadata.checkpointRevision===undefined)throw fault('recovery_checkpoint_missing'); - const checkpoint=await requestValue(transaction.objectStore('checkpoints').get([key,metadata.checkpointRevision])); - if(!checkpoint?.image)throw fault('recovery_checkpoint_missing'); - let header=cutHeader(checkpoint);if(header.rootRevision!==metadata.checkpointRevision)throw fault('corrupt_recovery_image'); + const checkpoint=await requestValue(transaction.objectStore('checkpoints').get([key,metadata.checkpointRevision])); + if(!checkpoint)throw fault('recovery_checkpoint_missing'); + let header=cutHeader(checkpoint);if(isDelta(checkpoint))throw fault('recovery_checkpoint_missing');if(header.rootRevision!==metadata.checkpointRevision)throw fault('corrupt_recovery_image'); const journal=metadata.checkpointRevision===metadata.throughRevision?[]:await requestValue(transaction.objectStore('journal').getAll(IDBKeyRange.bound([key,metadata.checkpointRevision+1],[key,metadata.throughRevision]))); for(const row of journal){const next=cutHeader(row);if(next.rootRevision!==header.rootRevision+1||next.localRevisioncheck(leader)}),checksum=await blobDigest(bytes);stored={image,checksum};candidate={checkpointVersion:1,header:{formatVersion:image.formatVersion,rootRevision:image.rootRevision,localRevision:image.localRevision},bytes,checksum};} const accepted=stored,cut=candidate; const receipt=await execute(leader,['checkpoints','journal','files',...depotStores],'readwrite',async(transaction,metadata,key)=>{ const previous=metadata.throughRevision; @@ -174,14 +189,14 @@ export function createBrowserStorage(options: { indexedDB?: IDBFactory; locks?: } if(previous!==undefined&&(!saved||image.localRevision({formatVersion:1,rootRevision:revision,localRevision:revision,payload:{rows:Array.from({length:250},(_,i)=>({id:i,text:`版本 ${revision} 👀 ${'x'.repeat(100)}`})),request:{id:'frozen',body:'{"exact":"original bytes"}'}}}); + const read=req=>new Promise((resolve,reject)=>{req.onsuccess=()=>resolve(req.result);req.onerror=()=>reject(req.error);}),done=tx=>new Promise((resolve,reject)=>{tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);}),hash=async blob=>[...new Uint8Array(await crypto.subtle.digest('SHA-256',await blob.arrayBuffer()))].map(n=>n.toString(16).padStart(2,'0')).join(''); + let raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name,4);request.onupgradeneeded=()=>{for(const store of ['metadata','checkpoints','journal','files'])request.result.createObjectStore(store);};request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); + const initial=image(0),checksum=await hash(new Blob([JSON.stringify(initial)]));let tx=raw.transaction(['metadata','checkpoints'],'readwrite'),terminal=done(tx);tx.objectStore('metadata').put({fence:0,throughRevision:0,checkpointRevision:0},key);tx.objectStore('checkpoints').put({image:initial,checksum},[key,0]);await terminal;raw.close();let storage=createBrowserStorage({name,checkpointEvery:1}),leader; + try{ + leader=await storage.claim(scope);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(initial),'DB v4 legacy checkpoint changed during upgrade');const first=image(1);await storage.commit(leader,{image:sealRecoveryImage(1,1,1,first.payload),files:[{id:'kept',bytes:new Blob(['kept bytes'])}]}); + raw=await read(indexedDB.open(name));assert(raw.version===5,'Blob cuts did not fence old DB clients');const saved=await read(raw.transaction('checkpoints').objectStore('checkpoints').get([key,1]));assert(saved.checkpointVersion===1&&saved.bytes instanceof Blob&&!saved.image,'full cut is still a cloned document object');assert(await saved.bytes.text()===JSON.stringify(first),'Blob checkpoint changed serialized bytes');assert(saved.checksum===await hash(saved.bytes),'Blob checksum did not cover exact source bytes'); + const old=await new Promise(resolve=>{const request=indexedDB.open(name,4);request.onerror=event=>{event.preventDefault();resolve(request.error.name);};request.onsuccess=()=>{request.result.close();resolve('accepted');};});assert(old==='VersionError','DB v4 reader accepted Blob checkpoint rows'); + const write=async value=>{const tx=raw.transaction('checkpoints','readwrite'),terminal=done(tx);tx.objectStore('checkpoints').put(value,[key,1]);await terminal;}; + for(const [kind,expected]of [['bytes','recovery_checksum_mismatch'],['header','recovery_checksum_mismatch'],['future','recovery_checkpoint_interpreter_unavailable'],['utf8','corrupt_recovery_image']]){ + const damaged=structuredClone(saved);if(kind==='bytes')damaged.bytes=new Blob([JSON.stringify(image(9))]);else if(kind==='header')damaged.header.formatVersion=2;else if(kind==='future')damaged.checkpointVersion=2;else{damaged.bytes=new Blob([new Uint8Array([123,34,120,34,58,34,255,34,125])]);damaged.checksum=await hash(damaged.bytes);} + await write(damaged);let code;try{await storage.recover(leader);}catch(error){code=error.code;}assert(code===expected,`${kind} cut was accepted: ${code}`);if(kind==='future'){code=undefined;try{await storage.commit(leader,{image:image(2)});}catch(error){code=error.code;}assert(code===expected,'writer replaced a future checkpoint');}await write(saved); + } + const nativePut=IDBObjectStore.prototype.put;let injected=false,error; + IDBObjectStore.prototype.put=function(value,k){const request=nativePut.call(this,value,k);if(this.name==='checkpoints'&&value?.checkpointVersion===1&&value.header.rootRevision===2){injected=true;throw Error('abort_blob_checkpoint');}return request;}; + try{await storage.commit(leader,{image:image(2),files:[{id:'orphan',bytes:new Blob(['must rollback'])}]});}catch(cause){error=cause.message;}finally{IDBObjectStore.prototype.put=nativePut;} + assert(injected&&error==='abort_blob_checkpoint','checkpoint fault was not injected after put');assert((await storage.recover(leader)).rootRevision===1&&await storage.readFile(leader,'orphan')===undefined,'aborted Blob cut leaked head or file bytes'); + await leader.release();storage.close();storage=createBrowserStorage({name,checkpointEvery:1});leader=await storage.claim(scope);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(first),'fresh adapter lost Blob content');assert(await(await storage.readFile(leader,'kept')).text()==='kept bytes','Blob reopening lost atomic file'); + const large=sealRecoveryImage(1,2,2,{rows:Array.from({length:16000},(_,i)=>({id:i,text:'x'.repeat(100)}))});let failure;const pending=storage.commit(leader,{image:large,files:[{id:'late',bytes:new Blob(['late'])}]}).catch(error=>{failure=error.code;});await Promise.resolve();await leader.release();await pending;assert(failure==='storage_leader_lost','released owner published or changed its cancellation error');leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===1&&await storage.readFile(leader,'late')===undefined,'released owner advanced Blob checkpoint'); + }finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(name);} +} + +export async function blockedUpgradeCase(assert){ + const name=`blocked-upgrade-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},read=req=>new Promise((resolve,reject)=>{req.onsuccess=()=>resolve(req.result);req.onerror=()=>reject(req.error);});let raw=await read(indexedDB.open(name,4)),storage=createBrowserStorage({name}),leader; + try{let code;try{await storage.claim(scope);}catch(error){code=error.code;}assert(code==='storage_upgrade_blocked','active old connection did not reject the upgrade');code=undefined;try{await storage.claim(scope);}catch(error){code=error.code;}assert(code==='storage_upgrade_blocked','blocked adapter lost the original failure reason');raw.close();raw=await read(indexedDB.open(name));assert(raw.version===4,'abandoned upgrade changed database version after rejection');raw.close();storage.close();storage=createBrowserStorage({name});leader=await storage.claim(scope);assert(await storage.recover(leader)===null,'fresh adapter could not open after blocked request was retired');raw=await read(indexedDB.open(name));assert(raw.version===5,'fresh upgrade failed after abandoned open');} + finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(name);} +} diff --git a/tests/headless/browser-storage-cases.mjs b/tests/headless/browser-storage-cases.mjs index d3d2706..fbed132 100644 --- a/tests/headless/browser-storage-cases.mjs +++ b/tests/headless/browser-storage-cases.mjs @@ -1,3 +1,4 @@ +import {blobCheckpointCase,blockedUpgradeCase} from './blob-checkpoint-browser-cases.mjs'; import {depotFilesCase,depotFilesUpgradeCase,depotAdoptionRaceCase} from './depot-files-browser-cases.mjs'; import {assetAdoptionStorageCase} from './asset-adoption-browser-cases.mjs'; import {assetDepotStorageCase} from './asset-depot-browser-cases.mjs'; @@ -500,7 +501,7 @@ async function deltaJournalCase(name,assert){ const checkpointHashes=[];crypto.subtle.digest=function(algorithm,bytes){checkpointHashes.push(bytes.byteLength);return digest.call(this,algorithm,bytes);};try{await storage.commit(leader,{image:image(3)});}finally{crypto.subtle.digest=digest;}assert(checkpointHashes.length===1&&checkpointHashes[0]===new TextEncoder().encode(JSON.stringify(image(3))).byteLength,'checkpoint did not hash the full image');assert((await storage.recover(leader)).rootRevision===3,'checkpoint after deltas failed');assert(await read(raw.transaction('journal').objectStore('journal').count())===0,'checkpoint did not compact prior deltas'); const detached=await storage.recover(leader);detached.payload.rows[0].text='caller mutation';const fourth=image(4),fifth=image(5),one=storage.commit(leader,{image:fourth}),two=storage.commit(leader,{image:fifth});fifth.payload.rows[0].text='late mutation';await Promise.all([one,two]);assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(image(5)),'queued commits or recovered callers mutated the cached base');await storage.commit(leader,{image:image(5)}); const reordered=image(5);reordered.payload={ledger:reordered.payload.ledger,rows:reordered.payload.rows};code=undefined;try{await storage.commit(leader,{image:reordered});}catch(error){code=error.code;}assert(code==='storage_revision_conflict','same-cut retry accepted different serialized property order'); - await storage.commit(leader,{image:image(6)});const extended={marker:'retained header',...image(7)};await storage.commit(leader,{image:extended});assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(extended),'noncanonical header lost fields or property order');assert((await read(raw.transaction('journal').objectStore('journal').get([key,7]))).image,'noncanonical header was encoded as a lossy delta'); + await storage.commit(leader,{image:image(6)});const extended={marker:'retained header',...image(7)};await storage.commit(leader,{image:extended});assert(JSON.stringify(await storage.recover(leader))===JSON.stringify(extended),'noncanonical header lost fields or property order');const fullRow=await read(raw.transaction('journal').objectStore('journal').get([key,7]));assert(fullRow.checkpointVersion===1&&fullRow.bytes instanceof Blob,'noncanonical header was encoded as a lossy delta'); const metadata=await read(raw.transaction('metadata').objectStore('metadata').get(key));let metadataTx=raw.transaction('metadata','readwrite'),metadataDone=done(metadataTx);const missingHead={...metadata};delete missingHead.throughRevision;metadataTx.objectStore('metadata').put(missingHead,key);await metadataDone;code=undefined;try{await storage.commit(leader,{image:image(8)});}catch(error){code=error.code;}assert(code==='recovery_delta_base_mismatch','cached baseline overwrote missing durable head');metadataTx=raw.transaction('metadata','readwrite');metadataDone=done(metadataTx);metadataTx.objectStore('metadata').put(metadata,key);await metadataDone;assert((await storage.recover(leader)).rootRevision===7,'head validation failure advanced cached state'); const oldError=await new Promise(resolve=>{const request=indexedDB.open(databaseName,1);request.onerror=event=>{event.preventDefault();resolve(request.error.name);};request.onsuccess=()=>{request.result.close();resolve('accepted');};});assert(oldError==='VersionError','old database reader accepted the delta format'); }finally{raw?.close();await leader?.release();storage.close();indexedDB.deleteDatabase(databaseName);} @@ -717,7 +718,7 @@ async function loadingRecoveryCase(name,assert,coverage=false,scoped=false,struc try{engine=await open();const target={id:'A',incarnation:1};assert(engine.editor.dispatch({type:'element.data',target,patch:{text:'local before tail'}}).status==='applied','loading field rejected');await engine.settle();assert(calls.length===0&&engine.getSnapshot().runtime.ledger.requests.size===0,'loading prefix froze or sent a request');await engine.close(); const owner=await storage.claim(scope),image=await storage.recover(owner);assert(image.formatVersion===format,'loading state did not require its versioned recovery envelope');decodeRecovery(image,registry);await owner.release();engine=await open();assert(engine.getSnapshot().document.elements.get('A').data.text==='local before tail','loading edit lost after IndexedDB reopen');assert(engine.editor.history.undo().status==='applied'&&engine.editor.history.redo().status==='applied','loading history did not reopen');await engine.settle();assert(calls.length===0,'history traversal escaped the loading hold'); const before=engine.getDurability().durableRootRevision,nativePut=IDBObjectStore.prototype.put;let injected=false,failure; - IDBObjectStore.prototype.put=function(value,key){const request=nativePut.call(this,value,key);if(!injected&&this.name==='checkpoints'&&value?.image?.formatVersion!==format){injected=true;throw Error('loading_full_cut_abort');}return request;}; + IDBObjectStore.prototype.put=function(value,key){const request=nativePut.call(this,value,key);if(!injected&&this.name==='checkpoints'&&(value?.image?.formatVersion??value?.header?.formatVersion)!==format){injected=true;throw Error('loading_full_cut_abort');}return request;}; try{await engine.loading.complete(full,versions,{sourceToken:loading.sourceToken});}catch(error){failure=error.message;}finally{IDBObjectStore.prototype.put=nativePut;} assert(injected&&failure==='loading_full_cut_abort','completion did not reach forced checkpoint rollback');const partial=await adapter.recover(ownerForTest);assert(partial.rootRevision===before&&partial.formatVersion===format&&calls.length===0,'aborted full checkpoint released loading writes or lost the partial cut'); await engine.persist();await engine.settle(); @@ -750,7 +751,9 @@ export async function run() { try { await depotAdoptionRaceCase(assert);results.push('real IndexedDB GC and a gated older Root cannot consume or release bytes belonging to a newer adoption cut'); await depotFilesCase(assert);results.push('stored uploads use shared-byte file pointers with original MIME, atomic promotion rollback, logical inventory and independent identity eviction/rehydration'); - await depotFilesUpgradeCase(assert);results.push('DB v3 original files/checkpoints upgrade to v4 and lazily promote through an idempotent Root cut'); + await blockedUpgradeCase(assert);results.push('blocked upgrades abort abandoned opens without upgrading after failure or leaking a connection'); + await blobCheckpointCase(assert);results.push('DB v4 legacy checkpoints upgrade to v5 Blob cuts with exact bytes, stale-reader rejection and unchanged request bodies');results.push('Blob cut corruption, future format and atomic-file rollback preserve the previous durable Root'); + await depotFilesUpgradeCase(assert);results.push('DB v3 original files/checkpoints upgrade to v5 and lazily promote through an idempotent Root cut'); await assetAdoptionStorageCase(assert);results.push('cross-project adoption uses fresh authorization, survives workspace/IndexedDB reopening and undo/redo, and preserves closed destination pins without project-local byte copies'); await assetDepotStorageCase(assert);results.push('workspace depot atomically shares digest bytes, preserves closed-project pins, blocks unknown/stale coverage and rolls back failed Root cuts'); leader = await storage.claim(scope); @@ -792,10 +795,10 @@ export async function run() { await other.commit(next, { image: image(4) }); const database = await new Promise((resolve, reject) => { const request = indexedDB.open(name); request.onsuccess = () => resolve(request.result); request.onerror = () => reject(request.error); }); const storageKey = [JSON.stringify([scope.accountId, scope.workspaceId, scope.projectId]), 4]; - const tamper = database.transaction('journal', 'readwrite'), read = tamper.objectStore('journal').get(storageKey); - let intact; - read.onsuccess = () => { intact = read.result; const corrupted = structuredClone(intact); corrupted.image.payload.document.title = 'altered'; tamper.objectStore('journal').put(corrupted, storageKey); }; - await new Promise((resolve, reject) => { tamper.oncomplete = resolve; tamper.onabort = () => reject(tamper.error); }); + const intact=await new Promise((resolve,reject)=>{const request=database.transaction('journal').objectStore('journal').get(storageKey);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}),corrupted=structuredClone(intact); + if(corrupted.image)corrupted.image.payload.document.title='altered';else{const image=JSON.parse(await corrupted.bytes.text());image.payload.document.title='altered';corrupted.bytes=new Blob([JSON.stringify(image)]);} + const tamper=database.transaction('journal','readwrite');tamper.objectStore('journal').put(corrupted,storageKey); + await new Promise((resolve,reject)=>{tamper.oncomplete=resolve;tamper.onabort=()=>reject(tamper.error);}); await rejects(() => other.recover(next), 'recovery_checksum_mismatch'); results.push('well-formed data tampering is detected by the stored checksum'); const transaction = database.transaction('journal', 'readwrite'); diff --git a/tests/headless/depot-files-browser-cases.mjs b/tests/headless/depot-files-browser-cases.mjs index 8061161..e00127c 100644 --- a/tests/headless/depot-files-browser-cases.mjs +++ b/tests/headless/depot-files-browser-cases.mjs @@ -10,7 +10,7 @@ export async function depotFilesCase(assert){ const rejects=async(fn,code)=>{try{await fn();}catch(error){assert(error.code===code,`expected ${code}, got ${error.code}`);return;}throw Error(`expected ${code}`);}; const collect=now=>storage.assetDepot.collect(p,{authority,now,retentionMs:10,limit:128}); try{ - p=await storage.claim(scope);q=await storage.claim({...scope,projectId:'q'});raw=await openRaw();assert(raw.version===4,'pointer storage did not upgrade to DB v4'); + p=await storage.claim(scope);q=await storage.claim({...scope,projectId:'q'});raw=await openRaw();assert(raw.version===5,'pointer storage did not upgrade to DB v5'); await storage.commit(p,{image:image(0,[['one',resource('one',a.type,false)],['two',resource('two',b.type,false)]]),files:[{id:'one',bytes:a},{id:'two',bytes:b}],depot:cut(pins)}); assert(await get('files',[key,'one']) instanceof Blob,'unconfirmed upload lost its original bytes'); // The first pointer is converted before the second row fails. Everything must roll back. @@ -30,7 +30,7 @@ export async function depotFilesUpgradeCase(assert){ const name=`depot-upgrade-${crypto.randomUUID()}`,scope={accountId:'a',workspaceId:'w',projectId:'p'},key=JSON.stringify(['a','w','p']),authority='https://assets.test',bytes=new Blob(['old bytes'],{type:'image/png'}),hex=buffer=>[...new Uint8Array(buffer)].map(n=>n.toString(16).padStart(2,'0')).join(''),digest=hex(await crypto.subtle.digest('SHA-256',await bytes.arrayBuffer())),asset={id:'old',url:`${authority}/old`,width:2,height:1,bytes:bytes.size,mime_type:bytes.type},image={formatVersion:1,rootRevision:0,localRevision:0,payload:{runtime:{resources:[['old',{id:'old',digest,byteLength:bytes.size,mimeType:bytes.type,stored:asset}]]}}},checksum=hex(await crypto.subtle.digest('SHA-256',new TextEncoder().encode(JSON.stringify(image)))); let raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name,3);request.onupgradeneeded=()=>{for(const store of ['metadata','checkpoints','journal','files','asset-depot','asset-bytes','asset-pins'])request.result.createObjectStore(store);};request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);}); await new Promise((resolve,reject)=>{const tx=raw.transaction(['metadata','checkpoints','files'],'readwrite');tx.objectStore('metadata').put({fence:1,throughRevision:0,checkpointRevision:0},key);tx.objectStore('checkpoints').put({image,checksum},[key,0]);tx.objectStore('files').put(bytes,[key,'old']);tx.oncomplete=resolve;tx.onabort=()=>reject(tx.error);});raw.close();const storage=createBrowserStorage({name});let leader; - try{leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===0,'v3 checkpoint lost during upgrade');await storage.commit(leader,{image,depot:{version:1,authority,complete:true,pins:[['old',['element:old']]],writes:[{ref:{version:1,authority,digest,asset},bytes}]}});assert(await(await storage.readFile(leader,'old')).text()==='old bytes','idempotent reindex lost old bytes');raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const row=await new Promise((resolve,reject)=>{const request=raw.transaction('files').objectStore('files').get([key,'old']);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(raw.version===4&&row.fileVersion===1&&!(row instanceof Blob),'old project was not lazily promoted');raw.close();}finally{await leader?.release();storage.close();indexedDB.deleteDatabase(name);} + try{leader=await storage.claim(scope);assert((await storage.recover(leader)).rootRevision===0,'v3 checkpoint lost during upgrade');await storage.commit(leader,{image,depot:{version:1,authority,complete:true,pins:[['old',['element:old']]],writes:[{ref:{version:1,authority,digest,asset},bytes}]}});assert(await(await storage.readFile(leader,'old')).text()==='old bytes','idempotent reindex lost old bytes');raw=await new Promise((resolve,reject)=>{const request=indexedDB.open(name);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});const row=await new Promise((resolve,reject)=>{const request=raw.transaction('files').objectStore('files').get([key,'old']);request.onsuccess=()=>resolve(request.result);request.onerror=()=>reject(request.error);});assert(raw.version===5&&row.fileVersion===1&&!(row instanceof Blob),'old project was not lazily promoted');raw.close();}finally{await leader?.release();storage.close();indexedDB.deleteDatabase(name);} } export async function depotAdoptionRaceCase(assert){ diff --git a/tests/headless/recovery-json.test.mjs b/tests/headless/recovery-json.test.mjs new file mode 100644 index 0000000..de4b124 --- /dev/null +++ b/tests/headless/recovery-json.test.mjs @@ -0,0 +1,17 @@ +import {test} from 'node:test'; +import assert from 'node:assert/strict'; +import {sealRecoveryImage,sealedRecoveryByteLength} from '../../src/lib/canvas-runtime/recovery-encoding.ts'; +import {serializeRecoveryImage} from '../../src/lib/canvas-runtime/recovery-json.ts'; +const options=()=>{let clock=0,yields=0;return{value:{now:()=>clock+=10,yieldTask:async()=>{yields++;}},get yields(){return yields;}};}; +test('chunked recovery JSON preserves native byte order, Unicode boundaries, escaping and scalar normalization',async()=>{ + const values=['a'.repeat(4095)+'👀'+'\n\t"\\'.repeat(4000),'a'.repeat(4095)+'\ud800x\udc00','\u2028\u2029中文'.repeat(5000)],properties=Object.fromEntries(Array.from({length:1800},(_,i)=>[String(1800-i),i%3?values[i%values.length].slice(0,20):null]));Object.defineProperty(properties,'__proto__',{value:{keep:true},enumerable:true}); + const image=sealRecoveryImage(17,8,5,{values,properties,scalars:[NaN,Infinity,-0,null,true,false],sparse:Array(3)}),scheduling=options(),blob=await serializeRecoveryImage(image,scheduling.value);assert.equal(await blob.text(),JSON.stringify(image));assert.equal(blob.size,new TextEncoder().encode(JSON.stringify(image)).byteLength);assert.equal(blob.size,sealedRecoveryByteLength(image));assert.ok(scheduling.yields>0);assert.deepEqual(JSON.parse(await blob.text()),JSON.parse(JSON.stringify(image))); +}); +test('small owned subtree reuse does not reread caller getters or infer ownership from external freezing',async()=>{ + let reads=0;const caller=Object.freeze({get text(){reads++;return'captured';}}),image=sealRecoveryImage(1,0,0,{rows:Array.from({length:1000},(_,i)=>({id:i,value:caller}))}),before=reads,a=await serializeRecoveryImage(image,options().value),b=await serializeRecoveryImage(image,options().value);assert.equal(reads,before);assert.equal(await a.text(),JSON.stringify(image));assert.equal(await b.text(),await a.text()); + const external={formatVersion:1,rootRevision:0,localRevision:0,payload:{nested:'external'}};const pending=serializeRecoveryImage(external);external.payload.nested='late';assert.equal(JSON.parse(await(await pending).text()).payload.nested,'external'); +}); +test('coarse clocks yield for large images and a lost owner or host failure cannot return partial bytes',async()=>{ + const image=sealRecoveryImage(1,1,1,{rows:Array.from({length:14000},(_,i)=>({id:i,text:'x'.repeat(100)}))});let yields=0;const blob=await serializeRecoveryImage(image,{now:()=>0,yieldTask:async()=>{yields++;}});assert.ok(yields>0);assert.equal(await blob.text(),JSON.stringify(image)); + const failure=Error('owner lost');let gone=false,clock=0;await assert.rejects(serializeRecoveryImage(image,{now:()=>clock+=10,yieldTask:async()=>{gone=true;},check(){if(gone)throw failure;}}),error=>error===failure);clock=0;await assert.rejects(serializeRecoveryImage(image,{now:()=>clock+=10,yieldTask:async()=>{throw failure;}}),error=>error===failure); +});