Zayden-Jung/Endoleap
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Negotiate and decode snapshot-bound canonical model streams

Browse Source
This commit is contained in:
Codex Bot
2026-11-09 02:06:07 +08:00
parent 0205f24904
commit a92ae72c8b
9 changed files with 172 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app/api/foundry/[...path]/route.ts
+3 -1
View File
@@ -72,6 +72,8 @@ async function handler(
const project = process.env.NEXT_PUBLIC_FOUNDRY_PROJECT || "endoleap";
const headers: Record<string, string> = { "Content-Type": asset ? (req.headers.get("content-type") || "application/octet-stream") : "application/json" };
if (canvas) headers.Accept = "application/vnd.foundry.canvas+json;version=2";
const modelStream = canonical && path.endsWith('/canvas/model') && req.method==='GET' && req.headers.get('accept')==='application/vnd.foundry.canvas-model+ndjson;version=1';
if(modelStream)headers.Accept='application/vnd.foundry.canvas-model+ndjson;version=1';
const session = req.cookies.get("foundry_session")?.value;
if (session) headers.Authorization = "Bearer " + session;
for (const h of ["x-foundry-code", "x-foundry-verifier", "if-match", "idempotency-key", "x-expected-account", "x-resource-id"]) {
@@ -133,7 +135,7 @@ async function handler(
// Snapshots contain no login exchange payload. Forward them without buffering
// and parsing/re-encoding multi-megabyte inline image JSON in the proxy.
if (canonical || canvas && path.endsWith("/canvas") && req.method === "GET" && upstream.ok) {
responseHeaders.set("Content-Type", "application/json; charset=utf-8");
responseHeaders.set("Content-Type", modelStream && upstream.ok && upstream.headers.get('content-type')?.startsWith('application/vnd.foundry.canvas-model+ndjson;') ? upstream.headers.get('content-type')! : "application/json; charset=utf-8");
responseHeaders.set("Server-Timing", `upstream;dur=${(headersReceived - started).toFixed(1)}, proxy_headers;dur=${(performance.now() - headersReceived).toFixed(1)}`);
return new NextResponse(upstream.body, { status: upstream.status, headers: responseHeaders });
}
src/lib/canvas-runtime/contracts.ts
+3 -1
View File
@@ -1,6 +1,6 @@
import type {OrphanFile} from './orphan-files.ts';
import type { ImportJob } from './imports.ts';
import type { Document, Id, Json, OperationIntent, Snapshot } from '../canvas-core/types.ts';
import type { CanvasRecord, ElementRecord, Document, Id, Json, OperationIntent, Snapshot } from '../canvas-core/types.ts';
import type { LedgerSnapshot, FrozenRequest } from './ledger.ts';
import type { CompiledRequest, OperationStatus } from './scheduler.ts';
import type { RunRecord } from './runs.ts';
@@ -22,6 +22,8 @@ export type RuntimeState = {
};
export type EngineSnapshot = Snapshot<RuntimeState>;
export type CanonicalProposal = { readonly document: Document; readonly versions: ReadonlyMap<Id, number> };
/** Receiving records are read deltas, not a validated Document or an ACK base. */
export type CanonicalReadUpdate = CanonicalReadResult | { readonly status:'receiving'; readonly projectId:Id; readonly canvases:readonly CanvasRecord[]; readonly elements:readonly ElementRecord[]; readonly tombstones:ReadonlyMap<Id,number>; readonly versions:ReadonlyMap<Id,number>; readonly receivedElements:number; readonly totalElements:number };
export type CanonicalReadResult = ({ readonly status: 'ready' } & CanonicalProposal) | { readonly status: 'migration-required'; readonly rootId: Id; readonly sourceVersions: ReadonlyMap<Id, number> };
/** A validated historical execution receipt; its old model is never published over a fresh read. */
export type AppliedModelProof = { readonly versions:ReadonlyMap<Id,number>; assertCanonical(proposal:CanonicalProposal):void };
src/lib/canvas-runtime/host.ts
+4 -4
View File
@@ -2,13 +2,13 @@ import type { Id, Registry } from '../canvas-core/types.ts';
import { fault, readMap } from '../canvas-core/values.ts';
import type { WorkspaceRuntime } from './workspace.ts';
import { createScopedLoader, type LoadPort, type ScopedLoader } from './loader.ts';
import type { CanonicalReadResult } from './contracts.ts';
import type { CanonicalReadUpdate } from './contracts.ts';
/** Host lifetime matches the signed-in application, independently of navigation. */
export function createRuntimeHost(options: { registry: Registry; workspace(accountId: Id, workspaceId: Id, authEpoch: number, current: () => boolean): WorkspaceRuntime; modelLoader?(accountId:Id,workspaceId:Id,authEpoch:number,current:()=>boolean):LoadPort<CanonicalReadResult>; onError?: (error: unknown) => void }) {
export function createRuntimeHost(options: { registry: Registry; workspace(accountId: Id, workspaceId: Id, authEpoch: number, current: () => boolean): WorkspaceRuntime; modelLoader?(accountId:Id,workspaceId:Id,authEpoch:number,current:()=>boolean):LoadPort<CanonicalReadUpdate>; onError?: (error: unknown) => void }) {
let accountId: Id | null = null, authEpoch = 0, disposed = false;
const workspaces = new Map<Id, WorkspaceRuntime>(), retired = new Set<WorkspaceRuntime>();
const readers = new Map<Id, ScopedLoader<CanonicalReadResult>>();
const readers = new Map<Id, ScopedLoader<CanonicalReadUpdate>>();
const closeRetired = async () => {
const results = await Promise.allSettled([...retired].map(async runtime => { await runtime.close(); retired.delete(runtime); }));
const errors = results.flatMap(result => result.status === 'rejected' ? [result.reason] : []);
@@ -34,7 +34,7 @@ export function createRuntimeHost(options: { registry: Registry; workspace(accou
let reader=readers.get(workspace);
if(!reader){const epoch=authEpoch,current=()=>!disposed&&accountId===account&&authEpoch===epoch;reader=createScopedLoader({load:options.modelLoader(account,workspace,epoch,current),current:()=>current(),maxCacheEntries:0});readers.set(workspace,reader);}
const handle=reader.acquire({accountId:account,workspaceId:workspace,projectId:null,canvasId:canvas,viewKey:'canonical-project',format:'canonical-model-v1',queryFingerprint:'full-project',source:'foundry',authEpoch,navigationGeneration:settings.navigationGeneration??0},{signal:settings.signal});
try{const snapshot=await handle.ready;if(disposed||accountId!==account||authEpoch!==readEpoch)throw fault('identity_changed');const last=snapshot.chunks.at(-1);if(!last||!last.complete)throw fault('incomplete_load');return last.value;}finally{handle.release();}
try{const snapshot=await handle.ready;if(disposed||accountId!==account||authEpoch!==readEpoch)throw fault('identity_changed');const last=snapshot.chunks.at(-1);if(!last||!last.complete||last.value.status==='receiving')throw fault('incomplete_load');return last.value;}finally{handle.release();}
},
workspace(account: Id, workspace: Id) {
activate(account);
src/lib/canvas-service/foundry.ts
+15 -3
View File
@@ -1,3 +1,4 @@
import { modelStreamType } from './model-stream.ts';
import type { CanvasRecord, Document, ElementRecord, Id, Json, OperationIntent, Registry } from '../canvas-core/types.ts';
import { createDocument, validateDocument } from '../canvas-core/document.ts';
import { equal, fault, freeze, readMap, validJson } from '../canvas-core/values.ts';
@@ -6,8 +7,8 @@ import type { FrozenRequest } from '../canvas-runtime/ledger.ts';
import { projectBatch } from '../canvas-runtime/batch-projection.ts';
import type { OperationStatus } from '../canvas-runtime/scheduler.ts';
export type ServiceRequest = { readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal };
export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly retryAfterMs?: number };
export type ServiceRequest = { readonly responseFormat?:'model-stream-v1'; readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal };
export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly stream?:AsyncIterable<Uint8Array>; readonly retryAfterMs?: number };
export type ServiceTransport = (request: ServiceRequest) => Promise<ServiceResponse>;
const uuid = /^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$/i;
const record = (value: unknown): Record<string, unknown> => { if (!value || Array.isArray(value) || typeof value !== 'object') throw fault('invalid_canonical_response'); return value as Record<string, unknown>; };
@@ -28,9 +29,20 @@ export function createFoundryTransport(options: { accountId: Id; isCurrentIdenti
if (!options.isCurrentIdentity()) throw fault('identity_changed');
if (input.expectedVersion !== undefined && (!Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1)) throw fault('invalid_server_version');
const response = await requestFetch(`${options.basePath ?? '/api/foundry/'}${input.path}`, { method: input.method, body: input.body, signal: input.signal, credentials: 'same-origin', cache: 'no-store', redirect: 'error', headers: {
Accept: 'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}),
Accept: input.responseFormat==='model-stream-v1'?modelStreamType:'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}),
} });
if (!options.isCurrentIdentity()) { await response.body?.cancel(); throw fault('identity_changed'); }
if(input.responseFormat==='model-stream-v1'&&response.status===200&&response.headers.get('content-type')?.split(';')[0].trim()==='application/vnd.foundry.canvas-model+ndjson'){
if(!/;\s*version=1(?:;|$)/.test(response.headers.get('content-type')??'')){await response.body?.cancel();throw fault('unsupported_model_stream');}
if(!response.body)throw fault('incomplete_model_stream');
const body=response.body;
async function* stream(){
const reader=body.getReader(),abort=()=>{void reader.cancel().catch(()=>{});};input.signal?.addEventListener('abort',abort,{once:true});
try{while(true){input.signal?.throwIfAborted();if(!options.isCurrentIdentity())throw fault('identity_changed');const next=await reader.read();input.signal?.throwIfAborted();if(!options.isCurrentIdentity())throw fault('identity_changed');if(next.done)break;yield next.value;}}
finally{input.signal?.removeEventListener('abort',abort);await reader.cancel().catch(()=>{});reader.releaseLock();}
}
return{status:response.status,body:null,stream:stream()};
}
let body: unknown;
try { body = await response.json(); } catch { throw fault('invalid_canonical_response'); }
if (!options.isCurrentIdentity()) throw fault('identity_changed');
src/lib/canvas-service/model-loader.ts
+11 -6
View File
@@ -1,16 +1,21 @@
import { decodeModelStream } from './model-stream.ts';
import type { Id, Registry } from '../canvas-core/types.ts';
import { fault } from '../canvas-core/values.ts';
import type { CanonicalReadResult } from '../canvas-runtime/contracts.ts';
import type { CanonicalReadUpdate } from '../canvas-runtime/contracts.ts';
import { loadQueryKey, type LoadPort } from '../canvas-runtime/loader.ts';
import { loadFoundryModel, type ServiceTransport } from './foundry.ts';
/** The current endpoint is complete-only. One validated chunk is intentional;
* this adapter does not claim to stream an already-buffered response progressively.
*/
export function createFoundryModelLoader(options: { workspaceId: Id; registry: Registry; transport: ServiceTransport }): LoadPort<CanonicalReadResult> {
/** Negotiate real streaming; older servers may explicitly return complete JSON. */
export function createFoundryModelLoader(options: { workspaceId: Id; registry: Registry; transport: ServiceTransport }): LoadPort<CanonicalReadUpdate> {
return async function* (scope, signal) {
if (scope.workspaceId !== options.workspaceId || scope.format !== 'canonical-model-v1' || scope.source !== 'foundry' || scope.viewKey !== 'canonical-project') throw fault('invalid_model_load_scope');
const value = await loadFoundryModel(options.transport, options.workspaceId, scope.canvasId, options.registry, signal);
const response=await options.transport({method:'GET',path:`workspaces/${encodeURIComponent(options.workspaceId)}/materials/${encodeURIComponent(scope.canvasId)}/canvas/model`,signal,responseFormat:'model-stream-v1'});
if(response.stream){
if(response.status!==200)throw fault('invalid_model_stream');
for await(const chunk of decodeModelStream(response.stream,scope.canvasId,options.registry,signal))yield Object.freeze({...chunk,serverVersions:Object.freeze(Object.fromEntries(chunk.versions)),permissionEpoch:scope.authEpoch,queryKey:loadQueryKey(scope)});
return;
}
const value = await loadFoundryModel(async()=>response, options.workspaceId, scope.canvasId, options.registry, signal);
signal.throwIfAborted();
const projectId = value.status === 'ready' ? value.document.projectId : value.rootId;
const versions = Object.fromEntries([...(value.status === 'ready' ? value.versions : value.sourceVersions)].sort(([a], [b]) => a.localeCompare(b)));
src/lib/canvas-service/model-stream.ts
+62
View File
@@ -0,0 +1,62 @@
import type { CanvasRecord, ElementRecord, Id, Registry } from '../canvas-core/types.ts';
import { createDocument, validateDocument } from '../canvas-core/document.ts';
import { fault, freeze, readMap, validJson } from '../canvas-core/values.ts';
import type { CanonicalReadUpdate } from '../canvas-runtime/contracts.ts';
export const modelStreamType = 'application/vnd.foundry.canvas-model+ndjson;version=1';
const maxBytes=64*1024*1024,maxLine=8*1024*1024,maxLines=4096;
const uuid=/^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$/i;
const object=(value:unknown):Record<string,unknown>=>{if(!value||typeof value!=='object'||Array.isArray(value)||!validJson(value))throw fault('invalid_model_stream');return value as Record<string,unknown>;};
/** Bounded NDJSON framing. EOF is required even after a terminal record. */
async function* lines(source:AsyncIterable<Uint8Array>,signal:AbortSignal){
const decoder=new TextDecoder('utf-8',{fatal:true});let pending='',total=0,lineBytes=0,count=0;
for await(const bytes of source){
signal.throwIfAborted();if(!(bytes instanceof Uint8Array))throw fault('invalid_model_stream');total+=bytes.byteLength;if(total>maxBytes)throw fault('model_stream_budget');
let start=0;
for(let index=0;index<bytes.length;index++)if(bytes[index]===10){
lineBytes+=index-start;if(lineBytes>maxLine)throw fault('model_stream_budget');
pending+=decoder.decode(bytes.subarray(start,index),{stream:true});pending+=decoder.decode();if(++count>maxLines||!pending)throw fault('invalid_model_stream');
yield{raw:JSON.parse(pending),byteLength:lineBytes+1};pending='';lineBytes=0;start=index+1;
}
lineBytes+=bytes.length-start;if(lineBytes>maxLine)throw fault('model_stream_budget');pending+=decoder.decode(bytes.subarray(start),{stream:true});
}
signal.throwIfAborted();pending+=decoder.decode();if(pending||lineBytes)throw fault('incomplete_model_stream');
}
export async function* decodeModelStream(source:AsyncIterable<Uint8Array>,canvasId:Id,registry:Registry,signal:AbortSignal){
let sequence=0,token='',projectId='',expected=0,ended=false;
let terminal: {sequence:number;snapshotToken:string;projectId:string;versions:ReadonlyMap<Id,number>;complete:boolean;byteLength:number;coverage:string[];value:CanonicalReadUpdate}|undefined;
let canvases:CanvasRecord[]=[],versions:ReadonlyMap<Id,number>=new Map(),tombstones:ReadonlyMap<Id,number>=new Map();
const elements:ElementRecord[]=[],ids=new Set<Id>(),canvasIds=new Set<Id>();
for await(const line of lines(source,signal)){
const raw=object(line.raw);
if(ended||raw.streamVersion!==1||raw.sequence!==sequence++||typeof raw.snapshotToken!=='string'||!raw.snapshotToken||raw.snapshotToken.length>256)throw fault('invalid_model_stream');
if(sequence===1){
if(raw.kind!=='header'||raw.documentFormatVersion!==1||typeof raw.projectId!=='string'||!uuid.test(raw.projectId)||!Number.isSafeInteger(raw.elementCount)||(raw.elementCount as number)<0||!Array.isArray(raw.canvases)||raw.canvases.length>256)throw fault('invalid_model_stream');
token=raw.snapshotToken;projectId=raw.projectId;expected=raw.elementCount as number;canvases=raw.canvases as CanvasRecord[];
for(const canvas of canvases){object(canvas);if(typeof canvas.id!=='string'||!uuid.test(canvas.id)||canvasIds.has(canvas.id))throw fault('invalid_model_stream');canvasIds.add(canvas.id);}
if(!canvasIds.has(canvasId)||!canvasIds.has(projectId))throw fault('project_mismatch');
const versionMap=new Map<Id,number>(),dead=new Map<Id,number>();
for(const [id,value]of Object.entries(object(raw.versions))){if(!uuid.test(id)||!Number.isSafeInteger(value)||(value as number)<1)throw fault('invalid_canonical_version');versionMap.set(id,value as number);}
if([...canvasIds].some(id=>!versionMap.has(id)))throw fault('invalid_model_stream');
if(!Array.isArray(raw.removedCanvasIds)||new Set(raw.removedCanvasIds).size!==raw.removedCanvasIds.length||raw.removedCanvasIds.some(id=>typeof id!=='string'||canvasIds.has(id)||!versionMap.has(id))||versionMap.size!==canvasIds.size+raw.removedCanvasIds.length)throw fault('invalid_model_stream');
for(const [id,value]of Object.entries(object(raw.tombstones))){if(!id||!Number.isSafeInteger(value)||(value as number)<1)throw fault('invalid_model_stream');dead.set(id,value as number);}
versions=readMap(versionMap);tombstones=readMap(dead);
const value:CanonicalReadUpdate=freeze({status:'receiving',projectId,canvases,elements:[],tombstones,versions,receivedElements:0,totalElements:expected});
yield{sequence:0,snapshotToken:token,projectId,versions,complete:false,byteLength:line.byteLength,coverage:['model-header'],value};continue;
}
if(raw.snapshotToken!==token)throw fault('load_snapshot_changed');
if(raw.kind==='elements'){
if(!Array.isArray(raw.elements)||!raw.elements.length||raw.elements.length>64||elements.length+raw.elements.length>expected)throw fault('invalid_model_stream');
for(const item of raw.elements){const element=object(item) as unknown as ElementRecord;if(typeof element.id!=='string'||!element.id||ids.has(element.id)||!canvasIds.has(element.ownerCanvasId))throw fault('invalid_model_stream');ids.add(element.id);elements.push(element);}
const value:CanonicalReadUpdate=freeze({status:'receiving',projectId,canvases:[],elements:raw.elements as ElementRecord[],tombstones:readMap(new Map()),versions,receivedElements:elements.length,totalElements:expected});
yield{sequence:sequence-1,snapshotToken:token,projectId,versions,complete:false,byteLength:line.byteLength,coverage:raw.elements.map(item=>`element:${(item as ElementRecord).id}`),value};
}else if(raw.kind==='complete'){
if(raw.elementCount!==expected||elements.length!==expected)throw fault('incomplete_model_stream');
const document=createDocument(projectId,canvases,elements,tombstones);validateDocument(document,registry);ended=true;
const value:CanonicalReadUpdate=Object.freeze({status:'ready',document,versions});
terminal={sequence:sequence-1,snapshotToken:token,projectId,versions,complete:true,byteLength:line.byteLength,coverage:[...canvasIds].map(id=>`canvas:${id}`),value};
}else throw fault('invalid_model_stream');
}
if(!ended||!terminal)throw fault('incomplete_model_stream');
yield terminal;
}