Negotiate and decode snapshot-bound canonical model streams

src/lib/canvas-service/foundry.ts

@@ -1,3 +1,4 @@
+import { modelStreamType } from './model-stream.ts';
 import type { CanvasRecord, Document, ElementRecord, Id, Json, OperationIntent, Registry } from '../canvas-core/types.ts';
 import { createDocument, validateDocument } from '../canvas-core/document.ts';
 import { equal, fault, freeze, readMap, validJson } from '../canvas-core/values.ts';
@@ -6,8 +7,8 @@ import type { FrozenRequest } from '../canvas-runtime/ledger.ts';
 import { projectBatch } from '../canvas-runtime/batch-projection.ts';
 import type { OperationStatus } from '../canvas-runtime/scheduler.ts';
 
-export type ServiceRequest = { readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal };
-export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly retryAfterMs?: number };
+export type ServiceRequest = { readonly responseFormat?:'model-stream-v1'; readonly path: string; readonly method: 'GET' | 'POST'; readonly requestId?: Id; readonly expectedVersion?: number; readonly body?: string | FormData; readonly resourceId?: Id; readonly signal?: AbortSignal };
+export type ServiceResponse = { readonly status: number; readonly body: unknown; readonly stream?:AsyncIterable<Uint8Array>; readonly retryAfterMs?: number };
 export type ServiceTransport = (request: ServiceRequest) => Promise<ServiceResponse>;
 const uuid = /^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$/i;
 const record = (value: unknown): Record<string, unknown> => { if (!value || Array.isArray(value) || typeof value !== 'object') throw fault('invalid_canonical_response'); return value as Record<string, unknown>; };
@@ -28,9 +29,20 @@ export function createFoundryTransport(options: { accountId: Id; isCurrentIdenti
     if (!options.isCurrentIdentity()) throw fault('identity_changed');
     if (input.expectedVersion !== undefined && (!Number.isSafeInteger(input.expectedVersion) || input.expectedVersion < 1)) throw fault('invalid_server_version');
     const response = await requestFetch(`${options.basePath ?? '/api/foundry/'}${input.path}`, { method: input.method, body: input.body, signal: input.signal, credentials: 'same-origin', cache: 'no-store', redirect: 'error', headers: {
-      Accept: 'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}),
+      Accept: input.responseFormat==='model-stream-v1'?modelStreamType:'application/json', ...(input.body instanceof FormData ? {} : { 'Content-Type': 'application/json' }), ...(input.resourceId ? { 'X-Resource-Id': input.resourceId } : {}), 'X-Expected-Account': options.accountId, ...(input.requestId ? { 'Idempotency-Key': input.requestId } : {}), ...(input.expectedVersion !== undefined ? { 'If-Match': `"${input.expectedVersion}"` } : {}),
     } });
     if (!options.isCurrentIdentity()) { await response.body?.cancel(); throw fault('identity_changed'); }
+    if(input.responseFormat==='model-stream-v1'&&response.status===200&&response.headers.get('content-type')?.split(';')[0].trim()==='application/vnd.foundry.canvas-model+ndjson'){
+      if(!/;\s*version=1(?:;|$)/.test(response.headers.get('content-type')??'')){await response.body?.cancel();throw fault('unsupported_model_stream');}
+      if(!response.body)throw fault('incomplete_model_stream');
+      const body=response.body;
+      async function* stream(){
+        const reader=body.getReader(),abort=()=>{void reader.cancel().catch(()=>{});};input.signal?.addEventListener('abort',abort,{once:true});
+        try{while(true){input.signal?.throwIfAborted();if(!options.isCurrentIdentity())throw fault('identity_changed');const next=await reader.read();input.signal?.throwIfAborted();if(!options.isCurrentIdentity())throw fault('identity_changed');if(next.done)break;yield next.value;}}
+        finally{input.signal?.removeEventListener('abort',abort);await reader.cancel().catch(()=>{});reader.releaseLock();}
+      }
+      return{status:response.status,body:null,stream:stream()};
+    }
     let body: unknown;
     try { body = await response.json(); } catch { throw fault('invalid_canonical_response'); }
     if (!options.isCurrentIdentity()) throw fault('identity_changed');

src/lib/canvas-service/model-loader.ts

@@ -1,16 +1,21 @@
+import { decodeModelStream } from './model-stream.ts';
 import type { Id, Registry } from '../canvas-core/types.ts';
 import { fault } from '../canvas-core/values.ts';
-import type { CanonicalReadResult } from '../canvas-runtime/contracts.ts';
+import type { CanonicalReadUpdate } from '../canvas-runtime/contracts.ts';
 import { loadQueryKey, type LoadPort } from '../canvas-runtime/loader.ts';
 import { loadFoundryModel, type ServiceTransport } from './foundry.ts';
 
-/** The current endpoint is complete-only. One validated chunk is intentional;
- * this adapter does not claim to stream an already-buffered response progressively.
- */
-export function createFoundryModelLoader(options: { workspaceId: Id; registry: Registry; transport: ServiceTransport }): LoadPort<CanonicalReadResult> {
+/** Negotiate real streaming; older servers may explicitly return complete JSON. */
+export function createFoundryModelLoader(options: { workspaceId: Id; registry: Registry; transport: ServiceTransport }): LoadPort<CanonicalReadUpdate> {
   return async function* (scope, signal) {
     if (scope.workspaceId !== options.workspaceId || scope.format !== 'canonical-model-v1' || scope.source !== 'foundry' || scope.viewKey !== 'canonical-project') throw fault('invalid_model_load_scope');
-    const value = await loadFoundryModel(options.transport, options.workspaceId, scope.canvasId, options.registry, signal);
+    const response=await options.transport({method:'GET',path:`workspaces/${encodeURIComponent(options.workspaceId)}/materials/${encodeURIComponent(scope.canvasId)}/canvas/model`,signal,responseFormat:'model-stream-v1'});
+    if(response.stream){
+      if(response.status!==200)throw fault('invalid_model_stream');
+      for await(const chunk of decodeModelStream(response.stream,scope.canvasId,options.registry,signal))yield Object.freeze({...chunk,serverVersions:Object.freeze(Object.fromEntries(chunk.versions)),permissionEpoch:scope.authEpoch,queryKey:loadQueryKey(scope)});
+      return;
+    }
+    const value = await loadFoundryModel(async()=>response, options.workspaceId, scope.canvasId, options.registry, signal);
     signal.throwIfAborted();
     const projectId = value.status === 'ready' ? value.document.projectId : value.rootId;
     const versions = Object.fromEntries([...(value.status === 'ready' ? value.versions : value.sourceVersions)].sort(([a], [b]) => a.localeCompare(b)));

src/lib/canvas-service/model-stream.ts

@@ -0,0 +1,62 @@
+import type { CanvasRecord, ElementRecord, Id, Registry } from '../canvas-core/types.ts';
+import { createDocument, validateDocument } from '../canvas-core/document.ts';
+import { fault, freeze, readMap, validJson } from '../canvas-core/values.ts';
+import type { CanonicalReadUpdate } from '../canvas-runtime/contracts.ts';
+
+export const modelStreamType = 'application/vnd.foundry.canvas-model+ndjson;version=1';
+const maxBytes=64*1024*1024,maxLine=8*1024*1024,maxLines=4096;
+const uuid=/^[a-f\d]{8}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{4}-[a-f\d]{12}$/i;
+const object=(value:unknown):Record<string,unknown>=>{if(!value||typeof value!=='object'||Array.isArray(value)||!validJson(value))throw fault('invalid_model_stream');return value as Record<string,unknown>;};
+/** Bounded NDJSON framing. EOF is required even after a terminal record. */
+async function* lines(source:AsyncIterable<Uint8Array>,signal:AbortSignal){
+ const decoder=new TextDecoder('utf-8',{fatal:true});let pending='',total=0,lineBytes=0,count=0;
+ for await(const bytes of source){
+  signal.throwIfAborted();if(!(bytes instanceof Uint8Array))throw fault('invalid_model_stream');total+=bytes.byteLength;if(total>maxBytes)throw fault('model_stream_budget');
+  let start=0;
+  for(let index=0;index<bytes.length;index++)if(bytes[index]===10){
+   lineBytes+=index-start;if(lineBytes>maxLine)throw fault('model_stream_budget');
+   pending+=decoder.decode(bytes.subarray(start,index),{stream:true});pending+=decoder.decode();if(++count>maxLines||!pending)throw fault('invalid_model_stream');
+   yield{raw:JSON.parse(pending),byteLength:lineBytes+1};pending='';lineBytes=0;start=index+1;
+  }
+  lineBytes+=bytes.length-start;if(lineBytes>maxLine)throw fault('model_stream_budget');pending+=decoder.decode(bytes.subarray(start),{stream:true});
+ }
+ signal.throwIfAborted();pending+=decoder.decode();if(pending||lineBytes)throw fault('incomplete_model_stream');
+}
+export async function* decodeModelStream(source:AsyncIterable<Uint8Array>,canvasId:Id,registry:Registry,signal:AbortSignal){
+ let sequence=0,token='',projectId='',expected=0,ended=false;
+ let terminal: {sequence:number;snapshotToken:string;projectId:string;versions:ReadonlyMap<Id,number>;complete:boolean;byteLength:number;coverage:string[];value:CanonicalReadUpdate}|undefined;
+ let canvases:CanvasRecord[]=[],versions:ReadonlyMap<Id,number>=new Map(),tombstones:ReadonlyMap<Id,number>=new Map();
+ const elements:ElementRecord[]=[],ids=new Set<Id>(),canvasIds=new Set<Id>();
+ for await(const line of lines(source,signal)){
+  const raw=object(line.raw);
+  if(ended||raw.streamVersion!==1||raw.sequence!==sequence++||typeof raw.snapshotToken!=='string'||!raw.snapshotToken||raw.snapshotToken.length>256)throw fault('invalid_model_stream');
+  if(sequence===1){
+   if(raw.kind!=='header'||raw.documentFormatVersion!==1||typeof raw.projectId!=='string'||!uuid.test(raw.projectId)||!Number.isSafeInteger(raw.elementCount)||(raw.elementCount as number)<0||!Array.isArray(raw.canvases)||raw.canvases.length>256)throw fault('invalid_model_stream');
+   token=raw.snapshotToken;projectId=raw.projectId;expected=raw.elementCount as number;canvases=raw.canvases as CanvasRecord[];
+   for(const canvas of canvases){object(canvas);if(typeof canvas.id!=='string'||!uuid.test(canvas.id)||canvasIds.has(canvas.id))throw fault('invalid_model_stream');canvasIds.add(canvas.id);}
+   if(!canvasIds.has(canvasId)||!canvasIds.has(projectId))throw fault('project_mismatch');
+   const versionMap=new Map<Id,number>(),dead=new Map<Id,number>();
+   for(const [id,value]of Object.entries(object(raw.versions))){if(!uuid.test(id)||!Number.isSafeInteger(value)||(value as number)<1)throw fault('invalid_canonical_version');versionMap.set(id,value as number);}
+   if([...canvasIds].some(id=>!versionMap.has(id)))throw fault('invalid_model_stream');
+   if(!Array.isArray(raw.removedCanvasIds)||new Set(raw.removedCanvasIds).size!==raw.removedCanvasIds.length||raw.removedCanvasIds.some(id=>typeof id!=='string'||canvasIds.has(id)||!versionMap.has(id))||versionMap.size!==canvasIds.size+raw.removedCanvasIds.length)throw fault('invalid_model_stream');
+   for(const [id,value]of Object.entries(object(raw.tombstones))){if(!id||!Number.isSafeInteger(value)||(value as number)<1)throw fault('invalid_model_stream');dead.set(id,value as number);}
+   versions=readMap(versionMap);tombstones=readMap(dead);
+   const value:CanonicalReadUpdate=freeze({status:'receiving',projectId,canvases,elements:[],tombstones,versions,receivedElements:0,totalElements:expected});
+   yield{sequence:0,snapshotToken:token,projectId,versions,complete:false,byteLength:line.byteLength,coverage:['model-header'],value};continue;
+  }
+  if(raw.snapshotToken!==token)throw fault('load_snapshot_changed');
+  if(raw.kind==='elements'){
+   if(!Array.isArray(raw.elements)||!raw.elements.length||raw.elements.length>64||elements.length+raw.elements.length>expected)throw fault('invalid_model_stream');
+   for(const item of raw.elements){const element=object(item) as unknown as ElementRecord;if(typeof element.id!=='string'||!element.id||ids.has(element.id)||!canvasIds.has(element.ownerCanvasId))throw fault('invalid_model_stream');ids.add(element.id);elements.push(element);}
+   const value:CanonicalReadUpdate=freeze({status:'receiving',projectId,canvases:[],elements:raw.elements as ElementRecord[],tombstones:readMap(new Map()),versions,receivedElements:elements.length,totalElements:expected});
+   yield{sequence:sequence-1,snapshotToken:token,projectId,versions,complete:false,byteLength:line.byteLength,coverage:raw.elements.map(item=>`element:${(item as ElementRecord).id}`),value};
+  }else if(raw.kind==='complete'){
+   if(raw.elementCount!==expected||elements.length!==expected)throw fault('incomplete_model_stream');
+   const document=createDocument(projectId,canvases,elements,tombstones);validateDocument(document,registry);ended=true;
+   const value:CanonicalReadUpdate=Object.freeze({status:'ready',document,versions});
+   terminal={sequence:sequence-1,snapshotToken:token,projectId,versions,complete:true,byteLength:line.byteLength,coverage:[...canvasIds].map(id=>`canvas:${id}`),value};
+  }else throw fault('invalid_model_stream');
+ }
+ if(!ended||!terminal)throw fault('incomplete_model_stream');
+ yield terminal;
+}