package platform import ( "bytes" "context" "crypto/rand" "crypto/sha256" "database/sql" "encoding/base64" "encoding/hex" "encoding/json" "image" "image/png" "net" "net/http" "net/http/httptest" "os" "path/filepath" "strconv" "sync/atomic" "testing" "gitea.entroia.com/Zayden-Jung/foundry-core/internal/codex" ) func TestCanvasAIJobLifecycle(t *testing.T) { dsn := os.Getenv("PLATFORM_TEST_DATABASE_URL") if dsn == "" { t.Skip("requires an isolated integration database") } db, err := sql.Open("pgx", dsn) if err != nil { t.Fatal(err) } defer db.Close() ctx := context.Background() var random [32]byte _, _ = rand.Read(random[:]) project := "canvas-ai-" + hex.EncodeToString(random[:8]) token := base64.RawURLEncoding.EncodeToString(random[:]) digest := sha256.Sum256([]byte(token)) var account, workspace, board string if err = db.QueryRowContext(ctx, `INSERT INTO project_accounts(project,id) VALUES($1,gen_random_uuid()) RETURNING id::text`, project).Scan(&account); err != nil { t.Fatal(err) } defer func() { _, _ = db.ExecContext(ctx, `DELETE FROM canvas_ai_runs WHERE project=$1`, project) _, _ = db.ExecContext(ctx, `DELETE FROM canvas_asset_bindings WHERE project=$1`, project) _, _ = db.ExecContext(ctx, `DELETE FROM canvas_assets WHERE project=$1`, project) _, _ = db.ExecContext(ctx, `DELETE FROM workspaces WHERE project=$1`, project) _, _ = db.ExecContext(ctx, `DELETE FROM project_accounts WHERE project=$1`, project) }() if _, err = db.ExecContext(ctx, `INSERT INTO project_sessions VALUES($1,$2,$3,now()+interval '1 hour')`, base64.RawURLEncoding.EncodeToString(digest[:]), project, account); err != nil { t.Fatal(err) } if err = db.QueryRowContext(ctx, `INSERT INTO workspaces(project,name,owner_id) VALUES($1,'AI fixture',$2) RETURNING id::text`, project, account).Scan(&workspace); err != nil { t.Fatal(err) } if _, err = db.ExecContext(ctx, `INSERT INTO workspace_members VALUES($1,$2,$3,'admin',now())`, project, workspace, account); err != nil { t.Fatal(err) } if err = db.QueryRowContext(ctx, `INSERT INTO materials(project,workspace_id,kind,title,created_by,canvas_document) VALUES($1,$2,'board','AI fixture',$3,$4) RETURNING id::text`, project, workspace, account, json.RawMessage(canvasFixture)).Scan(&board); err != nil { t.Fatal(err) } // A Unix-socket fixture exercises Core's actual worker client without any // external model calls. Production worker/model verification is separate. temp, err := os.MkdirTemp("", "fc-ai-") if err != nil { t.Fatal(err) } defer os.RemoveAll(temp) socket := filepath.Join(temp, "worker.sock") listener, err := net.Listen("unix", socket) if err != nil { t.Fatal(err) } var pngOutput bytes.Buffer if err := png.Encode(&pngOutput, image.NewRGBA(image.Rect(0, 0, 3, 2))); err != nil { t.Fatal(err) } var imageInputs atomic.Int32 var busyNext atomic.Bool var failNext atomic.Bool hooks := make(chan func(), 1) var calls atomic.Int32 worker := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/canvas/models": reply(w, 200, response{"models": []any{codex.CanvasModel{ID: "fixture-model", Label: "Fixture", Provider: "fixture", Available: true, InputModalities: []string{"text", "image"}, OutputModalities: []string{"text"}, MaxInputBytes: 1 << 20, MaxExecutionMS: 90000}, codex.CanvasModel{ID: "fixture-image", Label: "Image fixture", Provider: "fixture", Available: true, InputModalities: []string{"text", "image"}, OutputModalities: []string{"image"}, MaxInputBytes: 1 << 20, MaxExecutionMS: 180000}}}) case "/canvas/exec": var task codex.CanvasTask if err := json.NewDecoder(r.Body).Decode(&task); err != nil { reject(w, 400, "invalid_request") return } if busyNext.Swap(false) { reject(w, 429, "codex_busy") return } if failNext.Swap(false) { reject(w, 502, "generation_failed") return } calls.Add(1) select { case hook := <-hooks: hook() default: } if task.OutputModality == "image" { if len(task.Images) == 1 && task.Images[0].MIME == "image/png" && bytes.Equal(task.Images[0].Data, pngOutput.Bytes()) { imageInputs.Add(1) } reply(w, 200, response{"output": codex.CanvasOutput{Kind: "image", Image: &codex.CanvasImage{MIME: "image/png", Data: pngOutput.Bytes()}}}) } else { reply(w, 200, response{"result": "generated result"}) } default: reject(w, 404, "not_found") } })} go worker.Serve(listener) defer worker.Close() s := &service{db: db, origin: "https://api.example.com", canvasAI: codex.NewCanvasClient(socket)} callbacks := map[string]string{project: "https://app.example.com/auth/callback"} mux := http.NewServeMux() base := "/projects/" + project + "/workspaces/" + workspace + "/materials/" + board + "/canvas" prefix := "/projects/{project}/workspaces/{workspace}/materials/{material}/canvas" mux.HandleFunc(prefix, s.protected(callbacks, s.canvas)) mux.HandleFunc("/projects/{project}/workspaces/{workspace}/assets/{asset}", s.protected(callbacks, s.getAsset)) mux.HandleFunc("POST "+prefix+"/ai/runs", s.protected(callbacks, s.createCanvasAIRun)) mux.HandleFunc("GET "+prefix+"/ai/runs", s.protected(callbacks, s.listCanvasAIRuns)) mux.HandleFunc(prefix+"/ai/runs/{run}", s.protected(callbacks, s.getCanvasAIRun)) mux.HandleFunc(prefix+"/ai/runs/{run}/cancel", s.protected(callbacks, s.cancelCanvasAIRun)) mux.HandleFunc(prefix+"/ai/runs/{run}/apply", s.protected(callbacks, s.applyCanvasAIOutput)) call := func(method, path string, body any, status int, version int64, key string) response { t.Helper() raw, _ := json.Marshal(body) r := httptest.NewRequest(method, path, bytes.NewReader(raw)) r.Header.Set("Accept", "application/vnd.foundry.canvas+json;version=2") r.Header.Set("Authorization", "Bearer "+token) if version > 0 { r.Header.Set("If-Match", `"`+strconv.FormatInt(version, 10)+`"`) } if key != "" { r.Header.Set("Idempotency-Key", key) } w := httptest.NewRecorder() mux.ServeHTTP(w, r) if w.Code != status { t.Fatalf("%s %s: got %d %s want %d", method, path, w.Code, w.Body.String(), status) } out := response{} if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil { t.Fatal(err) } return out } command := response{"node_id": "a", "model_id": "fixture-model", "prompt": "Generate", "parameters": response{}, "propagate": false} key := func(n int) string { return "00000000-0000-4000-8000-" + strconv.FormatInt(int64(100000000000+n), 10) } start := func(version int64, n int) response { return call("POST", base+"/ai/runs", command, 202, version, key(n)) } execute := func(id string) { t.Helper() run, err := s.claimCanvasRun(ctx) if err != nil || run == nil || run.ID != id { t.Fatalf("claim: %v %+v", err, run) } s.executeCanvasRun(ctx, *run) } mutate := func(path, text string) { _, err := db.ExecContext(ctx, `UPDATE materials SET canvas_document=jsonb_set(canvas_document,$4::text[],$5::jsonb),canvas_version=canvas_version+1 WHERE project=$1 AND workspace_id=$2 AND id=$3`, project, workspace, board, path, text) if err != nil { t.Errorf("concurrent fixture edit: %v", err) } } first := start(1, 1) retry := start(1, 1) if first["run_id"] != retry["run_id"] || first["canvas_version"] != float64(2) { t.Fatal("creation retry duplicated configuration/job") } call("POST", base+"/ai/runs", command, 409, 2, key(2)) hooks <- func() { mutate("{nodes,0,position,x}", "777") } execute(first["run_id"].(string)) saved := call("GET", base, nil, 200, 0, "") if saved["version"] != float64(4) { t.Fatal("result did not increment concurrent version") } nodes := saved["document"].(map[string]any)["nodes"].([]any) a := nodes[0].(map[string]any) if a["position"].(map[string]any)["x"] != float64(777) || a["data"].(map[string]any)["text"] != "generated result" || a["width"] != float64(320) { t.Fatal("result overwrote geometry/style or failed to update text") } firstPath := base + "/ai/runs/" + first["run_id"].(string) after := int64(0) completed := false for page := 0; page < 30; page++ { result := call("GET", firstPath+"?limit=2&after_seq="+strconv.FormatInt(after, 10), nil, 200, 0, "") if result["status"] != "succeeded" { t.Fatal("run not succeeded") } for _, v := range result["events"].([]any) { event := v.(map[string]any) seq := int64(event["seq"].(float64)) if seq <= after { t.Fatal("event sequence not monotonic") } after = seq if event["type"] == "run.completed" { completed = true } } if result["has_more"] == false { break } } if !completed { t.Fatal("terminal event not replayable") } second := start(4, 3) hooks <- func() { mutate("{nodes,0,data,text}", `"user edit"`) } execute(second["run_id"].(string)) secondPath := base + "/ai/runs/" + second["run_id"].(string) conflict := call("GET", secondPath, nil, 200, 0, "") if conflict["status"] != "conflict" { t.Fatalf("content edit not a conflict: %v", conflict["status"]) } outputID := conflict["outputs"].([]any)[0].(map[string]any)["output"].(map[string]any)["output_id"].(string) apply := response{"node_id": "a", "output_id": outputID} call("POST", secondPath+"/apply", apply, 412, 5, key(4)) applied := call("POST", secondPath+"/apply", apply, 200, 6, key(4)) appliedRetry := call("POST", secondPath+"/apply", apply, 200, 6, key(4)) if applied["canvas_version"] != float64(7) || appliedRetry["canvas_version"] != applied["canvas_version"] { t.Fatal("explicit application not idempotent") } third := start(7, 5) thirdPath := base + "/ai/runs/" + third["run_id"].(string) cancelled := call("POST", thirdPath+"/cancel", response{}, 200, 0, key(6)) call("POST", thirdPath+"/cancel", response{}, 200, 0, key(6)) if cancelled["status"] != "cancelled" { t.Fatal("queued cancel") } if calls.Load() != 2 { t.Fatal("cancelled queued run invoked worker") } fourth := start(8, 7) run, err := s.claimCanvasRun(ctx) if err != nil || run == nil || run.ID != fourth["run_id"] { t.Fatalf("recovery claim %v", err) } if started, err := s.startCanvasStep(ctx, *run, run.Plan.Steps[0]); err != nil || !started { t.Fatalf("recovery start %v", err) } if _, err = db.ExecContext(ctx, `UPDATE canvas_ai_runs SET lease_until=now()-interval '1 second' WHERE project=$1 AND id=$2`, project, run.ID); err != nil { t.Fatal(err) } if err = s.recoverCanvasRuns(ctx); err != nil { t.Fatal(err) } recovered := call("GET", base+"/ai/runs/"+run.ID, nil, 200, 0, "") if recovered["status"] != "failed" || recovered["error"] != "execution_interrupted" || calls.Load() != 2 { t.Fatal("uncertain execution was retried or misreported") } fifth := start(9, 8) removedRun, err := s.claimCanvasRun(ctx) if err != nil || removedRun == nil || removedRun.ID != fifth["run_id"] { t.Fatalf("deletion claim %v", err) } if started, err := s.startCanvasStep(ctx, *removedRun, removedRun.Plan.Steps[0]); err != nil || !started { t.Fatalf("deletion start %v", err) } beforeDelete := call("GET", base, nil, 200, 0, "")["document"].(map[string]any) beforeBytes, _ := json.Marshal(beforeDelete) var without map[string]any _ = json.Unmarshal(beforeBytes, &without) kept := []any{} for _, v := range without["nodes"].([]any) { if v.(map[string]any)["id"] != "a" { kept = append(kept, v) } } without["nodes"] = kept without["edges"] = []any{} call("PATCH", base, response{"document": without}, 200, 10, "") call("PATCH", base, response{"document": beforeDelete}, 200, 11, "") state, err := s.finishCanvasStep(ctx, *removedRun, removedRun.Plan.Steps[0], "late output", "", true) if err != nil || state != "cancelled" { t.Fatalf("deleted target accepted late output: %s %v", state, err) } if err = s.completeCanvasRun(ctx, *removedRun, nil); err != nil { t.Fatal(err) } afterRecreation := call("GET", base, nil, 200, 0, "") if afterRecreation["version"] != float64(12) || afterRecreation["document"].(map[string]any)["nodes"].([]any)[0].(map[string]any)["data"].(map[string]any)["text"] == "late output" { t.Fatal("recreated node overwritten") } active := call("GET", base+"/ai/runs?status=active&include_descendants=true", nil, 200, 0, "") if len(active["runs"].([]any)) != 0 { t.Fatal("terminal jobs returned as active") } imageDoc := call("GET", base, nil, 200, 0, "")["document"].(map[string]any) imageDoc["nodes"] = append(imageDoc["nodes"].([]any), response{"id": "image-node", "type": "plainImage", "position": response{"x": 40, "y": 50}, "width": 240, "height": 160, "data": response{"kind": "plain-image", "title": "Image", "text": "", "src": "data:image/png;base64," + base64.StdEncoding.EncodeToString(pngOutput.Bytes())}}) call("PATCH", base, response{"document": imageDoc}, 200, 12, "") imageCommand := response{"node_id": "image-node", "model_id": "fixture-image", "prompt": "Generate image", "parameters": response{}, "propagate": false} imageRun := call("POST", base+"/ai/runs", imageCommand, 202, 13, key(9)) execute(imageRun["run_id"].(string)) imageSaved := call("GET", base, nil, 200, 0, "") if imageSaved["version"] != float64(15) { t.Fatal("image result version") } imageNode := imageSaved["document"].(map[string]any)["nodes"].([]any)[3].(map[string]any) imageData := imageNode["data"].(map[string]any) assetID, ok := imageData["assetId"].(string) if !ok || assetID == "" || imageNode["width"] != float64(240) { t.Fatal("image result lost identity/geometry or managed asset") } download := httptest.NewRequest("GET", imageData["src"].(string), nil) download.Header.Set("Authorization", "Bearer "+token) downloaded := httptest.NewRecorder() mux.ServeHTTP(downloaded, download) if downloaded.Code != 200 || downloaded.Header().Get("Content-Type") != "image/png" || !bytes.Equal(downloaded.Body.Bytes(), pngOutput.Bytes()) { t.Fatal("generated asset cannot be re-read") } imageConflictRun := call("POST", base+"/ai/runs", imageCommand, 202, 15, key(10)) hooks <- func() { mutate("{nodes,3,data,ai,prompt}", `"user changed image prompt"`) } execute(imageConflictRun["run_id"].(string)) imageConflictPath := base + "/ai/runs/" + imageConflictRun["run_id"].(string) imageConflict := call("GET", imageConflictPath, nil, 200, 0, "") if imageConflict["status"] != "conflict" { t.Fatal("image prompt edit not preserved as conflict") } imageOutputID := imageConflict["outputs"].([]any)[0].(map[string]any)["output"].(map[string]any)["output_id"].(string) call("POST", imageConflictPath+"/apply", response{"node_id": "image-node", "output_id": imageOutputID}, 200, 17, key(11)) if imageInputs.Load() != 2 { t.Fatal("image inputs were not frozen/hydrated through authorized assets") } chainDoc := call("GET", base, nil, 200, 0, "")["document"].(map[string]any) chainDoc["nodes"].([]any)[1].(map[string]any)["data"].(map[string]any)["ai"] = response{"enabled": true, "modelId": "fixture-model", "prompt": "Use upstream result", "parameters": response{}} call("PATCH", base, response{"document": chainDoc}, 200, 18, "") chainCommand := response{"node_id": "a", "model_id": "fixture-model", "prompt": "Chain result", "parameters": response{}, "propagate": true} chainRun := call("POST", base+"/ai/runs", chainCommand, 202, 19, key(12)) callsBeforeBusy := calls.Load() hooks <- func() { busyNext.Store(true) } execute(chainRun["run_id"].(string)) chainPath := base + "/ai/runs/" + chainRun["run_id"].(string) queuedAgain := call("GET", chainPath, nil, 200, 0, "") if queuedAgain["status"] != "queued" || calls.Load() != callsBeforeBusy+1 { t.Fatal("worker admission failure did not preserve successful predecessor") } if pending, err := s.claimCanvasRun(ctx); err != nil || pending != nil { t.Fatal("retry delay not persisted") } if _, err = db.ExecContext(ctx, `UPDATE canvas_ai_runs SET available_at=now() WHERE project=$1 AND id=$2`, project, chainRun["run_id"]); err != nil { t.Fatal(err) } execute(chainRun["run_id"].(string)) chainFinished := call("GET", chainPath, nil, 200, 0, "") if chainFinished["status"] != "succeeded" || calls.Load() != callsBeforeBusy+2 { t.Fatal("resuming queue repeated predecessor or failed downstream") } chainSaved := call("GET", base, nil, 200, 0, "") if chainSaved["version"] != float64(22) { t.Fatal("retry duplicated applied canvas versions") } // Cancellation after worker admission must reject an otherwise successful late output. cancelRunning := start(22, 13) cancelRunningPath := base + "/ai/runs/" + cancelRunning["run_id"].(string) hooks <- func() { call("POST", cancelRunningPath+"/cancel", response{}, 200, 0, key(14)) } execute(cancelRunning["run_id"].(string)) if call("GET", cancelRunningPath, nil, 200, 0, "")["status"] != "cancelled" || call("GET", base, nil, 200, 0, "")["version"] != float64(23) { t.Fatal("running cancellation accepted late output") } // Permission is checked again inside the final write transaction. revoked := start(23, 15) hooks <- func() { if _, err := db.ExecContext(ctx, `DELETE FROM workspace_members WHERE project=$1 AND workspace_id=$2 AND account_id=$3`, project, workspace, account); err != nil { t.Error(err) } } execute(revoked["run_id"].(string)) if _, err := db.ExecContext(ctx, `INSERT INTO workspace_members VALUES($1,$2,$3,'admin',now())`, project, workspace, account); err != nil { t.Fatal(err) } revokedResult := call("GET", base+"/ai/runs/"+revoked["run_id"].(string), nil, 200, 0, "") if revokedResult["status"] != "failed" || call("GET", base, nil, 200, 0, "")["version"] != float64(24) { t.Fatal("permission revocation accepted late output") } // A failed branch stops its dependents while its independent sibling finishes. branchSnapshot := aiPlanFixture([][2]string{{"a", "b"}, {"a", "c"}, {"b", "d"}}, nil, "")["board"] var branchDoc any if err := json.Unmarshal(branchSnapshot.Document, &branchDoc); err != nil { t.Fatal(err) } call("PATCH", base, response{"document": branchDoc}, 200, 24, "") branchRun := call("POST", base+"/ai/runs", chainCommand, 202, 25, key(16)) hooks <- func() { failNext.Store(true) } execute(branchRun["run_id"].(string)) branchResult := call("GET", base+"/ai/runs/"+branchRun["run_id"].(string), nil, 200, 0, "") states := map[string]string{} for _, value := range branchResult["steps"].([]any) { step := value.(map[string]any) states[step["node_id"].(string)] = step["status"].(string) } if branchResult["status"] != "partially_succeeded" || states["a"] != "succeeded" || states["b"] != "failed" || states["c"] != "succeeded" || states["d"] != "skipped" { t.Fatalf("partial failure lost independent branch: %+v %v", states, branchResult["status"]) } cyclic := call("GET", base, nil, 200, 0, "") cyclicVersion := int64(cyclic["version"].(float64)) cyclicDoc := cyclic["document"].(map[string]any) cyclicDoc["edges"] = append(cyclicDoc["edges"].([]any), response{"id": "cycle-back", "source": "c", "target": "a", "sourceHandle": "right", "targetHandle": "left"}) call("PATCH", base, response{"document": cyclicDoc}, 200, cyclicVersion, "") callsBeforeCycle := calls.Load() cycleRejected := call("POST", base+"/ai/runs", chainCommand, 409, cyclicVersion+1, key(17)) if cycleRejected["error"] != "execution_cycle" || calls.Load() != callsBeforeCycle || call("GET", base, nil, 200, 0, "")["version"] != float64(cyclicVersion+1) { t.Fatal("cycle admission had side effects") } }