Add AI company research workbench backed by the shared site database

apps/docs/src/lib/ai-research-store.ts

@@ -0,0 +1,87 @@
+import seed from "../../../../content/research/ai-companies.seed.json";
+import { createHash } from "node:crypto";
+import { openSiteDatabase } from "./site-database";
+import { validateResearch, type ResearchSnapshot } from "./ai-research";
+
+// Each release selects an immutable research revision. New revisions are added
+// transactionally, preserving both earlier research and all authentication rows.
+export function readResearchSnapshot(snapshot = seed as ResearchSnapshot): ResearchSnapshot {
+  validateResearch(snapshot);
+  const digest = createHash("sha256").update(JSON.stringify(snapshot)).digest("hex");
+  const db = openSiteDatabase();
+  try {
+    db.exec(`
+      CREATE TABLE IF NOT EXISTS research_datasets (
+        id TEXT PRIMARY KEY, as_of TEXT NOT NULL, metadata TEXT NOT NULL, content_hash TEXT NOT NULL
+      );
+      CREATE TABLE IF NOT EXISTS research_categories (
+        dataset_id TEXT NOT NULL REFERENCES research_datasets(id),
+        id TEXT NOT NULL, payload TEXT NOT NULL, PRIMARY KEY(dataset_id,id)
+      );
+      CREATE TABLE IF NOT EXISTS research_companies (
+        dataset_id TEXT NOT NULL, id TEXT NOT NULL, category_id TEXT NOT NULL,
+        name TEXT NOT NULL, founded_year INTEGER, team_size INTEGER,
+        revenue_status TEXT NOT NULL, payload TEXT NOT NULL,
+        PRIMARY KEY(dataset_id,id),
+        FOREIGN KEY(dataset_id,category_id) REFERENCES research_categories(dataset_id,id)
+      );
+      CREATE TABLE IF NOT EXISTS research_evidence (
+        dataset_id TEXT NOT NULL REFERENCES research_datasets(id),
+        id TEXT NOT NULL, url TEXT NOT NULL, payload TEXT NOT NULL,
+        PRIMARY KEY(dataset_id,id)
+      );
+      CREATE TABLE IF NOT EXISTS research_observations (
+        dataset_id TEXT NOT NULL, id TEXT NOT NULL, company_id TEXT NOT NULL,
+        evidence_id TEXT NOT NULL, kind TEXT NOT NULL, amount REAL NOT NULL,
+        unit TEXT NOT NULL, record_use TEXT NOT NULL, payload TEXT NOT NULL,
+        PRIMARY KEY(dataset_id,id),
+        FOREIGN KEY(dataset_id,company_id) REFERENCES research_companies(dataset_id,id),
+        FOREIGN KEY(dataset_id,evidence_id) REFERENCES research_evidence(dataset_id,id)
+      );
+      CREATE TABLE IF NOT EXISTS research_reviews (
+        dataset_id TEXT NOT NULL REFERENCES research_datasets(id),
+        id TEXT NOT NULL, payload TEXT NOT NULL, PRIMARY KEY(dataset_id,id)
+      );
+    `);
+    // The database transaction also serializes first requests from two workers.
+    db.exec("BEGIN IMMEDIATE");
+    try {
+      const exists = db.prepare("SELECT content_hash FROM research_datasets WHERE id=?")
+        .get(snapshot.version) as { content_hash: string } | undefined;
+      if (exists && exists.content_hash !== digest)
+        throw new Error("Research data changed without a new revision identifier");
+      if (!exists) {
+        const { categories, companies, observations, evidence, reviews, ...metadata } = snapshot;
+        db.prepare("INSERT INTO research_datasets VALUES (?,?,?,?)")
+          .run(snapshot.version, snapshot.asOf, JSON.stringify(metadata), digest);
+        const insertCategory = db.prepare("INSERT INTO research_categories VALUES (?,?,?)");
+        for (const row of categories) insertCategory.run(snapshot.version, row.id, JSON.stringify(row));
+        const insertCompany = db.prepare("INSERT INTO research_companies VALUES (?,?,?,?,?,?,?,?)");
+        for (const row of companies) insertCompany.run(snapshot.version, row.id, row.cat,
+          row.name, row.year, row.team, row.revenueStatus, JSON.stringify(row));
+        const insertEvidence = db.prepare("INSERT INTO research_evidence VALUES (?,?,?,?)");
+        for (const row of evidence) insertEvidence.run(snapshot.version, row.id, row.url, JSON.stringify(row));
+        const insertObservation = db.prepare("INSERT INTO research_observations VALUES (?,?,?,?,?,?,?,?,?)");
+        for (const row of observations) insertObservation.run(snapshot.version, row.id, row.company,
+          row.evidence, row.kind, row.value, row.unit, row.selection, JSON.stringify(row));
+        const insertReview = db.prepare("INSERT INTO research_reviews VALUES (?,?,?)");
+        for (const row of reviews) insertReview.run(snapshot.version, row.id, JSON.stringify(row));
+      }
+      db.exec("COMMIT");
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw error;
+    }
+    const meta = db.prepare("SELECT metadata FROM research_datasets WHERE id=?").get(snapshot.version) as { metadata: string };
+    // All displayed records come from the site database, not the seed arrays.
+    const rows = (table: string) => (db.prepare(
+      `SELECT payload FROM ${table} WHERE dataset_id=? ORDER BY id`,
+    ).all(snapshot.version) as { payload: string }[]).map((r) => JSON.parse(r.payload));
+    const stored: ResearchSnapshot = { ...JSON.parse(meta.metadata),
+      categories: rows("research_categories"), companies: rows("research_companies"),
+      observations: rows("research_observations"), evidence: rows("research_evidence"),
+      reviews: rows("research_reviews") };
+    validateResearch(stored);
+    return stored;
+  } finally { db.close(); }
+}

apps/docs/src/lib/ai-research.ts

@@ -0,0 +1,79 @@
+export type ResearchCategory = { id: string; name: string; description: string };
+export type ResearchCompany = {
+  id: string; name: string; url: string; cat: string; year: number | null;
+  team: number | null; location: string | null; business: string; buyer: string;
+  delivery: string; notes: string; cohort: string; teamBasis: string | null;
+  price: string; stage: string; sourceType: string; evidenceIds: string[];
+  revenueStatus: "self" | "estimate" | "unknown";
+  extraUrl?: string; pricingEvidence?: string;
+};
+export type ResearchObservation = {
+  id: string; company: string; name: string; kind: string; value: number;
+  unit: string; timing: string; evidence: string; scope: string;
+  selection: string; reason: string; bound: string; comparisonGroup: string | null;
+};
+export type ResearchEvidence = {
+  id: string; companyId: string | null; title: string; url: string;
+  publishedAt: string | null; accessedAt: string; kind: string;
+  summary: string; limitation: string;
+};
+export type ResearchReview = {
+  id: string; type: string; company: string; summary: string;
+  evidenceIds: string[]; handling: string;
+};
+export type ResearchSnapshot = {
+  version: string; asOf: string; scope: string; limitations: string[];
+  categories: ResearchCategory[]; companies: ResearchCompany[];
+  observations: ResearchObservation[]; evidence: ResearchEvidence[];
+  reviews: ResearchReview[];
+};
+
+export function evidenceUrl(value: string): string | undefined {
+  try {
+    const url = new URL(value);
+    return ["https:", "http:"].includes(url.protocol) ? url.href : undefined;
+  } catch { return undefined; }
+}
+
+export function validateResearch(snapshot: ResearchSnapshot) {
+  const unique = (values: string[], label: string) => {
+    if (new Set(values).size !== values.length || values.some((v) => !v))
+      throw new Error(`Duplicate or missing research ${label}`);
+    return new Set(values);
+  };
+  if (!snapshot.version || !/^\d{4}-\d{2}-\d{2}$/.test(snapshot.asOf))
+    throw new Error("Invalid research dataset metadata");
+  const categories = unique(snapshot.categories.map((c) => c.id), "categories");
+  const companies = unique(snapshot.companies.map((c) => c.id), "companies");
+  const evidence = unique(snapshot.evidence.map((e) => e.id), "evidence");
+  unique(snapshot.observations.map((o) => o.id), "observations");
+  unique(snapshot.reviews.map((r) => r.id), "reviews");
+  for (const c of snapshot.companies) {
+    if (!categories.has(c.cat) || !evidenceUrl(c.url) ||
+        c.evidenceIds.some((id) => !evidence.has(id)) ||
+        (c.pricingEvidence && !evidence.has(c.pricingEvidence)))
+      throw new Error(`Invalid research company reference: ${c.id}`);
+  }
+  for (const e of snapshot.evidence) {
+    if (!evidenceUrl(e.url) || (e.companyId && !companies.has(e.companyId)))
+      throw new Error(`Invalid research evidence: ${e.id}`);
+  }
+  for (const o of snapshot.observations) {
+    if (!companies.has(o.company) || !evidence.has(o.evidence) ||
+        !Number.isFinite(o.value))
+      throw new Error(`Invalid research observation: ${o.id}`);
+  }
+  for (const r of snapshot.reviews)
+    if (r.evidenceIds.some((id) => !evidence.has(id)))
+      throw new Error(`Invalid research review: ${r.id}`);
+}
+
+export function categorySummary(snapshot: ResearchSnapshot) {
+  return snapshot.categories.map((category) => {
+    const companies = snapshot.companies.filter((c) => c.cat === category.id);
+    return { ...category, count: companies.length,
+      self: companies.filter((c) => c.revenueStatus === "self").length,
+      estimate: companies.filter((c) => c.revenueStatus === "estimate").length,
+      unknown: companies.filter((c) => c.revenueStatus === "unknown").length };
+  });
+}

apps/docs/src/lib/auth-store.ts

@@ -1,14 +1,8 @@
-import { chmodSync } from "node:fs";
-import { dirname, join } from "node:path";
-import { DatabaseSync } from "node:sqlite";
 import { authId, matchingStep, type AuthConfig } from "./auth";
+import { openSiteDatabase } from "./site-database";
 
 function database() {
-  const configFile = process.env.AUTH_CONFIG_FILE;
-  if (!configFile) throw new Error("Authentication is not configured");
-  const file = join(dirname(configFile), "login-state.sqlite");
-  const db = new DatabaseSync(file);
-  chmodSync(file, 0o600);
+  const db = openSiteDatabase();
   db.exec(`PRAGMA busy_timeout=5000;
     CREATE TABLE IF NOT EXISTS login_state (
       id TEXT PRIMARY KEY, window_start INTEGER NOT NULL,

apps/docs/src/lib/site-database.ts

@@ -0,0 +1,20 @@
+import { chmodSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+// Keep the historical filename so existing authentication state is preserved.
+// Research and authentication use different tables in this same site database.
+export function openSiteDatabase() {
+  const configFile = process.env.AUTH_CONFIG_FILE;
+  if (!configFile) throw new Error("Authentication is not configured");
+  const file = join(dirname(configFile), "login-state.sqlite");
+  const db = new DatabaseSync(file);
+  try {
+    chmodSync(file, 0o600);
+    db.exec("PRAGMA busy_timeout=5000; PRAGMA foreign_keys=ON;");
+    return db;
+  } catch (error) {
+    db.close();
+    throw error;
+  }
+}