feat: make the site database authoritative for documents and catalog

apps/docs/scripts/auth-cli.ts

@@ -18,6 +18,7 @@ import {
   totp,
   isLoginRedirect,
 } from "../src/lib/auth";
+import { readDocument, saveDocument } from "../src/lib/docs";
 import { consumeCode, initializeStore } from "../src/lib/auth-store";
 
 async function main() {
@@ -132,11 +133,6 @@ async function main() {
     join(__dirname, ".next/BUILD_ID"),
     "utf8",
   ).trim();
-  const routes = Object.keys(
-    JSON.parse(
-      readFileSync(join(__dirname, ".next/prerender-manifest.json"), "utf8"),
-    ).routes,
-  ) as string[];
   for (const path of [
     "/",
     "/docs/00-endor-concepts-draft",
@@ -160,7 +156,7 @@ async function main() {
       path,
     );
   }
-  for (const path of ["/api/private", "/api/unknown", "/api/search", "/api/research/ai-companies"])
+  for (const path of ["/api/private", "/api/unknown", "/api/search", "/api/documents/source?slug=endor/endor-purpose-draft", "/api/research/ai-companies"])
     assert.equal((await request(path)).status, 401, path);
   assert.equal(
     (
@@ -209,6 +205,32 @@ async function main() {
     assert.equal(destination.searchParams.get("from"), "legacy");
     assert.ok(legacy.headers.get("cache-control")?.includes("no-store"));
   }
+  const routes = ["/", ...searchIndex.map((doc: { slug: string }) => `/docs/${doc.slug}`)];
+  const raw = await request("/api/documents/source?slug=endor/endor-purpose-draft", { headers });
+  assert.equal(raw.status, 200);
+  assert.ok((await raw.text()).startsWith("# Endor"));
+  if (process.env.DOCUMENT_SMOKE_TEST) {
+    const sourceSlug = "ci-live-document";
+    const previous = readDocument(sourceSlug);
+    if (command === "exercise") {
+      const first = saveDocument({ slug: "endor/ci-live-document", sourceSlug,
+        title: "Runtime document", category: "Endor", collection: "Endor 文档", group: "CI",
+        description: "Isolated smoke test", draft: true, featured: false,
+        content: "# Runtime document\n\nInitial body" }, previous?.revision ?? 0);
+      assert.equal((await request(`/docs/${first.slug}`, { headers })).status, 200);
+      saveDocument({ ...first, slug: "endor/ci-live-document-renamed", content: "# Runtime document\n\nUpdated without rebuilding" }, first.revision);
+    }
+    const stored = readDocument(sourceSlug);
+    assert.ok(stored?.content.includes("Updated without rebuilding"), "DB content survives restart");
+    const live = await request(`/docs/${stored.slug}`, { headers });
+    assert.equal(live.status, 200);
+    assert.ok((await live.text()).includes("Updated without rebuilding"));
+    const alias = await request("/docs/endor/ci-live-document?live=1", { headers });
+    assert.equal(alias.status, 308);
+    assert.ok(alias.headers.get("location")?.includes("ci-live-document-renamed?live=1"));
+    const updatedIndex = await (await request("/api/search", { headers })).json();
+    assert.ok(updatedIndex.some((doc: { searchText: string }) => doc.searchText.includes("Updated without rebuilding")));
+  }
   let home = "";
   let purpose = "";
   for (const route of routes) {

apps/docs/scripts/build-auth.mjs

@@ -7,3 +7,8 @@ await build({
   format: "cjs",
   outfile: ".next/standalone/apps/docs/auth-cli.cjs",
 });
+
+await build({
+  entryPoints: ["scripts/docs-cli.ts"], bundle: true, platform: "node", target: "node22",
+  format: "cjs", outfile: ".next/standalone/apps/docs/docs-cli.cjs",
+});

apps/docs/scripts/build-doc-images.mjs

@@ -23,19 +23,6 @@ for (const name of await files(path.join(root, "apps/docs/public"))) {
     throw new Error(`Original raster exposed in public: ${path.relative(root, name)}. Move it to tools/figures/assets and run images.`);
   }
 }
-for (const name of await files(path.join(root, "content/docs"))) {
-  if (!name.endsWith(".md")) continue;
-  const content = await readFile(name, "utf8");
-  const images = [
-    ...content.matchAll(/!\[[^\]]*\]\(<?([^\s)>]+)/g),
-    ...content.matchAll(/<img\b[^>]*\bsrc=["']([^"']+)/gi),
-  ];
-  for (const [, source] of images) {
-    if (/\.(?:png|jpe?g|tiff?|bmp)(?:[?#]|$)/i.test(source)) {
-      throw new Error(`${path.relative(root, name)} embeds an original raster: ${source}. Use the compressed WebP.`);
-    }
-  }
-}
 
 await mkdir(output, { recursive: true });
 for (const source of await files(sources)) {

apps/docs/scripts/docs-cli.ts

@@ -0,0 +1,44 @@
+import { readFileSync, writeFileSync } from "node:fs";
+import { openSiteDatabase } from "../src/lib/site-database";
+import { allDocuments, readDocument, saveDocument, importDocuments, documentHistory } from "../src/lib/docs";
+
+process.umask(0o077);
+const [command, argument] = process.argv.slice(2);
+try {
+  switch (command) {
+    case "backup": {
+      if (!argument) throw Error("Provide a private backup filename");
+      const db = openSiteDatabase();
+      try { db.prepare("VACUUM INTO ?").run(argument); } finally { db.close(); }
+      console.log("Full site database backup created; existing files are never overwritten"); break;
+    }
+    case "list":
+      console.log(JSON.stringify(allDocuments().map(({ content: _, ...doc }) => { void _; return doc; }), null, 2)); break;
+    case "get": {
+      const doc = readDocument(argument ?? "");
+      if (!doc) throw Error("Document not found");
+      console.log(JSON.stringify(doc, null, 2)); break;
+    }
+    case "save": {
+      const { revision, ...input } = JSON.parse(readFileSync(0, "utf8"));
+      const doc = saveDocument(input, revision);
+      console.log(JSON.stringify({ slug: doc.slug, revision: doc.revision })); break;
+    }
+    case "history":
+      console.log(JSON.stringify(documentHistory(argument ?? ""), null, 2)); break;
+    case "import": {
+      const bundle = JSON.parse(readFileSync(0, "utf8"));
+      console.log(JSON.stringify({ imported: importDocuments(bundle, argument === "--if-empty") })); break;
+    }
+    case "export": {
+      if (!argument) throw Error("Provide a private export filename");
+      writeFileSync(argument, JSON.stringify({ format: "tabularium-documents-v1", documents: allDocuments() }),
+        { flag: "wx", mode: 0o600 });
+      console.log("Document export created; existing files are never overwritten"); break;
+    }
+    default: throw Error("Usage: docs backup <private-file> | list | get <slug> | save < input.json | history <sourceSlug> | import [--if-empty] < export.json | export <private-file>");
+  }
+} catch (error) {
+  console.error(error instanceof Error ? error.message : "Document operation failed");
+  process.exitCode = 1;
+}

apps/docs/scripts/test-docs.mjs

@@ -0,0 +1,14 @@
+import { build } from "esbuild";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { spawnSync } from "node:child_process";
+
+const dir = await mkdtemp(join(tmpdir(), "tabularium-docs-tests-"));
+try {
+  const file = join(dir, "docs.test.cjs");
+  await build({ entryPoints: ["tests/docs.test.ts"], bundle: true,
+    platform: "node", target: "node22", format: "cjs", outfile: file });
+  const result = spawnSync(process.execPath, ["--test", file], { stdio: "inherit" });
+  process.exitCode = result.status ?? 1;
+} finally { await rm(dir, { recursive: true, force: true }); }