feat: make the site database authoritative for documents and catalog

scripts/audit-ai-research.py

@@ -9,7 +9,8 @@ from urllib.parse import urlsplit
 ROOT = Path(__file__).resolve().parents[1]
 DATA = json.loads((ROOT / 'content/research/ai-companies.seed.json').read_text())
 AUDIT = json.loads((ROOT / 'content/research/ai-companies.coverage.json').read_text())
-TARGET = ROOT / 'content/docs/ai-company-research/02-coverage-and-gaps.md'
+TARGET = ROOT / '.artifacts/reports/02-coverage-and-gaps.md'
+SOURCE_SLUG = 'ai-company-research/02-coverage-and-gaps'
 BASE = '/docs/other-research/ai-company-research/overview'
 
 
@@ -95,13 +96,17 @@ def render():
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--check', action='store_true')
+    parser.add_argument('--document-export', type=Path, default=ROOT / '.artifacts/documents-migration.json')
     args = parser.parse_args()
     content = render()
     if args.check:
-        if not TARGET.exists() or TARGET.read_text() != content:
+        bundle = json.loads(args.document_export.read_text())
+        stored = next((d['content'] for d in bundle['documents'] if d['sourceSlug'] == SOURCE_SLUG), None)
+        if stored != content:
             sys.exit('Research coverage report differs; review and regenerate')
         print('Research coverage accounting and report are consistent')
     else:
+        TARGET.parent.mkdir(parents=True, exist_ok=True)
         TARGET.write_text(content)
         print(TARGET.relative_to(ROOT))
 

scripts/check-container.sh

@@ -7,6 +7,7 @@ pnpm run lint
 pnpm run typecheck
 pnpm --filter @endor/docs run test:auth
 pnpm --filter @endor/docs run test:research
+pnpm --filter @endor/docs run test:docs
 pnpm --filter @endor/docs run images
 pnpm --filter @endor/docs run test:images
 cp -R apps/docs/public/figures /tmp/expected-figures

scripts/ci.py

@@ -15,7 +15,7 @@ APP = "endorphin-tabularium-docs"
 INPUTS = ["Dockerfile", "Dockerfile.ci", ".dockerignore", ".npmrc", "package.json",
           "pnpm-lock.yaml", "pnpm-workspace.yaml", "apps/docs", "content", "tools/figures",
           "scripts/check-container.sh", "scripts/ci.py", "scripts/summarize-ai-income.py",
-          "scripts/audit-ai-research.py"]
+          "scripts/audit-ai-research.py", "scripts/export-legacy-documents.py"]
 
 
 def run(*args: str, capture: bool = False) -> str:
@@ -36,9 +36,17 @@ def fingerprint() -> str:
 
 
 def smoke_image(image: str, name: str) -> None:
-    run("docker", "run", "-d", "--name", name, "--mount", "type=volume,dst=/auth", "-p", "127.0.0.1::3000", image)
+    run("docker", "run", "-d", "--name", name, "--mount", "type=volume,dst=/auth", "-e", "DOCUMENT_SMOKE_TEST=1", "-p", "127.0.0.1::3000", image)
     try:
         run("docker", "exec", name, "node", "apps/docs/auth-cli.cjs", "init")
+        subprocess.run(["docker", "exec", "-i", name, "node", "apps/docs/docs-cli.cjs", "import"],
+                       input=(ROOT / ".artifacts/documents-migration.json").read_bytes(), check=True)
+        run("docker", "exec", name, "node", "apps/docs/docs-cli.cjs", "export", "/auth/migrated-documents.json")
+        migrated = json.loads(run("docker", "exec", name, "cat", "/auth/migrated-documents.json", capture=True))
+        expected = json.loads((ROOT / ".artifacts/documents-migration.json").read_text())
+        if migrated != expected:
+            raise RuntimeError("Migration changed document content or catalog metadata")
+        run("docker", "exec", name, "node", "apps/docs/docs-cli.cjs", "backup", "/auth/smoke-backup.sqlite")
         for _ in range(60):
             state = json.loads(run("docker", "inspect", name, capture=True))[0]["State"]
             if state.get("Health", {}).get("Status") == "healthy":
@@ -64,6 +72,10 @@ def smoke_image(image: str, name: str) -> None:
 
 def main() -> None:
     os.chdir(ROOT)
+    migration = ROOT / ".artifacts/documents-migration.json"
+    migration.parent.mkdir(exist_ok=True)
+    migration.unlink(missing_ok=True)
+    run(sys.executable, "scripts/export-legacy-documents.py", "--output", str(migration))
     run(sys.executable, "scripts/summarize-ai-income.py", "--check")
     run(sys.executable, "scripts/audit-ai-research.py", "--check")
     run(sys.executable, "-m", "unittest", "discover", "-s", "scripts", "-p", "test_*.py")

scripts/deploy.py

@@ -7,6 +7,7 @@ import re
 import shutil
 import socket
 import subprocess
+import time
 import urllib.request
 from ci import APP, ROOT, fingerprint, run
 
@@ -108,6 +109,21 @@ def main() -> None:
         run("docker", "run", "--rm", "--network", "none", "--mount",
             f"type=bind,src={auth_dir},dst=/auth", actual,
             "node", "apps/docs/auth-cli.cjs", "init")
+        # Back up the entire shared database, then bootstrap documents only when empty.
+        # A normal release never reimports, overwrites or resets authored DB content.
+        migration = ROOT / ".artifacts/documents-migration.json"
+        if not migration.is_file():
+            raise RuntimeError("Verified document migration export is missing")
+        backup = f"/auth/before-documents-{sha}-{time.time_ns()}.sqlite"
+        run("docker", "run", "--rm", "--network", "none", "--mount",
+            f"type=bind,src={auth_dir},dst=/auth", actual,
+            "node", "apps/docs/docs-cli.cjs", "backup", backup)
+        subprocess.run(["docker", "run", "--rm", "-i", "--network", "none", "--mount",
+                        f"type=bind,src={auth_dir},dst=/auth", actual,
+                        "node", "apps/docs/docs-cli.cjs", "import", "--if-empty"],
+                       input=migration.read_bytes(), check=True)
+        shutil.copyfile(ROOT / "scripts/docs-admin.sh", DEPLOY_ROOT / "docs")
+        (DEPLOY_ROOT / "docs").chmod(0o750)
         shutil.copyfile(ROOT / "scripts/auth-admin.sh", DEPLOY_ROOT / "auth")
         (DEPLOY_ROOT / "auth").chmod(0o750)
         current = DEPLOY_ROOT / "current"

scripts/docs-admin.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+# Run on ces-control. JSON content travels through stdin, never shell arguments.
+set -eu
+cd /opt/endorphin-tabularium-docs/current
+case "${1:-}" in
+  list|get|save|history|export|backup|import)
+    docker compose -p endorphin-tabularium-docs --env-file /etc/endorphin-tabularium-docs/production.env --env-file release.env -f compose.yml exec -T docs node apps/docs/docs-cli.cjs "$@" ;;
+  *) echo 'Usage: docs list | get <slug> | save < JSON | history <sourceSlug> | export <private-file> | backup <private-file> | import < JSON' >&2; exit 1 ;;
+esac

scripts/export-legacy-documents.py

@@ -0,0 +1,48 @@
+"""One-time export from an explicit historical Git revision, never a runtime seed."""
+import argparse
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[1]
+BASELINE = '25c5e83d08eae4277fe429a692955bd3bb2e717f'  # Last main revision before database-backed documents.
+
+
+def export(ref):
+    commit = subprocess.check_output(['git', 'rev-parse', '--verify', ref + '^{commit}'], cwd=ROOT, text=True).strip()
+
+    def read(name):
+        return subprocess.check_output(['git', 'show', f'{commit}:{name}'], cwd=ROOT).decode()
+
+    catalog = json.loads(read('content/catalog.json'))
+    names = subprocess.check_output(['git', 'ls-tree', '-r', '--name-only', commit, '--', 'content/docs'], cwd=ROOT, text=True).splitlines()
+    documents = []
+    for name in sorted(n for n in names if n.endswith('.md')):
+        source = name.removeprefix('content/docs/').removesuffix('.md')
+        metadata = catalog['documents'][source]
+        collection = next(c for c in catalog['collections'] if not c['path'] or source.startswith(c['path'] + '/'))
+        content = read(name)
+        title = re.search(r'^# (.+)$', content, re.M)
+        documents.append(dict(slug=metadata['slug'], sourceSlug=source, category=metadata['category'],
+            title=title[1].strip() if title else source, content=content,
+            group=metadata.get('group', collection['title']), collection=metadata.get('collection', collection['title']),
+            description=metadata.get('description', collection['description']),
+            draft=metadata.get('draft', False), featured=metadata.get('featured', False), revision=1))
+    if not documents:
+        raise ValueError('Refusing empty migration')
+    return dict(format='tabularium-documents-v1', documents=documents)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--ref', default=BASELINE)
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    payload = export(args.ref)
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
+    with os.fdopen(descriptor, 'w') as output:
+        json.dump(payload, output, ensure_ascii=False)
+    print(f"Exported {len(payload['documents'])} documents; source is immutable Git history")

scripts/summarize-ai-income.py

@@ -9,7 +9,8 @@ import sys
 ROOT = Path(__file__).resolve().parents[1]
 DATA = json.loads((ROOT / 'content/research/ai-companies.seed.json').read_text())
 REFERENCE = json.loads((ROOT / 'content/research/ai-income-analysis.json').read_text())
-TARGET = ROOT / 'content/docs/ai-company-research/01-income-snapshot.md'
+TARGET = ROOT / '.artifacts/reports/01-income-snapshot.md'
+SOURCE_SLUG = 'ai-company-research/01-income-snapshot'
 BASE = '/docs/other-research/ai-company-research/overview'
 COMPANIES = {c['id']: c for c in DATA['companies']}
 CATEGORIES = {c['id']: c['name'] for c in DATA['categories']}
@@ -137,14 +138,18 @@ def render():
 def main():
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument('--check', action='store_true')
+    parser.add_argument('--document-export', type=Path, default=ROOT / '.artifacts/documents-migration.json')
     args = parser.parse_args()
     verify()
     content = render()
     if args.check:
-        if not TARGET.exists() or TARGET.read_text() != content:
+        bundle = json.loads(args.document_export.read_text())
+        stored = next((d['content'] for d in bundle['documents'] if d['sourceSlug'] == SOURCE_SLUG), None)
+        if stored != content:
             sys.exit('Income snapshot differs; regenerate after reviewing data changes')
         print('Income reference, samples and report are consistent')
     else:
+        TARGET.parent.mkdir(parents=True, exist_ok=True)
         TARGET.write_text(content)
         print(TARGET.relative_to(ROOT))
 

scripts/test_ci.py

@@ -1,4 +1,4 @@
-"""Guard cache invalidation against stale documents and needless metadata rebuilds."""
+"""Guard cache invalidation against stale application code and needless metadata rebuilds."""
 from pathlib import Path
 import subprocess
 import tempfile
@@ -13,7 +13,7 @@ class FingerprintTests(unittest.TestCase):
             root = Path(directory)
             subprocess.run(["git", "init", "-q", str(root)], check=True)
             (root / ".gitignore").write_text(".artifacts/\n**/.next/\n")
-            doc = root / "content/docs/concept.md"
+            doc = root / "apps/docs/src/lib/docs.ts"
             doc.parent.mkdir(parents=True)
             doc.write_text("# Original\n")
             package = root / "package.json"