merge: integrate database documents, preserve all proposals and keep CI main-only
This commit is contained in:
@@ -10,7 +10,9 @@ ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA = json.loads((ROOT / 'content/research/ai-companies.seed.json').read_text())
|
||||
AUDIT = json.loads((ROOT / 'content/research/ai-companies.coverage.json').read_text())
|
||||
SOURCES = json.loads((ROOT / 'content/research/ai-companies.sources.json').read_text())
|
||||
TARGET = ROOT / 'content/docs/ai-company-research/02-coverage-and-gaps.md'
|
||||
TARGET = ROOT / '.artifacts/reports/02-coverage-and-gaps.md'
|
||||
SOURCE_SLUG = 'ai-company-research/02-coverage-and-gaps'
|
||||
|
||||
BASE = '/docs/other-research/ai-company-research/overview'
|
||||
|
||||
|
||||
@@ -208,13 +210,18 @@ def render():
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--check', action='store_true')
|
||||
parser.add_argument('--document-export', type=Path)
|
||||
args = parser.parse_args()
|
||||
content = render()
|
||||
if args.check:
|
||||
if not TARGET.exists() or TARGET.read_text() != content:
|
||||
sys.exit('Research coverage report differs; review and regenerate')
|
||||
print('Research coverage accounting and report are consistent')
|
||||
if args.document_export is not None:
|
||||
bundle = json.loads(args.document_export.read_text())
|
||||
stored = next((d['content'] for d in bundle['documents'] if d['sourceSlug'] == SOURCE_SLUG), None)
|
||||
if stored != content:
|
||||
sys.exit('Research coverage report differs; review and regenerate')
|
||||
print('Research coverage accounting verified; supplied document export compared when requested')
|
||||
else:
|
||||
TARGET.parent.mkdir(parents=True, exist_ok=True)
|
||||
TARGET.write_text(content)
|
||||
print(TARGET.relative_to(ROOT))
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ pnpm run lint
|
||||
pnpm run typecheck
|
||||
pnpm --filter @endor/docs run test:auth
|
||||
pnpm --filter @endor/docs run test:research
|
||||
pnpm --filter @endor/docs run test:docs
|
||||
pnpm --filter @endor/docs run images
|
||||
pnpm --filter @endor/docs run test:images
|
||||
cp -R apps/docs/public/figures /tmp/expected-figures
|
||||
|
||||
+14
-2
@@ -15,7 +15,7 @@ APP = "endorphin-tabularium-docs"
|
||||
INPUTS = ["Dockerfile", "Dockerfile.ci", ".dockerignore", ".npmrc", "package.json",
|
||||
"pnpm-lock.yaml", "pnpm-workspace.yaml", "apps/docs", "content", "tools/figures",
|
||||
"scripts/check-container.sh", "scripts/ci.py", "scripts/summarize-ai-income.py",
|
||||
"scripts/audit-ai-research.py"]
|
||||
"scripts/audit-ai-research.py", "scripts/export-legacy-documents.py"]
|
||||
|
||||
|
||||
def run(*args: str, capture: bool = False) -> str:
|
||||
@@ -36,9 +36,17 @@ def fingerprint() -> str:
|
||||
|
||||
|
||||
def smoke_image(image: str, name: str) -> None:
|
||||
run("docker", "run", "-d", "--name", name, "--mount", "type=volume,dst=/auth", "-p", "127.0.0.1::3000", image)
|
||||
run("docker", "run", "-d", "--name", name, "--mount", "type=volume,dst=/auth", "-e", "DOCUMENT_SMOKE_TEST=1", "-p", "127.0.0.1::3000", image)
|
||||
try:
|
||||
run("docker", "exec", name, "node", "apps/docs/auth-cli.cjs", "init")
|
||||
subprocess.run(["docker", "exec", "-i", name, "node", "apps/docs/docs-cli.cjs", "import"],
|
||||
input=(ROOT / ".artifacts/documents-migration.json").read_bytes(), check=True)
|
||||
run("docker", "exec", name, "node", "apps/docs/docs-cli.cjs", "export", "/auth/migrated-documents.json")
|
||||
migrated = json.loads(run("docker", "exec", name, "cat", "/auth/migrated-documents.json", capture=True))
|
||||
expected = json.loads((ROOT / ".artifacts/documents-migration.json").read_text())
|
||||
if migrated != expected:
|
||||
raise RuntimeError("Migration changed document content or catalog metadata")
|
||||
run("docker", "exec", name, "node", "apps/docs/docs-cli.cjs", "backup", "/auth/smoke-backup.sqlite")
|
||||
for _ in range(60):
|
||||
state = json.loads(run("docker", "inspect", name, capture=True))[0]["State"]
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
@@ -64,6 +72,10 @@ def smoke_image(image: str, name: str) -> None:
|
||||
|
||||
def main() -> None:
|
||||
os.chdir(ROOT)
|
||||
migration = ROOT / ".artifacts/documents-migration.json"
|
||||
migration.parent.mkdir(exist_ok=True)
|
||||
migration.unlink(missing_ok=True)
|
||||
run(sys.executable, "scripts/export-legacy-documents.py", "--output", str(migration))
|
||||
run(sys.executable, "scripts/summarize-ai-income.py", "--check")
|
||||
run(sys.executable, "scripts/audit-ai-research.py", "--check")
|
||||
run(sys.executable, "-m", "unittest", "discover", "-s", "scripts", "-p", "test_*.py")
|
||||
|
||||
@@ -7,6 +7,7 @@ import re
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.request
|
||||
from ci import APP, ROOT, fingerprint, run
|
||||
|
||||
@@ -108,6 +109,21 @@ def main() -> None:
|
||||
run("docker", "run", "--rm", "--network", "none", "--mount",
|
||||
f"type=bind,src={auth_dir},dst=/auth", actual,
|
||||
"node", "apps/docs/auth-cli.cjs", "init")
|
||||
# Back up the entire shared database, then bootstrap documents only when empty.
|
||||
# A normal release never reimports, overwrites or resets authored DB content.
|
||||
migration = ROOT / ".artifacts/documents-migration.json"
|
||||
if not migration.is_file():
|
||||
raise RuntimeError("Verified document migration export is missing")
|
||||
backup = f"/auth/before-documents-{sha}-{time.time_ns()}.sqlite"
|
||||
run("docker", "run", "--rm", "--network", "none", "--mount",
|
||||
f"type=bind,src={auth_dir},dst=/auth", actual,
|
||||
"node", "apps/docs/docs-cli.cjs", "backup", backup)
|
||||
subprocess.run(["docker", "run", "--rm", "-i", "--network", "none", "--mount",
|
||||
f"type=bind,src={auth_dir},dst=/auth", actual,
|
||||
"node", "apps/docs/docs-cli.cjs", "import", "--if-empty"],
|
||||
input=migration.read_bytes(), check=True)
|
||||
shutil.copyfile(ROOT / "scripts/docs-admin.sh", DEPLOY_ROOT / "docs")
|
||||
(DEPLOY_ROOT / "docs").chmod(0o750)
|
||||
shutil.copyfile(ROOT / "scripts/auth-admin.sh", DEPLOY_ROOT / "auth")
|
||||
(DEPLOY_ROOT / "auth").chmod(0o750)
|
||||
current = DEPLOY_ROOT / "current"
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Run on ces-control. JSON content travels through stdin, never shell arguments.
|
||||
set -eu
|
||||
cd /opt/endorphin-tabularium-docs/current
|
||||
case "${1:-}" in
|
||||
list|get|save|history|export|backup|import)
|
||||
docker compose -p endorphin-tabularium-docs --env-file /etc/endorphin-tabularium-docs/production.env --env-file release.env -f compose.yml exec -T docs node apps/docs/docs-cli.cjs "$@" ;;
|
||||
*) echo 'Usage: docs list | get <slug> | save < JSON | history <sourceSlug> | export <private-file> | backup <private-file> | import < JSON' >&2; exit 1 ;;
|
||||
esac
|
||||
@@ -0,0 +1,48 @@
|
||||
"""One-time export from an explicit historical Git revision, never a runtime seed."""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASELINE = 'cffca6942b1bc0005321e1685f38593fb0b853d5' # Integration snapshot of PRs 14, 15 and 16 before database migration.
|
||||
|
||||
|
||||
def export(ref):
|
||||
commit = subprocess.check_output(['git', 'rev-parse', '--verify', ref + '^{commit}'], cwd=ROOT, text=True).strip()
|
||||
|
||||
def read(name):
|
||||
return subprocess.check_output(['git', 'show', f'{commit}:{name}'], cwd=ROOT).decode()
|
||||
|
||||
catalog = json.loads(read('content/catalog.json'))
|
||||
names = subprocess.check_output(['git', 'ls-tree', '-r', '--name-only', commit, '--', 'content/docs'], cwd=ROOT, text=True).splitlines()
|
||||
documents = []
|
||||
for name in sorted(n for n in names if n.endswith('.md')):
|
||||
source = name.removeprefix('content/docs/').removesuffix('.md')
|
||||
metadata = catalog['documents'][source]
|
||||
collection = next(c for c in catalog['collections'] if not c['path'] or source.startswith(c['path'] + '/'))
|
||||
content = read(name)
|
||||
title = re.search(r'^# (.+)$', content, re.M)
|
||||
documents.append(dict(slug=metadata['slug'], sourceSlug=source, category=metadata['category'],
|
||||
title=title[1].strip() if title else source, content=content,
|
||||
group=metadata.get('group', collection['title']), collection=metadata.get('collection', collection['title']),
|
||||
description=metadata.get('description', collection['description']),
|
||||
draft=metadata.get('draft', False), featured=metadata.get('featured', False), revision=1))
|
||||
if not documents:
|
||||
raise ValueError('Refusing empty migration')
|
||||
return dict(format='tabularium-documents-v1', documents=documents)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--ref', default=BASELINE)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
payload = export(args.ref)
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
descriptor = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
|
||||
with os.fdopen(descriptor, 'w') as output:
|
||||
json.dump(payload, output, ensure_ascii=False)
|
||||
print(f"Exported {len(payload['documents'])} documents; source is immutable Git history")
|
||||
@@ -9,7 +9,8 @@ import sys
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DATA = json.loads((ROOT / 'content/research/ai-companies.seed.json').read_text())
|
||||
REFERENCE = json.loads((ROOT / 'content/research/ai-income-analysis.json').read_text())
|
||||
TARGET = ROOT / 'content/docs/ai-company-research/01-income-snapshot.md'
|
||||
TARGET = ROOT / '.artifacts/reports/01-income-snapshot.md'
|
||||
SOURCE_SLUG = 'ai-company-research/01-income-snapshot'
|
||||
BASE = '/docs/other-research/ai-company-research/overview'
|
||||
COMPANIES = {c['id']: c for c in DATA['companies']}
|
||||
CATEGORIES = {c['id']: c['name'] for c in DATA['categories']}
|
||||
@@ -137,14 +138,19 @@ def render():
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--check', action='store_true')
|
||||
parser.add_argument('--document-export', type=Path)
|
||||
args = parser.parse_args()
|
||||
verify()
|
||||
content = render()
|
||||
if args.check:
|
||||
if not TARGET.exists() or TARGET.read_text() != content:
|
||||
sys.exit('Income snapshot differs; regenerate after reviewing data changes')
|
||||
print('Income reference, samples and report are consistent')
|
||||
if args.document_export is not None:
|
||||
bundle = json.loads(args.document_export.read_text())
|
||||
stored = next((d['content'] for d in bundle['documents'] if d['sourceSlug'] == SOURCE_SLUG), None)
|
||||
if stored != content:
|
||||
sys.exit('Income snapshot differs; regenerate after reviewing data changes')
|
||||
print('Income reference and samples verified; supplied document export compared when requested')
|
||||
else:
|
||||
TARGET.parent.mkdir(parents=True, exist_ok=True)
|
||||
TARGET.write_text(content)
|
||||
print(TARGET.relative_to(ROOT))
|
||||
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
"""Guard cache invalidation against stale documents and needless metadata rebuilds."""
|
||||
"""Guard cache invalidation against stale application code and needless metadata rebuilds."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
@@ -13,7 +13,7 @@ class FingerprintTests(unittest.TestCase):
|
||||
root = Path(directory)
|
||||
subprocess.run(["git", "init", "-q", str(root)], check=True)
|
||||
(root / ".gitignore").write_text(".artifacts/\n**/.next/\n")
|
||||
doc = root / "content/docs/concept.md"
|
||||
doc = root / "apps/docs/src/lib/docs.ts"
|
||||
doc.parent.mkdir(parents=True)
|
||||
doc.write_text("# Original\n")
|
||||
package = root / "package.json"
|
||||
|
||||
Reference in New Issue
Block a user