From d57e9b77d7254ceb2e0aefa45c799b6550a39577 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 30 Oct 2026 16:23:54 +0800 Subject: [PATCH] feat: make the site database authoritative for documents and catalog --- .dockerignore | 4 + .gitea/workflows/ci-deploy.yml | 1 + .gitea/workflows/docs-database-verify.yml | 22 ++ .gitignore | 5 + AGENTS.md | 2 +- README.md | 84 +++-- apps/docs/package.json | 3 +- apps/docs/scripts/auth-cli.ts | 34 +- apps/docs/scripts/build-auth.mjs | 5 + apps/docs/scripts/build-doc-images.mjs | 13 - apps/docs/scripts/docs-cli.ts | 44 +++ apps/docs/scripts/test-docs.mjs | 14 + apps/docs/src/lib/docs.ts | 173 +++++++-- apps/docs/src/pages/api/documents/source.ts | 22 ++ apps/docs/src/pages/api/search.ts | 13 +- apps/docs/src/pages/docs/[...slug].tsx | 30 +- apps/docs/src/pages/index.tsx | 11 +- apps/docs/src/proxy.ts | 13 - apps/docs/tests/docs.test.ts | 99 +++++ content/catalog.json | 315 ---------------- content/docs/00-endor-concepts-draft.md | 133 ------- content/docs/01-getting-started.md | 30 -- content/docs/02-mounting-draft.md | 25 -- content/docs/03-figures.md | 42 --- content/docs/04-endor-purpose-draft.md | 337 ------------------ content/docs/05-object-graphs-gc.md | 148 -------- content/docs/06-agent-programming-draft.md | 83 ----- content/docs/07-composable-judgments-draft.md | 180 ---------- .../docs/ai-company-research/00-overview.md | 64 ---- .../ai-company-research/01-income-snapshot.md | 64 ---- .../02-coverage-and-gaps.md | 73 ---- content/docs/language-design/00-overview.md | 50 --- content/docs/language-design/02-questions.md | 196 ---------- content/docs/language-design/03-questions.md | 250 ------------- content/docs/language-design/04-questions.md | 161 --------- content/docs/language-design/05-questions.md | 144 -------- content/docs/language-design/06-questions.md | 173 --------- content/docs/language-design/07-questions.md | 182 ---------- content/docs/language-design/08-questions.md | 157 -------- content/docs/language-design/09-questions.md | 168 --------- content/docs/language-design/10-questions.md | 136 ------- content/docs/language-design/11-questions.md | 168 --------- content/docs/language-design/12-questions.md | 163 --------- content/docs/language-design/13-questions.md | 175 --------- content/docs/language-design/14-questions.md | 136 ------- content/docs/pretext/00-overview.md | 24 -- content/docs/pretext/01-chapter.md | 45 --- content/docs/pretext/02-chapter.md | 61 ---- content/docs/pretext/03-chapter.md | 54 --- content/docs/pretext/04-chapter.md | 56 --- content/docs/pretext/05-chapter.md | 60 ---- content/docs/pretext/06-chapter.md | 52 --- content/docs/pretext/07-chapter.md | 38 -- content/docs/pretext/08-chapter.md | 41 --- content/docs/pretext/09-chapter.md | 94 ----- content/docs/pretext/10-chapter.md | 75 ---- content/docs/pretext/11-chapter.md | 21 -- content/docs/pretext/12-chapter.md | 27 -- content/docs/smalltalk/00-overview.md | 51 --- content/docs/smalltalk/01-messages.md | 53 --- .../docs/smalltalk/02-syntax-and-blocks.md | 81 ----- .../docs/smalltalk/03-classes-and-methods.md | 76 ---- content/docs/smalltalk/04-object-protocol.md | 51 --- content/docs/smalltalk/05-reflection.md | 44 --- .../smalltalk/06-collections-and-streams.md | 58 --- content/docs/smalltalk/07-processes.md | 35 -- content/docs/smalltalk/08-graphics.md | 49 --- content/docs/smalltalk/09-live-environment.md | 47 --- content/docs/smalltalk/10-mvc.md | 61 ---- content/docs/smalltalk/11-reference.md | 68 ---- package.json | 5 +- scripts/audit-ai-research.py | 9 +- scripts/check-container.sh | 1 + scripts/ci.py | 16 +- scripts/deploy.py | 16 + scripts/docs-admin.sh | 9 + scripts/export-legacy-documents.py | 48 +++ scripts/summarize-ai-income.py | 9 +- scripts/test_ci.py | 4 +- 79 files changed, 562 insertions(+), 5222 deletions(-) create mode 100644 .gitea/workflows/docs-database-verify.yml create mode 100644 apps/docs/scripts/docs-cli.ts create mode 100644 apps/docs/scripts/test-docs.mjs create mode 100644 apps/docs/src/pages/api/documents/source.ts create mode 100644 apps/docs/tests/docs.test.ts delete mode 100644 content/catalog.json delete mode 100644 content/docs/00-endor-concepts-draft.md delete mode 100644 content/docs/01-getting-started.md delete mode 100644 content/docs/02-mounting-draft.md delete mode 100644 content/docs/03-figures.md delete mode 100644 content/docs/04-endor-purpose-draft.md delete mode 100644 content/docs/05-object-graphs-gc.md delete mode 100644 content/docs/06-agent-programming-draft.md delete mode 100644 content/docs/07-composable-judgments-draft.md delete mode 100644 content/docs/ai-company-research/00-overview.md delete mode 100644 content/docs/ai-company-research/01-income-snapshot.md delete mode 100644 content/docs/ai-company-research/02-coverage-and-gaps.md delete mode 100644 content/docs/language-design/00-overview.md delete mode 100644 content/docs/language-design/02-questions.md delete mode 100644 content/docs/language-design/03-questions.md delete mode 100644 content/docs/language-design/04-questions.md delete mode 100644 content/docs/language-design/05-questions.md delete mode 100644 content/docs/language-design/06-questions.md delete mode 100644 content/docs/language-design/07-questions.md delete mode 100644 content/docs/language-design/08-questions.md delete mode 100644 content/docs/language-design/09-questions.md delete mode 100644 content/docs/language-design/10-questions.md delete mode 100644 content/docs/language-design/11-questions.md delete mode 100644 content/docs/language-design/12-questions.md delete mode 100644 content/docs/language-design/13-questions.md delete mode 100644 content/docs/language-design/14-questions.md delete mode 100644 content/docs/pretext/00-overview.md delete mode 100644 content/docs/pretext/01-chapter.md delete mode 100644 content/docs/pretext/02-chapter.md delete mode 100644 content/docs/pretext/03-chapter.md delete mode 100644 content/docs/pretext/04-chapter.md delete mode 100644 content/docs/pretext/05-chapter.md delete mode 100644 content/docs/pretext/06-chapter.md delete mode 100644 content/docs/pretext/07-chapter.md delete mode 100644 content/docs/pretext/08-chapter.md delete mode 100644 content/docs/pretext/09-chapter.md delete mode 100644 content/docs/pretext/10-chapter.md delete mode 100644 content/docs/pretext/11-chapter.md delete mode 100644 content/docs/pretext/12-chapter.md delete mode 100644 content/docs/smalltalk/00-overview.md delete mode 100644 content/docs/smalltalk/01-messages.md delete mode 100644 content/docs/smalltalk/02-syntax-and-blocks.md delete mode 100644 content/docs/smalltalk/03-classes-and-methods.md delete mode 100644 content/docs/smalltalk/04-object-protocol.md delete mode 100644 content/docs/smalltalk/05-reflection.md delete mode 100644 content/docs/smalltalk/06-collections-and-streams.md delete mode 100644 content/docs/smalltalk/07-processes.md delete mode 100644 content/docs/smalltalk/08-graphics.md delete mode 100644 content/docs/smalltalk/09-live-environment.md delete mode 100644 content/docs/smalltalk/10-mvc.md delete mode 100644 content/docs/smalltalk/11-reference.md create mode 100644 scripts/docs-admin.sh create mode 100644 scripts/export-legacy-documents.py diff --git a/.dockerignore b/.dockerignore index 557f4ea..a0d3814 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,3 +8,7 @@ .artifacts apps/docs/public/figures/**/*.webp .env* + +**/*.sqlite +**/*.sqlite-* +**/*.db diff --git a/.gitea/workflows/ci-deploy.yml b/.gitea/workflows/ci-deploy.yml index 59b3211..1b8a614 100644 --- a/.gitea/workflows/ci-deploy.yml +++ b/.gitea/workflows/ci-deploy.yml @@ -22,6 +22,7 @@ jobs: with: github-server-url: ${{ gitea.server_url }} persist-credentials: false + fetch-depth: 0 - name: Verify with persistent caches or reuse verified image run: python3 scripts/ci.py - name: Deploy latest main with health checks and rollback diff --git a/.gitea/workflows/docs-database-verify.yml b/.gitea/workflows/docs-database-verify.yml new file mode 100644 index 0000000..db91bcf --- /dev/null +++ b/.gitea/workflows/docs-database-verify.yml @@ -0,0 +1,22 @@ +name: Verify database-backed documents +on: + pull_request: + branches: [main] + workflow_dispatch: + +jobs: + verify: + if: gitea.head_ref == 'codex/database-docs' || (gitea.event_name == 'workflow_dispatch' && gitea.ref == 'refs/heads/codex/database-docs') + runs-on: ces-control + timeout-minutes: 45 + env: + DOCKER_BUILDKIT: "0" + CI_CACHE_SCOPE: pr-${{ gitea.event.pull_request.number || '0' }} + steps: + - uses: actions/checkout@v4 + with: + github-server-url: ${{ gitea.server_url }} + persist-credentials: false + fetch-depth: 0 + - name: Validate migration and runtime without deployment + run: python3 scripts/ci.py diff --git a/.gitignore b/.gitignore index 785350d..b6504da 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,8 @@ __pycache__/ # Browser raster derivatives are generated on the confirmed remote runner. /apps/docs/public/figures/**/*.webp + +# Application data and content exports belong in private persistent storage. +*.sqlite +*.sqlite-* +*.db diff --git a/AGENTS.md b/AGENTS.md index ab3fbac..bc62dc8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ - 本仓库专门维护 Endor 文档与制图脚本。支持本地开发；CI/CD 使用现有 ces-control Runner，在 main 通过验证后自动部署。不得修改其他项目、共享 Runner 配置或清理共享 Docker 缓存。 - Web 技术栈为 TypeScript、React、Tailwind CSS、Next.js Pages Router；不要切换到 App Router。 - 使用 pnpm 管理 JavaScript workspace，使用 uv 管理 `tools/figures` 的 Python 项目，保留两套锁文件。 -- Markdown 正文放在 `content/docs`，Python 图形源码放在 `tools/figures/src/endor_figures`，可复现的 SVG 输出放在 `apps/docs/public/figures`。 +- 文档正文、目录元数据与修订记录以站点现有 SQLite 数据库为唯一权威源，使用 docs 管理工具直接更新；不得把新正文、目录 JSON、数据库文件或日常内容快照提交到仓库。历史 Git 文档只用于显式的一次性迁移，不作为运行时种子。Python 图形源码放在 `tools/figures/src/endor_figures`，可复现的 SVG 输出放在 `apps/docs/public/figures`。 - 未确认的架构提议明确标注为草案，不能表述为现有实现。 - 本地开发与预览只监听 `127.0.0.1`。部署容器监听 `0.0.0.0:3000`，服务器入口默认 18135，可通过独立 production.env 配置。 - 对 gitea.entroia.com 仅使用 codex-bot，提交身份为 Codex Bot 。SSH 使用隔离配置：`ssh -F /dev/null -o IdentitiesOnly=yes -i /Users/zzyang/.codex/credentials/gitea-entroia/codex_entroia_ed25519 -p 22222`。 diff --git a/README.md b/README.md index ee59b68..1dcc267 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,67 @@ # Endor · Tabularium -Endor 文档与制图 monorepo。Tabularium 取古罗马档案馆之意。支持本地开发，并由 Gitea Actions 在 ces-control 上验证与部署。 +Endor 文档站、研究工作台与制图程序。应用代码在 Git 中评审；全部文档正文、目录元数据和内容修订以站点 SQLite 数据库为唯一权威源。日常写作不需要 commit、构建或部署。 -## 本地开发 +## 存储与读取 -需要 Node.js 22.13+（推荐 LTS）、pnpm 11.25、Python 3.11+ 和 uv。 -首次安装依赖需要访问软件包源。本地开发服务只监听 127.0.0.1。 +沿用 `AUTH_CONFIG_FILE` 同目录的 `login-state.sqlite` 和 `/auth` 持久挂载。`documents` 存正文及分类、合集、分组、摘要、草案与推荐状态；`document_aliases` 保留旧地址；`document_revisions` 保存每次修改；`document_imports` 标记一次性迁移。没有第二套文档数据库。 + +首页、阅读页、目录和全文搜索按请求查询数据库，使用 `private, no-store`。旧文件式地址与改名前的语义地址通过数据库别名跳转，保留查询参数；不再在构建阶段枚举文档。Markdown 原文通过受认证保护的 `/api/documents/source?slug=...` 获取。 + +仓库不保存日常正文、目录 JSON、数据库文件或内容快照；历史 Git 记录仅用于迁移。页面不再读取 `content/docs` 或 `content/catalog.json`。Markdown 与 GFM 表格仍可用，不解析原始 HTML。图片在写入时检查压缩格式；图形程序与版本化静态资源仍在代码仓库，本轮不迁移媒体存储。研究业务数据留待下一步。 + +## 在服务器上管理文档 + +部署安装 `/opt/endorphin-tabularium-docs/docs`，通过当前容器执行管理 CLI。需要服务器的运维权限；不增加可公开写入的 API，不改变现有动态码登录。 ```sh -pnpm install --frozen-lockfile -uv sync --project tools/figures --locked -pnpm dev +sudo /opt/endorphin-tabularium-docs/docs list +sudo /opt/endorphin-tabularium-docs/docs get endor/endor-purpose-draft +sudo /opt/endorphin-tabularium-docs/docs history 04-endor-purpose-draft ``` -访问 http://127.0.0.1:3000 ，先使用验证器动态码登录。首次本地使用需按下文初始化私有认证配置；缺少配置时正文关闭访问。站点无外部字体或 CDN 资源。 +新建或修改使用标准输入传入完整 JSON。必填字段：`slug`、`sourceSlug`、`title`、`category`、`collection`、`group`、`description`、`draft`、`featured`、`content` 和 `revision`。新建时 `revision: 0`；修改时先 get 并使用读到的 revision，避免覆盖别人刚写入的版本。`sourceSlug` 是稳定标识，修改公共 slug 时保持它不变，旧地址继续可用。调整目录直接修改 category、collection、group 等字段。 -```text -apps/docs/ Next.js Pages Router + React + TypeScript + Tailwind CSS -content/docs/ Markdown 文档源，支持目录，路径成为 /docs/ 路由 -tools/figures/ 独立 Python 项目，使用 drawsvg 编程制图 +```sh +sudo /opt/endorphin-tabularium-docs/docs get endor/endor-purpose-draft > /private/work/purpose.json +# 在私有文件中修改正文或目录字段，保留读到的 revision。 +sudo /opt/endorphin-tabularium-docs/docs save < /private/work/purpose.json ``` -## 命令 +路径仅为操作示例，应选择实际存在且受保护的目录。运行成功后刷新网页即可看到正文、导航和搜索更新。版本冲突时重新读取并合并修改；不要把旧 revision 改成新数字强行覆盖。草案是现有展示标记，不是按分支或用户隔离的访问控制。 -- `pnpm dev`：本地开发，修改 Markdown 后刷新页面即可读取新内容。 -- `pnpm figures`：运行 Python 制图，更新 `apps/docs/public/figures`。 -- `pnpm lint` / `pnpm typecheck`：前端静态检查。 -- `pnpm build`：构建文档站。 -- `pnpm start`：在本地预览已构建站点。 -- `pnpm check`：依次执行 lint、类型检查、制图与构建。 +新增正文不提交回 Git。奇思妙想这类内容同样通过数据库写入；代码分支与数据发布是不同动作，不能因为迁移程序合并就自动发布其他未合并分支的内容。 -站点默认浅色，使用本地系统字体。全站搜索支持正文关键词、`⌘/Ctrl K`、方向键与 Enter；代码块提供复制与反馈。桌面提供分组导航和滚动高亮的页内目录，小屏可展开文档菜单与页内目录。首页入口、草案提示、前后篇导航均引用现有 Markdown 内容和资料目录配置。顶部提供文档切换，阅读页左侧只展示当前文档，并有独立图标、名称与返回入口；搜索框标明当前文档范围。界面主要通过留白、轻渐变与状态反馈分层，支持减少动态效果的系统偏好。 +## 首次迁移与备份 -参考设计截图与研究记录保存在 `design/references`；浏览器验收截图与检查说明保存在 `design/verification`。这些文件不作为站点静态资源发布。 +迁移前主线的 50 篇正文及目录可从固定历史提交 `25c5e83d08eae4277fe429a692955bd3bb2e717f` 完整导出： -文档标题取首个 `# 标题`，文档排序按相对路径。支持多层目录（例如 `content/docs/pretext/01-chapter.md` 对应 `/docs/pretext/01-chapter`）；旧文档地址保持可用。`content/catalog.json` 管理合集、分类、摘要、草案与推荐，新增资料无需修改页面代码。首页推荐最多3篇，目录每页8条，可按合集与标题筛选；全站搜索仍覆盖全文。相邻篇目限于当前合集。 +```sh +python3 scripts/export-legacy-documents.py --output .artifacts/documents-migration.json +``` -Pretext 教程位于 `content/docs/pretext`，包含导读与12章，来自本地 Pretext 研究成果。源码链接锁定上游研究版本；实验脚本与原始记录位于 `apps/docs/public/materials/pretext`，和正文一样经过登录验证。普通 Markdown 与 GFM 表格可用，嵌入图使用 `/figures/.svg`。不解析原始 HTML。 +导出文件位于 Git 忽略目录、权限 0600，不进入镜像。脚本也支持显式 `--ref`，但不能用未评审分支替换生产迁移基线。它不会在站点读取时执行。CI 使用历史导出验证全部正文与目录的逐项一致性，构建镜像只含程序，不含文档快照或数据库。 -语言设计问题集位于 `content/docs/language-design`，是第三个资料合集，包含总览与原编号02–14的13个分类，共271项议题。原讨论已确认26项，其余保持待讨论；来源映射和本次状态记录位于 `content/language-design-sources.json`。维护时同步章节、总览计数与来源状态；不会自动从聊天同步，也不把设计记录等同于已实现功能。 +首次部署先备份整个共享数据库，再显式事务导入到空文档表。普通部署使用 `import --if-empty`，已有内容时跳过，不以历史数据覆盖数据库编辑；重复同一导入也不会重置内容。导入失败则事务整体回滚，既有登录与研究表保留，部署中止。 -生产构建会生成文档路由；新增或修改文档后重新运行 build。开发模式按请求读取内容。 -main 推送通过 CI 验证后自动部署；Pull Request 只验证，不部署。工作流也支持手动触发。 +```sh +sudo /opt/endorphin-tabularium-docs/docs backup /auth/manual-backup.sqlite +sudo /opt/endorphin-tabularium-docs/docs export /auth/current-documents.json +``` -## Endor 概念图 +backup 是包含登录、研究、正文、别名和修订的完整 SQLite 一致性备份；export 只导出当前正文与目录，作为空库初始化/外部迁移快照，不代替完整备份或恢复修订历史。输出路径均在容器内，文件已存在时拒绝覆盖。备份含认证状态，应留在私有挂载或受控备份位置，不能放到公开静态目录、仓库或日志。 -`content/docs/00-endor-concepts-draft.md` 解释全球入口、区域网关与 Runtime、分片、CDN、身份及跨区恢复的概念设计，页面为 `/docs/00-endor-concepts-draft`。 +代码回退不会撤销数据库编辑，但回退到旧文件式站点会显示旧正文。恢复完整数据库会同时影响登录和研究状态，必须先停止写入、保留当前备份并由管理员明确选择恢复点；部署脚本不会自动恢复数据库或删除新内容。 -图形源码位于 `tools/figures/src/endor_figures/architecture.py`，由 `pnpm figures` 生成 `apps/docs/public/figures/endor-architecture.svg`。Python 负责可复现的节点、全连接和复制路径，正文直接显示图片，点击可放大；大图支持原尺寸查看、滚动和 Escape/空白处关闭。无需外部图标、字体或绘图服务。图中的数量是示意，身份服务/CDN 接入属于建议，所有未实现的关系均以草案说明。 +## 代码验证 -拓扑检查：`uv run --locked --project tools/figures python -m unittest discover -s tools/figures/tests -v`。容器检查流程会运行同一检查。 +使用 React、Next.js Pages Router、TypeScript 和 Tailwind CSS。JavaScript 用 pnpm，制图 Python 用 uv。编译、构建、打包和构建产物全部在已确认的 ces-control Runner 执行；`pnpm test:docs` 等测试脚本会调用 esbuild，也只能在 Runner 执行。 + +迁移 PR 有专用验证工作流，执行 lint、类型检查、数据库事务/版本测试、现有认证与研究测试、图形检查、构建与容器冒烟，不部署。主线验证通过后才沿用既有部署流程，仍等待用户手动合并 PR。 + +冒烟使用隔离数据库卷：完整导入后逐项核对全部正文与目录，验证新建、更新、重命名、搜索立即变化及容器重启后的持久化，并覆盖全部文档地址、认证、媒体和原文端点。测试卷不会触及生产数据库。 + +`tools/figures/src/endor_figures` 与 `apps/docs/public/figures` 仍保存图形源码及可复现的静态资源。图形构建也在远程执行。 ## 动态码登录与二维码 @@ -68,13 +80,7 @@ sudo /opt/endorphin-tabularium-docs/auth verify `qr`只允许交互终端显示；`qr-png`用于通过SSH将PNG直接重定向到私有本地文件，不在CI中运行。二维码等同登录凭据，不放入本仓库或网页。只有明确执行 `auth rotate --confirm` 才更换密钥并使旧二维码、会话失效；重新显示或下载二维码不会轮换。 -本地先构建CLI，再初始化独立的本地测试配置： - -```sh -pnpm build -AUTH_CONFIG_FILE=/private/tmp/tabularium-auth/config.json node apps/docs/.next/standalone/apps/docs/auth-cli.cjs init -AUTH_CONFIG_FILE=/private/tmp/tabularium-auth/config.json AUTH_COOKIE_SECURE=false pnpm dev -``` +构建、打包与会触发构建的测试只在用户确认的 ces-control Runner 上执行。本地不执行 `pnpm dev`、`pnpm build`、`pnpm typecheck` 或打包测试。 当前服务器入口为HTTP，与参考项目一致使用 `AUTH_COOKIE_SECURE=false`；配置HTTPS后应在独立production.env设置 `AUTH_COOKIE_SECURE=true`。HTTP本身不加密动态码和会话传输。 @@ -87,7 +93,7 @@ AUTH_CONFIG_FILE=/private/tmp/tabularium-auth/config.json AUTH_COOKIE_SECURE=fal 公司明细、收入观测、原始证据和数据核查记录。所有页面与 `GET /api/research/ai-companies` 均使用文档站现有的登录认证。 -研究数据与登录状态**共用文档站现有的 SQLite 数据库文件**： +研究业务数据本轮暂不迁移，仍沿用版本化 JSON 导入机制。研究数据、文档与登录状态**共用文档站现有的 SQLite 数据库文件**： `AUTH_CONFIG_FILE` 同目录下的 `login-state.sqlite`。该文件名是历史名称，未创建第二个数据库。 `login_state` 保留登录限流与重放状态，新增的 `research_*` 表保存数据版本、分类、 公司、经营指标、证据与核查记录。沿用现有 `/auth` 持久挂载，数据库不会写入镜像或公开目录。 @@ -123,7 +129,7 @@ MRR×12仅作为显示单位转换，不改写为公司ARR；公司ARR、估算 覆盖审计页位于 `/docs/other-research/ai-company-research/coverage`，分类链接 `overview?category=` 的筛选数据。 `python3 scripts/audit-ai-research.py --check` 检查候选池逐条计数与覆盖报告一致性；CI在ces-control运行此检查，未更改触发范围。 收入快照报告位于 `/docs/other-research/ai-company-research/income-snapshot`，样本名称链接 `overview?company=` 的公司明细。 -`python3 scripts/summarize-ai-income.py --check` 独立复核统计与报告一致性；不触发编译或构建。数据变更后先审查方法，再运行不带参数的命令更新固定快照。 +`python3 scripts/summarize-ai-income.py --check` 独立复核统计与报告一致性；不触发编译或构建。数据变更后先审查方法，再运行不带参数的命令生成 `.artifacts/reports` 下的候选正文，使用 docs 管理工具显式更新数据库；不再生成受 Git 管理的正文。 默认7日样本14个产品；90日窗口新增Gojiberry的8月历史点，营销样本的均值和CV随之明显变化。 这提示先核查时间与样本构成，不能将高离散度直接解释为造假。两个近名营养应用按不同App Store ID分别建档。 diff --git a/apps/docs/package.json b/apps/docs/package.json index 2330f52..fa6fcea 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -11,7 +11,8 @@ "lint": "eslint . --max-warnings=0", "typecheck": "NEXT_TELEMETRY_DISABLED=1 next typegen && tsc --noEmit", "test:auth": "node scripts/test-auth.mjs", - "test:research": "node scripts/test-research.mjs" + "test:research": "node scripts/test-research.mjs", + "test:docs": "node scripts/test-docs.mjs" }, "dependencies": { "next": "16.4.3", diff --git a/apps/docs/scripts/auth-cli.ts b/apps/docs/scripts/auth-cli.ts index 4224810..0a48be3 100644 --- a/apps/docs/scripts/auth-cli.ts +++ b/apps/docs/scripts/auth-cli.ts @@ -18,6 +18,7 @@ import { totp, isLoginRedirect, } from "../src/lib/auth"; +import { readDocument, saveDocument } from "../src/lib/docs"; import { consumeCode, initializeStore } from "../src/lib/auth-store"; async function main() { @@ -132,11 +133,6 @@ async function main() { join(__dirname, ".next/BUILD_ID"), "utf8", ).trim(); - const routes = Object.keys( - JSON.parse( - readFileSync(join(__dirname, ".next/prerender-manifest.json"), "utf8"), - ).routes, - ) as string[]; for (const path of [ "/", "/docs/00-endor-concepts-draft", @@ -160,7 +156,7 @@ async function main() { path, ); } - for (const path of ["/api/private", "/api/unknown", "/api/search", "/api/research/ai-companies"]) + for (const path of ["/api/private", "/api/unknown", "/api/search", "/api/documents/source?slug=endor/endor-purpose-draft", "/api/research/ai-companies"]) assert.equal((await request(path)).status, 401, path); assert.equal( ( @@ -209,6 +205,32 @@ async function main() { assert.equal(destination.searchParams.get("from"), "legacy"); assert.ok(legacy.headers.get("cache-control")?.includes("no-store")); } + const routes = ["/", ...searchIndex.map((doc: { slug: string }) => `/docs/${doc.slug}`)]; + const raw = await request("/api/documents/source?slug=endor/endor-purpose-draft", { headers }); + assert.equal(raw.status, 200); + assert.ok((await raw.text()).startsWith("# Endor")); + if (process.env.DOCUMENT_SMOKE_TEST) { + const sourceSlug = "ci-live-document"; + const previous = readDocument(sourceSlug); + if (command === "exercise") { + const first = saveDocument({ slug: "endor/ci-live-document", sourceSlug, + title: "Runtime document", category: "Endor", collection: "Endor 文档", group: "CI", + description: "Isolated smoke test", draft: true, featured: false, + content: "# Runtime document\n\nInitial body" }, previous?.revision ?? 0); + assert.equal((await request(`/docs/${first.slug}`, { headers })).status, 200); + saveDocument({ ...first, slug: "endor/ci-live-document-renamed", content: "# Runtime document\n\nUpdated without rebuilding" }, first.revision); + } + const stored = readDocument(sourceSlug); + assert.ok(stored?.content.includes("Updated without rebuilding"), "DB content survives restart"); + const live = await request(`/docs/${stored.slug}`, { headers }); + assert.equal(live.status, 200); + assert.ok((await live.text()).includes("Updated without rebuilding")); + const alias = await request("/docs/endor/ci-live-document?live=1", { headers }); + assert.equal(alias.status, 308); + assert.ok(alias.headers.get("location")?.includes("ci-live-document-renamed?live=1")); + const updatedIndex = await (await request("/api/search", { headers })).json(); + assert.ok(updatedIndex.some((doc: { searchText: string }) => doc.searchText.includes("Updated without rebuilding"))); + } let home = ""; let purpose = ""; for (const route of routes) { diff --git a/apps/docs/scripts/build-auth.mjs b/apps/docs/scripts/build-auth.mjs index b28a834..b648e0d 100644 --- a/apps/docs/scripts/build-auth.mjs +++ b/apps/docs/scripts/build-auth.mjs @@ -7,3 +7,8 @@ await build({ format: "cjs", outfile: ".next/standalone/apps/docs/auth-cli.cjs", }); + +await build({ + entryPoints: ["scripts/docs-cli.ts"], bundle: true, platform: "node", target: "node22", + format: "cjs", outfile: ".next/standalone/apps/docs/docs-cli.cjs", +}); diff --git a/apps/docs/scripts/build-doc-images.mjs b/apps/docs/scripts/build-doc-images.mjs index 8989bf7..cdedcca 100644 --- a/apps/docs/scripts/build-doc-images.mjs +++ b/apps/docs/scripts/build-doc-images.mjs @@ -23,19 +23,6 @@ for (const name of await files(path.join(root, "apps/docs/public"))) { throw new Error(`Original raster exposed in public: ${path.relative(root, name)}. Move it to tools/figures/assets and run images.`); } } -for (const name of await files(path.join(root, "content/docs"))) { - if (!name.endsWith(".md")) continue; - const content = await readFile(name, "utf8"); - const images = [ - ...content.matchAll(/!\[[^\]]*\]\(]+)/g), - ...content.matchAll(/]*\bsrc=["']([^"']+)/gi), - ]; - for (const [, source] of images) { - if (/\.(?:png|jpe?g|tiff?|bmp)(?:[?#]|$)/i.test(source)) { - throw new Error(`${path.relative(root, name)} embeds an original raster: ${source}. Use the compressed WebP.`); - } - } -} await mkdir(output, { recursive: true }); for (const source of await files(sources)) { diff --git a/apps/docs/scripts/docs-cli.ts b/apps/docs/scripts/docs-cli.ts new file mode 100644 index 0000000..a3f6f50 --- /dev/null +++ b/apps/docs/scripts/docs-cli.ts @@ -0,0 +1,44 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import { openSiteDatabase } from "../src/lib/site-database"; +import { allDocuments, readDocument, saveDocument, importDocuments, documentHistory } from "../src/lib/docs"; + +process.umask(0o077); +const [command, argument] = process.argv.slice(2); +try { + switch (command) { + case "backup": { + if (!argument) throw Error("Provide a private backup filename"); + const db = openSiteDatabase(); + try { db.prepare("VACUUM INTO ?").run(argument); } finally { db.close(); } + console.log("Full site database backup created; existing files are never overwritten"); break; + } + case "list": + console.log(JSON.stringify(allDocuments().map(({ content: _, ...doc }) => { void _; return doc; }), null, 2)); break; + case "get": { + const doc = readDocument(argument ?? ""); + if (!doc) throw Error("Document not found"); + console.log(JSON.stringify(doc, null, 2)); break; + } + case "save": { + const { revision, ...input } = JSON.parse(readFileSync(0, "utf8")); + const doc = saveDocument(input, revision); + console.log(JSON.stringify({ slug: doc.slug, revision: doc.revision })); break; + } + case "history": + console.log(JSON.stringify(documentHistory(argument ?? ""), null, 2)); break; + case "import": { + const bundle = JSON.parse(readFileSync(0, "utf8")); + console.log(JSON.stringify({ imported: importDocuments(bundle, argument === "--if-empty") })); break; + } + case "export": { + if (!argument) throw Error("Provide a private export filename"); + writeFileSync(argument, JSON.stringify({ format: "tabularium-documents-v1", documents: allDocuments() }), + { flag: "wx", mode: 0o600 }); + console.log("Document export created; existing files are never overwritten"); break; + } + default: throw Error("Usage: docs backup | list | get | save < input.json | history | import [--if-empty] < export.json | export "); + } +} catch (error) { + console.error(error instanceof Error ? error.message : "Document operation failed"); + process.exitCode = 1; +} diff --git a/apps/docs/scripts/test-docs.mjs b/apps/docs/scripts/test-docs.mjs new file mode 100644 index 0000000..339d95b --- /dev/null +++ b/apps/docs/scripts/test-docs.mjs @@ -0,0 +1,14 @@ +import { build } from "esbuild"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawnSync } from "node:child_process"; + +const dir = await mkdtemp(join(tmpdir(), "tabularium-docs-tests-")); +try { + const file = join(dir, "docs.test.cjs"); + await build({ entryPoints: ["tests/docs.test.ts"], bundle: true, + platform: "node", target: "node22", format: "cjs", outfile: file }); + const result = spawnSync(process.execPath, ["--test", file], { stdio: "inherit" }); + process.exitCode = result.status ?? 1; +} finally { await rm(dir, { recursive: true, force: true }); } diff --git a/apps/docs/src/lib/docs.ts b/apps/docs/src/lib/docs.ts index dbdbc27..7f14bcb 100644 --- a/apps/docs/src/lib/docs.ts +++ b/apps/docs/src/lib/docs.ts @@ -1,50 +1,147 @@ -import fs from "node:fs/promises"; -import path from "node:path"; -import catalog from "../../../../content/catalog.json"; +import { randomUUID, createHash } from "node:crypto"; +import { openSiteDatabase } from "./site-database"; -const directory = path.resolve(process.cwd(), "../../content/docs"); -const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/; -type Metadata = { slug?: string; category?: string; collection?: string; group?: string; description?: string; draft?: boolean; featured?: boolean }; export type DocLink = { slug: string; sourceSlug: string; category: string; title: string; searchText?: string; group: string; collection: string; description: string; draft: boolean; featured: boolean; + revision: number; }; export type Doc = DocLink & { content: string }; +export type DocInput = Omit; +export type DocumentExport = { format: "tabularium-documents-v1"; documents: Doc[] }; +export const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/; + +function database() { + const db = openSiteDatabase(); + db.exec(` + CREATE TABLE IF NOT EXISTS documents ( + id TEXT PRIMARY KEY, slug TEXT NOT NULL UNIQUE, source_slug TEXT NOT NULL UNIQUE, + revision INTEGER NOT NULL, payload TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS document_aliases ( + alias TEXT PRIMARY KEY, document_id TEXT NOT NULL REFERENCES documents(id) + ); + CREATE TABLE IF NOT EXISTS document_revisions ( + document_id TEXT NOT NULL REFERENCES documents(id), revision INTEGER NOT NULL, + payload TEXT NOT NULL, created_at TEXT NOT NULL, PRIMARY KEY(document_id, revision) + ); + CREATE TABLE IF NOT EXISTS document_imports ( + digest TEXT PRIMARY KEY, document_count INTEGER NOT NULL, created_at TEXT NOT NULL + ); + `); + return db; +} + +export function validateDocument(input: DocInput): void { + if (!input || typeof input !== "object") throw Error("Invalid document"); + for (const key of ["slug", "sourceSlug"] as const) + if (typeof input[key] !== "string" || !validSlug.test(input[key])) throw Error(`Invalid ${key}`); + if (input.slug.split("/").some((part) => /^\d+-/.test(part))) throw Error("Use a semantic public slug"); + for (const key of ["title", "category", "collection", "group"] as const) + if (typeof input[key] !== "string" || !input[key].trim()) throw Error(`Missing ${key}`); + if (typeof input.content !== "string" || typeof input.description !== "string" || + typeof input.draft !== "boolean" || typeof input.featured !== "boolean") throw Error("Invalid document fields"); + if (input.content.length > 2_000_000) throw Error("Document exceeds content limit"); + const images = [ + ...input.content.matchAll(/!\[[^\]]*\]\(]+)/g), + ...input.content.matchAll(/]*\bsrc=["']([^"']+)/gi), + ]; + if (images.some(([, source]) => /\.(?:png|jpe?g|tiff?|bmp)(?:[?#]|$)/i.test(source))) + throw Error("Use compressed WebP images instead of original rasters"); +} + +function write(db: ReturnType, input: DocInput, expectedRevision: number): Doc { + validateDocument(input); + const existing = db.prepare("SELECT id, revision FROM documents WHERE source_slug=?") + .get(input.sourceSlug) as { id: string; revision: number } | undefined; + if ((existing?.revision ?? 0) !== expectedRevision) throw Error("Document revision conflict"); + const id = existing?.id ?? randomUUID(); + const doc: Doc = { slug: input.slug, sourceSlug: input.sourceSlug, category: input.category, + title: input.title, group: input.group, collection: input.collection, description: input.description, + draft: input.draft, featured: input.featured, content: input.content, revision: expectedRevision + 1 }; + const payload = JSON.stringify(doc); + // Aliases remain reserved after renaming; collision rolls back the whole write. + db.prepare(`INSERT INTO documents VALUES (?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET + slug=excluded.slug, revision=excluded.revision, payload=excluded.payload`) + .run(id, doc.slug, doc.sourceSlug, doc.revision, payload); + for (const alias of new Set([doc.slug, doc.sourceSlug])) { + const prior = db.prepare("SELECT document_id FROM document_aliases WHERE alias=?") + .get(alias) as { document_id: string } | undefined; + if (prior && prior.document_id !== id) throw Error(`Document alias collision: ${alias}`); + db.prepare("INSERT OR IGNORE INTO document_aliases VALUES (?,?)").run(alias, id); + } + db.prepare("INSERT INTO document_revisions VALUES (?,?,?,?)") + .run(id, doc.revision, payload, new Date().toISOString()); + return doc; +} + +export function saveDocument(input: DocInput, expectedRevision: number): Doc { + if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) throw Error("Provide expected revision"); + const db = database(); + try { + db.exec("BEGIN IMMEDIATE"); + try { const doc = write(db, input, expectedRevision); db.exec("COMMIT"); return doc; } + catch (error) { db.exec("ROLLBACK"); throw error; } + } finally { db.close(); } +} + +export function readDocument(slug: string): Doc | null { + if (!validSlug.test(slug)) return null; + const db = database(); + try { + const row = db.prepare(`SELECT d.payload FROM documents d JOIN document_aliases a + ON a.document_id=d.id WHERE a.alias=?`).get(slug) as { payload: string } | undefined; + return row ? JSON.parse(row.payload) as Doc : null; + } finally { db.close(); } +} + +export function allDocuments(): Doc[] { + const db = database(); + try { + return (db.prepare("SELECT payload FROM documents ORDER BY source_slug").all() as { payload: string }[]) + .map((row) => JSON.parse(row.payload) as Doc); + } finally { db.close(); } +} export async function readDoc(slug: string): Promise { - if (!validSlug.test(slug)) throw new Error("Invalid document slug"); - const sourceSlug = Object.entries(catalog.documents).find(([, metadata]) => metadata.slug === slug)?.[0] ?? slug; - const content = await fs.readFile(path.join(directory, `${sourceSlug}.md`), "utf8"); - const title = content.match(/^# (.+)$/m)?.[1]?.trim() ?? slug; - const collection = catalog.collections.find((c) => !c.path || sourceSlug.startsWith(`${c.path}/`))!; - const metadata: Metadata = (catalog.documents as Record)[sourceSlug] ?? {}; - if (!metadata.slug || !metadata.category || !validSlug.test(metadata.slug) || metadata.slug.split("/").some((part) => /^\d+-/.test(part))) { - throw new Error(`Register a semantic slug and category in content/catalog.json: ${sourceSlug}`); - } - return { slug: metadata.slug, sourceSlug, category: metadata.category, title, content, searchText: content, - group: metadata.group ?? collection.title, collection: metadata.collection ?? collection.title, - description: metadata.description ?? collection.description, - draft: metadata.draft ?? false, featured: metadata.featured ?? false, - }; + const doc = readDocument(slug); + if (!doc) throw Error("Document not found"); + return doc; } export async function listDocs(): Promise { - async function walk(folder: string): Promise { - const entries = await fs.readdir(path.join(directory, folder), { withFileTypes: true }); - const files = await Promise.all(entries.map(async (entry) => { - const relative = folder ? `${folder}/${entry.name}` : entry.name; - if (entry.isDirectory()) return walk(relative); - if (entry.isFile() && relative.endsWith(".md") && validSlug.test(relative.slice(0, -3))) - return [relative.slice(0, -3)]; - return []; - })); - return files.flat(); - } - const docs = await Promise.all((await walk("")).sort().map(async (slug) => { - const { content: _, searchText: __, ...doc } = await readDoc(slug); - void _; void __; - return doc; - })); - if (new Set(docs.map((doc) => doc.slug)).size !== docs.length) throw new Error("Duplicate public document slug"); - return docs; + return allDocuments().map(({ content: _, ...doc }) => { void _; return doc; }); +} + +// Explicit, atomic bootstrap only. Runtime reads never import repository data. +export function importDocuments(bundle: DocumentExport, onlyIfEmpty = false): number { + if (bundle?.format !== "tabularium-documents-v1" || !Array.isArray(bundle.documents) || !bundle.documents.length) + throw Error("Invalid or empty document export"); + bundle.documents.forEach(validateDocument); + const digest = createHash("sha256").update(JSON.stringify(bundle)).digest("hex"); + const db = database(); + try { + db.exec("BEGIN IMMEDIATE"); + try { + const imported = db.prepare("SELECT digest FROM document_imports WHERE digest=?").get(digest); + if (imported) { db.exec("COMMIT"); return 0; } + const count = (db.prepare("SELECT COUNT(*) AS n FROM documents").get() as { n: number }).n; + if (count) { + if (!onlyIfEmpty) throw Error("Bootstrap requires an empty document database"); + db.exec("COMMIT"); return 0; + } + for (const doc of bundle.documents) write(db, doc, 0); + db.prepare("INSERT INTO document_imports VALUES (?,?,?)").run(digest, bundle.documents.length, new Date().toISOString()); + db.exec("COMMIT"); return bundle.documents.length; + } catch (error) { db.exec("ROLLBACK"); throw error; } + } finally { db.close(); } +} + +export function documentHistory(sourceSlug: string): Doc[] { + const db = database(); + try { + return (db.prepare(`SELECT r.payload FROM document_revisions r JOIN documents d + ON r.document_id=d.id WHERE d.source_slug=? ORDER BY r.revision`).all(sourceSlug) as { payload: string }[]) + .map((row) => JSON.parse(row.payload) as Doc); + } finally { db.close(); } } diff --git a/apps/docs/src/pages/api/documents/source.ts b/apps/docs/src/pages/api/documents/source.ts new file mode 100644 index 0000000..3ca3a80 --- /dev/null +++ b/apps/docs/src/pages/api/documents/source.ts @@ -0,0 +1,22 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { readDocument } from "@/lib/docs"; + +// The proxy protects Markdown just like the rendered document and search index. +export default function handler(request: NextApiRequest, response: NextApiResponse) { + response.setHeader("Cache-Control", "private, no-store"); + if (request.method !== "GET") { + response.setHeader("Allow", "GET"); + return response.status(405).end(); + } + const slug = request.query.slug; + if (typeof slug !== "string") return response.status(400).json({ error: "缺少文档地址" }); + try { + const doc = readDocument(slug); + if (!doc) return response.status(404).end(); + response.setHeader("Content-Type", "text/plain; charset=utf-8"); + response.setHeader("X-Content-Type-Options", "nosniff"); + return response.status(200).send(doc.content); + } catch { + return response.status(503).json({ error: "文档数据库暂时不可用" }); + } +} diff --git a/apps/docs/src/pages/api/search.ts b/apps/docs/src/pages/api/search.ts index e531dde..0dccf33 100644 --- a/apps/docs/src/pages/api/search.ts +++ b/apps/docs/src/pages/api/search.ts @@ -1,5 +1,5 @@ import type { NextApiRequest, NextApiResponse } from "next"; -import { listDocs, readDoc } from "@/lib/docs"; +import { allDocuments } from "@/lib/docs"; // The proxy authenticates this endpoint, including index content. export default async function handler(request: NextApiRequest, response: NextApiResponse) { @@ -8,9 +8,10 @@ export default async function handler(request: NextApiRequest, response: NextApi response.setHeader("Allow", "GET"); return response.status(405).end(); } - const docs = await listDocs(); - const index = await Promise.all(docs.map(async (doc) => ({ - ...doc, searchText: (await readDoc(doc.slug)).content, - }))); - response.status(200).json(index); + try { + const index = allDocuments().map(({ content, ...doc }) => ({ ...doc, searchText: content })); + return response.status(200).json(index); + } catch { + return response.status(503).json({ error: "文档数据库暂时不可用" }); + } } diff --git a/apps/docs/src/pages/docs/[...slug].tsx b/apps/docs/src/pages/docs/[...slug].tsx index dfb7efc..b3a91f7 100644 --- a/apps/docs/src/pages/docs/[...slug].tsx +++ b/apps/docs/src/pages/docs/[...slug].tsx @@ -1,7 +1,6 @@ import type { - GetStaticPaths, - GetStaticProps, - InferGetStaticPropsType, + GetServerSideProps, + InferGetServerSidePropsType, } from "next"; import Head from "next/head"; import dynamic from "next/dynamic"; @@ -15,7 +14,7 @@ import { } from "react"; import ReactMarkdown from "react-markdown"; import remarkGfm from "remark-gfm"; -import { listDocs, readDoc, type Doc, type DocLink } from "@/lib/docs"; +import { listDocs, readDocument, type Doc, type DocLink } from "@/lib/docs"; import { Shell, Navigation } from "@/components/Shell"; import { ArchitectureFigure } from "@/components/ArchitectureFigure"; import { FigureGallery } from "@/components/FigureGallery"; @@ -26,16 +25,17 @@ import { headingsFor } from "@/lib/navigation"; type Props = { doc: Doc; docs: DocLink[] }; const AICompanyResearch = dynamic(() => import("@/components/AICompanyResearch")); -export const getStaticPaths: GetStaticPaths = async () => ({ - paths: (await listDocs()).map(({ slug }) => ({ params: { slug: slug.split("/") } })), - fallback: false, -}); -export const getStaticProps: GetStaticProps = async ({ params }) => { +export const getServerSideProps: GetServerSideProps = async ({ params, res, resolvedUrl }) => { + res.setHeader("Cache-Control", "private, no-store"); const slug = Array.isArray(params?.slug) ? params.slug.join("/") : params?.slug; if (typeof slug !== "string") return { notFound: true }; - const docs = await listDocs(); - if (!docs.some((d) => d.slug === slug)) return { notFound: true }; - return { props: { doc: await readDoc(slug), docs } }; + const doc = readDocument(slug); + if (!doc) return { notFound: true }; + if (slug !== doc.slug) { + const query = resolvedUrl.includes("?") ? resolvedUrl.slice(resolvedUrl.indexOf("?")) : ""; + return { redirect: { destination: `/docs/${doc.slug}${query}`, permanent: true } }; + } + return { props: { doc, docs: await listDocs() } }; }; function textOf(value: ReactNode): string { return Children.toArray(value) @@ -49,7 +49,7 @@ function textOf(value: ReactNode): string { export default function DocPage({ doc, docs, -}: InferGetStaticPropsType) { +}: InferGetServerSidePropsType) { const headings = headingsFor(doc.content); const [active, setActive] = useState(""); @@ -231,9 +231,9 @@ export default function DocPage({