From 6bfeea4e7ecf9651057a83783b633fc643d661fe Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Fri, 30 Oct 2026 18:14:33 +0800 Subject: [PATCH] feat: add scoped document API and resumable batch uploader --- AGENTS.md | 2 +- README.md | 70 +++++++- apps/docs/scripts/auth-cli.ts | 41 +++++ apps/docs/scripts/docs-cli.ts | 32 +++- apps/docs/scripts/test-docs.mjs | 7 +- apps/docs/src/lib/docs.ts | 120 +++++++++++-- apps/docs/src/lib/document-keys.ts | 56 ++++++ apps/docs/src/lib/documents-api.ts | 51 ++++++ apps/docs/src/pages/api/documents/batch.ts | 22 +++ apps/docs/src/pages/api/documents/index.ts | 21 +++ .../src/pages/api/documents/item/[...slug].ts | 30 ++++ apps/docs/src/proxy.ts | 9 + apps/docs/tests/documents-api.test.ts | 136 ++++++++++++++ scripts/docs-admin.sh | 4 +- scripts/documents-client.py | 167 ++++++++++++++++++ scripts/test_documents_client.py | 85 +++++++++ 16 files changed, 826 insertions(+), 27 deletions(-) create mode 100644 apps/docs/src/lib/document-keys.ts create mode 100644 apps/docs/src/lib/documents-api.ts create mode 100644 apps/docs/src/pages/api/documents/batch.ts create mode 100644 apps/docs/src/pages/api/documents/index.ts create mode 100644 apps/docs/src/pages/api/documents/item/[...slug].ts create mode 100644 apps/docs/tests/documents-api.test.ts create mode 100644 scripts/documents-client.py create mode 100644 scripts/test_documents_client.py diff --git a/AGENTS.md b/AGENTS.md index c44c3b7..e6e5432 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ - 本仓库专门维护 Endor 文档与制图脚本。支持本地开发；CI/CD 仅在 main 使用现有 ces-control Runner，验证通过后自动部署；工作分支与 PR 不运行 CI，确认 Runner 不构成扩大触发范围的授权。不得修改其他项目、共享 Runner 配置或清理共享 Docker 缓存。 - Web 技术栈为 TypeScript、React、Tailwind CSS、Next.js Pages Router；不要切换到 App Router。 - 使用 pnpm 管理 JavaScript workspace，使用 uv 管理 `tools/figures` 的 Python 项目，保留两套锁文件。 -- 文档正文、目录元数据与修订记录以站点现有 SQLite 数据库为唯一权威源，使用 docs 管理工具直接更新；不得把新正文、目录 JSON、数据库文件或日常内容快照提交到仓库。历史 Git 文档只用于显式的一次性迁移，不作为运行时种子。Python 图形源码放在 `tools/figures/src/endor_figures`，可复现的 SVG 输出放在 `apps/docs/public/figures`。 +- 文档正文、目录元数据与修订记录以站点现有 SQLite 数据库为唯一权威源，使用文档 API 或 docs 管理工具直接更新；不得把新正文、目录 JSON、数据库文件或日常内容快照提交到仓库。历史 Git 文档只用于显式的一次性迁移，不作为运行时种子。Python 图形源码放在 `tools/figures/src/endor_figures`，可复现的 SVG 输出放在 `apps/docs/public/figures`。 - 未确认的架构提议明确标注为草案，不能表述为现有实现。 - 本地开发与预览只监听 `127.0.0.1`。部署容器监听 `0.0.0.0:3000`，服务器入口默认 18135，可通过独立 production.env 配置。 - 对 gitea.entroia.com 仅使用 codex-bot，提交身份为 Codex Bot 。SSH 使用隔离配置：`ssh -F /dev/null -o IdentitiesOnly=yes -i /Users/zzyang/.codex/credentials/gitea-entroia/codex_entroia_ed25519 -p 22222`。 diff --git a/README.md b/README.md index 242379d..7eb67af 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Endor 的文档站与 AI 公司研究工作台，包含文档阅读、全文搜 | 内容 | 权威来源与维护方式 | | --- | --- | -| 文档正文、分类、合集、分组、摘要、草案和推荐状态 | 数据库；通过服务器上的 `docs` 管理工具读写 | +| 文档正文、分类、合集、分组、摘要、草案和推荐状态 | 数据库；通过文档 API 或服务器上的 `docs` 管理工具读写 | | 文档旧地址与修订历史 | 数据库；改名保留别名，修改追加修订 | | AI 公司、观测、证据、候选与核查记录 | `content/research` 中的版本化快照；服务端显式按版本导入同库后查询 | | 页面、组件、管理工具和部署逻辑 | 本仓库，通过工作分支与 PR 评审 | @@ -32,7 +32,7 @@ Endor 的文档站与 AI 公司研究工作台，包含文档阅读、全文搜 ## 管理文档 -部署后，管理员可在服务器上使用 `/opt/endorphin-tabularium-docs/docs`。该工具在当前容器中操作数据库，需要服务器运维权限；站点没有公开的文档写入 API。 +部署后，管理员可在服务器上使用 `/opt/endorphin-tabularium-docs/docs`。该工具在当前容器中操作数据库，需要服务器运维权限。自动化上传使用下文的文档 API，日常读写无需登录服务器。 ```sh sudo /opt/endorphin-tabularium-docs/docs list @@ -63,6 +63,72 @@ sudo /opt/endorphin-tabularium-docs/docs save < "$doc_work_dir/document.json" 保存成功后，正文、目录和搜索通过后续请求读取新内容。发生版本冲突时重新读取并合并修改，不能仅改大 `revision` 覆盖并发编辑。调整分类或分组也走同一保存流程；`history` 使用稳定的 `sourceSlug` 查询修订。 +## 文档上传 API + +文档 API 直接读写同一个数据库，供自动化整理和大量文档上传使用。它只接受独立的文档 API Key，不接受网页 Cookie 或通用网页会话令牌；Key 不能用于研究、登录或其他站点 API。 + +| 方法与地址 | 用途 | +| --- | --- | +| `GET /api/documents?limit=50&offset=0` | 分页读取目录元数据；不返回全部正文 | +| `GET /api/documents/item/` | 读取单篇完整正文、目录字段和当前 revision；支持旧地址 | +| `GET /api/documents/item/?history=1&limit=50&offset=0` | 分页读取修订元数据与时间，按新到旧排列 | +| `GET /api/documents/item/?revision=1` | 读取指定修订的完整正文 | +| `POST /api/documents` | 保存一个完整文档 JSON；字段与 `docs save` 相同 | +| `POST /api/documents/batch` | 保存 `{ "documents": [...] }`，每个请求 1—100 篇 | + +写入使用 `Authorization: Bearer <文档 API Key>`。新建传 `revision: 0`；更新先读取当前全文，再使用读到的 revision。每篇正文最多 2,000,000 个 JavaScript 字符单位；每次请求最多 8 MiB。目录和修订分页的 `limit` 范围为 1—100，响应包含 `total` 和 `nextOffset`。 + +单个批量请求在同一事务提交，任意一篇格式错误、版本冲突或地址冲突都会整体回滚。成功返回文档地址、稳定标识和新修订号；版本或幂等键冲突返回 `409`，格式错误返回 `400`，超出请求大小返回 `413`。缺少、过期或撤销的 Key 返回 `401`，只读 Key 写入返回 `403`。 + +建议写入时提供 `Idempotency-Key`，由 1—128 个字母、数字或 `._:-` 组成。同一个 Key、相同请求 ID 和相同输入重试时，返回原回执并标记 `replayed: true`，不追加修订；相同 ID 改换输入会冲突。回执描述原请求，不代表之后没有其他修改；需要当前内容时重新读取文档。改过正文或 revision 后，应使用新的请求 ID。 + +### 创建、查看与撤销 Key + +管理员只需通过已验证的服务器连接进行 Key 管理： + +```sh +sudo /opt/endorphin-tabularium-docs/docs key-create codex-upload /auth/codex-docs-key.json read-write 90 +sudo /opt/endorphin-tabularium-docs/docs key-list +sudo /opt/endorphin-tabularium-docs/docs key-revoke +``` + +创建命令将完整 Key 写入认证私有目录中的指定新文件，权限 `0600`，终端仅输出 ID、权限和到期信息。文件已存在时拒绝覆盖。权限可选 `read` 或 `read-write`；有效期默认 90 天，可设 1—365 天。数据库仅保存 Key 摘要，列表不返回原文；撤销后不能继续认证。独立 Key 文件和认证 config.json 都不包含在 SQLite 整库备份中。 + +### 使用上传客户端 + +`scripts/documents-client.py` 仅使用 Python 标准库。通过 `TABULARIUM_DOCS_URL` 指定站点 origin，通过 `TABULARIUM_DOCS_KEY_FILE` 指定本机私有 Key 文件（支持创建命令输出的 JSON），不把 Key 放入命令行参数。 + +客户端要求 HTTPS；尚未配置 HTTPS 时，可通过已验证的 SSH 连接建立本机端口转发，连接 `http://127.0.0.1:18136`。以下 `$DOCS_SSH_TARGET` 应事先设置为受信任的 SSH 别名或 `user@host`： + +```sh +ssh -N -L 18136:127.0.0.1:18135 "$DOCS_SSH_TARGET" +``` + +将服务器私有目录中的 Key 文件复制到受保护的本机位置后，在另一个终端配置客户端。`$TABULARIUM_DOCS_KEY_FILE` 文件须仅本人可读写，不能提交到仓库： + +```sh +export TABULARIUM_DOCS_URL=http://127.0.0.1:18136 +export TABULARIUM_DOCS_KEY_FILE="$HOME/.config/tabularium/codex-docs-key.json" +python3 scripts/documents-client.py list --limit 50 +python3 scripts/documents-client.py get endor/endor-purpose-draft +python3 scripts/documents-client.py get endor/endor-purpose-draft --history +python3 scripts/documents-client.py get endor/endor-purpose-draft --revision 1 +``` + +修改流程与 CLI 相同： + +```sh +umask 077 +api_work_dir=$(mktemp -d) +python3 scripts/documents-client.py get endor/endor-purpose-draft > "$api_work_dir/document.json" +# 编辑正文或目录字段，保留 sourceSlug 和读取到的 revision。 +python3 scripts/documents-client.py save "$api_work_dir/document.json" +``` + +大量文档可使用仅含 `documents` 数组的私有 JSON 文件。客户端按 100 篇和 8 MiB 请求上限自动分批，并为各批派生固定的幂等请求 ID。每批独立提交；后续批次失败时，前面的成功批次保留。客户端会打印根请求 ID 和各批进度，使用相同 ID 与未修改的输入重试可跳过已完成的批次；可通过 `--request-id` 显式指定根 ID（客户端最多 100 个字符）。若后续批次因业务冲突失败，按进度中已提交的文档数量提取尚未成功的部分，修正后使用新的根 ID 上传。 + +客户端只自动重试一次临时网络或 `502/503/504` 错误，保持正文和请求 ID 不变；`409` 等业务错误不会自动覆盖。它不跟随 HTTP 重定向，避免将 Bearer 凭据转发到其他地址。正文和 Key 继续保存在数据库或私有文件中，不因使用 API 而进入 Git。 + ## 研究数据与检查 研究工作台提供公司筛选、经营指标、原始证据、收费与交付模式、技术材料及核查记录。它区分公司披露、支付平台指标、第三方估算、融资和客户收益；未知收入不填零，材料可访问也不代表性能或商业效果已经验证。研究范围、样本限制和个案结论以工作台记录为准。 diff --git a/apps/docs/scripts/auth-cli.ts b/apps/docs/scripts/auth-cli.ts index 9c7e393..7d3f60c 100644 --- a/apps/docs/scripts/auth-cli.ts +++ b/apps/docs/scripts/auth-cli.ts @@ -18,6 +18,7 @@ import { totp, isLoginRedirect, } from "../src/lib/auth"; +import { createDocumentKey, revokeDocumentKey } from "../src/lib/document-keys"; import { readDocument, saveDocument } from "../src/lib/docs"; import { consumeCode, initializeStore } from "../src/lib/auth-store"; @@ -210,6 +211,7 @@ async function main() { assert.equal(raw.status, 200); assert.ok((await raw.text()).startsWith("# Endor")); if (process.env.DOCUMENT_SMOKE_TEST) { + if (command === "exercise") await exerciseDocumentApi(request, headers); const sourceSlug = "ci-live-document"; const previous = readDocument(sourceSlug); if (command === "exercise") { @@ -356,3 +358,42 @@ main().catch((error) => { ); process.exitCode = 1; }); + +async function exerciseDocumentApi(request: (path: string, init?: RequestInit) => Promise, session: Record) { + const writer = createDocumentKey("isolated API smoke", "read-write", 1); + const reader = createDocumentKey("isolated read smoke", "read", 1); + const headers = { Authorization: "Bearer " + writer.token, "Content-Type": "application/json", "Idempotency-Key": "smoke-batch" }; + const readHeaders = { Authorization: "Bearer " + reader.token, "Content-Type": "application/json" }; + const make = (name: string) => ({ slug: `endor/${name}`, sourceSlug: name, title: "API smoke", + category: "Endor", collection: "Endor 文档", group: "CI", description: "Isolated API smoke", + draft: true, featured: false, content: "# API smoke", revision: 0 }); + const payload = { documents: [make("ci-api-first"), make("ci-api-second")] }; + try { + assert.equal((await request("/api/documents")).status, 401); + assert.equal((await request("/api/documents", { method: "POST", headers: { ...session, "Content-Type": "application/json" }, body: JSON.stringify(payload.documents[0]) })).status, 401); + assert.equal((await request("/api/research/ai-companies", { headers })).status, 401, "Document keys cannot read research"); + assert.equal((await request("/api/search", { headers })).status, 401, "Document keys cannot authenticate other APIs"); + assert.equal((await request("/api/documents/batch", { method: "POST", headers: readHeaders, body: JSON.stringify(payload) })).status, 403); + const saved = await request("/api/documents/batch", { method: "POST", headers, body: JSON.stringify(payload) }); + assert.equal(saved.status, 200); + assert.equal((await saved.json()).documents.length, 2); + const replay = await request("/api/documents/batch", { method: "POST", headers, body: JSON.stringify(payload) }); + assert.equal(replay.status, 200); assert.equal((await replay.json()).replayed, true); + const current = await request("/api/documents/item/endor/ci-api-first", { headers: readHeaders }); + assert.equal(current.status, 200); assert.equal((await current.json()).revision, 1); + const page = await request("/api/documents?limit=1", { headers: readHeaders }); + assert.equal(page.status, 200); + const listing = await page.json(); + assert.equal(listing.documents.length, 1); assert.equal(listing.documents[0].content, undefined); + const bad = { documents: [{ ...payload.documents[0], revision: 1, content: "Must roll back" }, payload.documents[1]] }; + const conflict = await request("/api/documents/batch", { method: "POST", headers: { ...headers, "Idempotency-Key": "smoke-conflict" }, body: JSON.stringify(bad) }); + assert.equal(conflict.status, 409); + const unchanged = await request("/api/documents/item/endor/ci-api-first", { headers: readHeaders }); + assert.equal((await unchanged.json()).content, "# API smoke"); + const history = await request("/api/documents/item/endor/ci-api-first?history=1", { headers }); + assert.equal(history.status, 200); assert.equal((await history.json()).total, 1); + revokeDocumentKey(writer.id); + assert.equal((await request("/api/documents", { headers })).status, 401); + } finally { revokeDocumentKey(writer.id); revokeDocumentKey(reader.id); } + console.log("Document API smoke passed: scoped keys, batch rollback, pagination, history, replay and revocation; no secrets displayed"); +} diff --git a/apps/docs/scripts/docs-cli.ts b/apps/docs/scripts/docs-cli.ts index a3f6f50..22a7b8e 100644 --- a/apps/docs/scripts/docs-cli.ts +++ b/apps/docs/scripts/docs-cli.ts @@ -1,11 +1,37 @@ -import { readFileSync, writeFileSync } from "node:fs"; +import { readFileSync, writeFileSync, openSync, closeSync, unlinkSync } from "node:fs"; +import { resolve, dirname, sep } from "node:path"; +import { createDocumentKey, listDocumentKeys, revokeDocumentKey, type DocumentKeyScope } from "../src/lib/document-keys"; import { openSiteDatabase } from "../src/lib/site-database"; import { allDocuments, readDocument, saveDocument, importDocuments, documentHistory } from "../src/lib/docs"; process.umask(0o077); -const [command, argument] = process.argv.slice(2); +const [command, argument, ...extra] = process.argv.slice(2); try { switch (command) { + case "key-create": { + const authFile = process.env.AUTH_CONFIG_FILE; + if (!authFile || !argument || !extra[0]) throw Error("Usage: key-create [read|read-write] [days]"); + const file = resolve(extra[0]); + const directory = resolve(dirname(authFile)); + if (!file.startsWith(directory + sep)) throw Error("Key output must be inside the private authentication directory"); + const fd = openSync(file, "wx", 0o600); + let created: { id: string } | undefined; + try { + const key = createDocumentKey(argument, (extra[1] ?? "read-write") as DocumentKeyScope, Number(extra[2] ?? 90)); + created = key; + writeFileSync(fd, JSON.stringify(key) + "\n"); + console.log(JSON.stringify({ id: key.id, scope: key.scope, expiresAt: key.expiresAt, saved: file })); + } catch (error) { + if (created) revokeDocumentKey(created.id); + unlinkSync(file); + throw error; + } finally { closeSync(fd); } + break; + } + case "key-list": console.log(JSON.stringify(listDocumentKeys(), null, 2)); break; + case "key-revoke": + if (!argument) throw Error("Provide key id"); + revokeDocumentKey(argument); console.log("Document key revoked"); break; case "backup": { if (!argument) throw Error("Provide a private backup filename"); const db = openSiteDatabase(); @@ -36,7 +62,7 @@ try { { flag: "wx", mode: 0o600 }); console.log("Document export created; existing files are never overwritten"); break; } - default: throw Error("Usage: docs backup | list | get | save < input.json | history | import [--if-empty] < export.json | export "); + default: throw Error("Usage: docs key-create [read|read-write] [days] | key-list | key-revoke | backup | list | get | save < input.json | history | import [--if-empty] < export.json | export "); } } catch (error) { console.error(error instanceof Error ? error.message : "Document operation failed"); diff --git a/apps/docs/scripts/test-docs.mjs b/apps/docs/scripts/test-docs.mjs index 339d95b..c4f4531 100644 --- a/apps/docs/scripts/test-docs.mjs +++ b/apps/docs/scripts/test-docs.mjs @@ -6,9 +6,8 @@ import { spawnSync } from "node:child_process"; const dir = await mkdtemp(join(tmpdir(), "tabularium-docs-tests-")); try { - const file = join(dir, "docs.test.cjs"); - await build({ entryPoints: ["tests/docs.test.ts"], bundle: true, - platform: "node", target: "node22", format: "cjs", outfile: file }); - const result = spawnSync(process.execPath, ["--test", file], { stdio: "inherit" }); + await build({ entryPoints: ["tests/docs.test.ts", "tests/documents-api.test.ts"], bundle: true, + platform: "node", target: "node22", format: "cjs", outdir: dir, outExtension: { ".js": ".cjs" } }); + const result = spawnSync(process.execPath, ["--test", join(dir, "docs.test.cjs"), join(dir, "documents-api.test.cjs")], { stdio: "inherit" }); process.exitCode = result.status ?? 1; } finally { await rm(dir, { recursive: true, force: true }); } diff --git a/apps/docs/src/lib/docs.ts b/apps/docs/src/lib/docs.ts index 7f14bcb..5e16128 100644 --- a/apps/docs/src/lib/docs.ts +++ b/apps/docs/src/lib/docs.ts @@ -9,6 +9,9 @@ export type DocLink = { export type Doc = DocLink & { content: string }; export type DocInput = Omit; export type DocumentExport = { format: "tabularium-documents-v1"; documents: Doc[] }; +export class DocumentValidationError extends Error {} +export class DocumentConflictError extends Error {} +export type DocumentReceipt = Pick; export const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/; function database() { @@ -25,6 +28,10 @@ function database() { document_id TEXT NOT NULL REFERENCES documents(id), revision INTEGER NOT NULL, payload TEXT NOT NULL, created_at TEXT NOT NULL, PRIMARY KEY(document_id, revision) ); + CREATE TABLE IF NOT EXISTS document_write_requests ( + namespace TEXT NOT NULL, request_id TEXT NOT NULL, digest TEXT NOT NULL, + response TEXT NOT NULL, PRIMARY KEY(namespace, request_id) + ); CREATE TABLE IF NOT EXISTS document_imports ( digest TEXT PRIMARY KEY, document_count INTEGER NOT NULL, created_at TEXT NOT NULL ); @@ -33,50 +40,51 @@ function database() { } export function validateDocument(input: DocInput): void { - if (!input || typeof input !== "object") throw Error("Invalid document"); + if (!input || typeof input !== "object") throw new DocumentValidationError("Invalid document"); for (const key of ["slug", "sourceSlug"] as const) - if (typeof input[key] !== "string" || !validSlug.test(input[key])) throw Error(`Invalid ${key}`); - if (input.slug.split("/").some((part) => /^\d+-/.test(part))) throw Error("Use a semantic public slug"); + if (typeof input[key] !== "string" || !validSlug.test(input[key])) throw new DocumentValidationError(`Invalid ${key}`); + if (input.slug.split("/").some((part) => /^\d+-/.test(part))) throw new DocumentValidationError("Use a semantic public slug"); for (const key of ["title", "category", "collection", "group"] as const) - if (typeof input[key] !== "string" || !input[key].trim()) throw Error(`Missing ${key}`); + if (typeof input[key] !== "string" || !input[key].trim()) throw new DocumentValidationError(`Missing ${key}`); if (typeof input.content !== "string" || typeof input.description !== "string" || - typeof input.draft !== "boolean" || typeof input.featured !== "boolean") throw Error("Invalid document fields"); - if (input.content.length > 2_000_000) throw Error("Document exceeds content limit"); + typeof input.draft !== "boolean" || typeof input.featured !== "boolean") throw new DocumentValidationError("Invalid document fields"); + if (input.content.length > 2_000_000) throw new DocumentValidationError("Document exceeds content limit"); const images = [ ...input.content.matchAll(/!\[[^\]]*\]\(]+)/g), ...input.content.matchAll(/]*\bsrc=["']([^"']+)/gi), ]; if (images.some(([, source]) => /\.(?:png|jpe?g|tiff?|bmp)(?:[?#]|$)/i.test(source))) - throw Error("Use compressed WebP images instead of original rasters"); + throw new DocumentValidationError("Use compressed WebP images instead of original rasters"); } function write(db: ReturnType, input: DocInput, expectedRevision: number): Doc { validateDocument(input); const existing = db.prepare("SELECT id, revision FROM documents WHERE source_slug=?") .get(input.sourceSlug) as { id: string; revision: number } | undefined; - if ((existing?.revision ?? 0) !== expectedRevision) throw Error("Document revision conflict"); + if ((existing?.revision ?? 0) !== expectedRevision) throw new DocumentConflictError("Document revision conflict"); const id = existing?.id ?? randomUUID(); const doc: Doc = { slug: input.slug, sourceSlug: input.sourceSlug, category: input.category, title: input.title, group: input.group, collection: input.collection, description: input.description, draft: input.draft, featured: input.featured, content: input.content, revision: expectedRevision + 1 }; const payload = JSON.stringify(doc); - // Aliases remain reserved after renaming; collision rolls back the whole write. - db.prepare(`INSERT INTO documents VALUES (?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET - slug=excluded.slug, revision=excluded.revision, payload=excluded.payload`) - .run(id, doc.slug, doc.sourceSlug, doc.revision, payload); + // Reserve both canonical and historical names before updating any rows. for (const alias of new Set([doc.slug, doc.sourceSlug])) { const prior = db.prepare("SELECT document_id FROM document_aliases WHERE alias=?") .get(alias) as { document_id: string } | undefined; - if (prior && prior.document_id !== id) throw Error(`Document alias collision: ${alias}`); - db.prepare("INSERT OR IGNORE INTO document_aliases VALUES (?,?)").run(alias, id); + if (prior && prior.document_id !== id) throw new DocumentConflictError(`Document alias collision: ${alias}`); } + db.prepare(`INSERT INTO documents VALUES (?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET + slug=excluded.slug, revision=excluded.revision, payload=excluded.payload`) + .run(id, doc.slug, doc.sourceSlug, doc.revision, payload); + for (const alias of new Set([doc.slug, doc.sourceSlug])) + db.prepare("INSERT OR IGNORE INTO document_aliases VALUES (?,?)").run(alias, id); db.prepare("INSERT INTO document_revisions VALUES (?,?,?,?)") .run(id, doc.revision, payload, new Date().toISOString()); return doc; } export function saveDocument(input: DocInput, expectedRevision: number): Doc { - if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) throw Error("Provide expected revision"); + if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) throw new DocumentValidationError("Provide expected revision"); const db = database(); try { db.exec("BEGIN IMMEDIATE"); @@ -145,3 +153,85 @@ export function documentHistory(sourceSlug: string): Doc[] { .map((row) => JSON.parse(row.payload) as Doc); } finally { db.close(); } } + +// API batches commit entirely or not at all; receipts make lost-response retries safe. +export function saveDocumentBatch(documents: Doc[], request?: { namespace: string; id: string }) { + if (!Array.isArray(documents) || !documents.length || documents.length > 100) + throw new DocumentValidationError("Provide 1 to 100 documents"); + const seen = new Set(); + for (const doc of documents) { + validateDocument(doc); + if (!Number.isSafeInteger(doc.revision) || doc.revision < 0) + throw new DocumentValidationError("Provide expected revision"); + if (seen.has(doc.sourceSlug)) throw new DocumentValidationError("Duplicate sourceSlug in batch"); + seen.add(doc.sourceSlug); + } + if (request && (!request.namespace || !/^[A-Za-z0-9._:-]{1,128}$/.test(request.id))) + throw new DocumentValidationError("Invalid Idempotency-Key"); + const digest = createHash("sha256").update(JSON.stringify(documents)).digest("hex"); + const db = database(); + try { + db.exec("BEGIN IMMEDIATE"); + try { + if (request) { + const prior = db.prepare("SELECT digest,response FROM document_write_requests WHERE namespace=? AND request_id=?") + .get(request.namespace, request.id) as { digest: string; response: string } | undefined; + if (prior) { + if (prior.digest !== digest) throw new DocumentConflictError("Idempotency-Key was used with different content"); + db.exec("COMMIT"); + return { documents: JSON.parse(prior.response) as DocumentReceipt[], replayed: true }; + } + } + const receipts = documents.map((input) => { + const { slug, sourceSlug, revision } = write(db, input, input.revision); + return { slug, sourceSlug, revision }; + }); + if (request) db.prepare("INSERT INTO document_write_requests VALUES (?,?,?,?)") + .run(request.namespace, request.id, digest, JSON.stringify(receipts)); + db.exec("COMMIT"); + return { documents: receipts, replayed: false }; + } catch (error) { db.exec("ROLLBACK"); throw error; } + } finally { db.close(); } +} + +export function documentPage(limit = 50, offset = 0) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(offset) || offset < 0) + throw new DocumentValidationError("Invalid pagination"); + const db = database(); + try { + db.exec("BEGIN"); + const total = (db.prepare("SELECT COUNT(*) AS n FROM documents").get() as { n: number }).n; + const documents = (db.prepare("SELECT json_remove(payload,'$.content') AS payload FROM documents ORDER BY source_slug LIMIT ? OFFSET ?") + .all(limit, offset) as { payload: string }[]).map((row) => JSON.parse(row.payload) as DocLink); + db.exec("COMMIT"); + return { documents, total, limit, offset, nextOffset: offset + documents.length < total ? offset + documents.length : null }; + } finally { db.close(); } +} + +export function documentHistoryPage(sourceSlug: string, limit = 50, offset = 0) { + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 100 || !Number.isSafeInteger(offset) || offset < 0) + throw new DocumentValidationError("Invalid pagination"); + const db = database(); + try { + db.exec("BEGIN"); + const total = (db.prepare(`SELECT COUNT(*) AS n FROM document_revisions r JOIN documents d + ON d.id=r.document_id WHERE d.source_slug=?`).get(sourceSlug) as { n: number }).n; + const revisions = (db.prepare(`SELECT json_remove(r.payload,'$.content') AS payload, r.created_at AS createdAt + FROM document_revisions r JOIN documents d ON d.id=r.document_id + WHERE d.source_slug=? ORDER BY r.revision DESC LIMIT ? OFFSET ?`) + .all(sourceSlug, limit, offset) as { payload: string; createdAt: string }[]) + .map((row) => ({ ...JSON.parse(row.payload) as DocLink, createdAt: row.createdAt })); + db.exec("COMMIT"); + return { revisions, total, limit, offset, nextOffset: offset + revisions.length < total ? offset + revisions.length : null }; + } finally { db.close(); } +} + +export function documentRevision(sourceSlug: string, revision: number): Doc | null { + if (!Number.isSafeInteger(revision) || revision < 1) throw new DocumentValidationError("Invalid revision"); + const db = database(); + try { + const row = db.prepare(`SELECT r.payload FROM document_revisions r JOIN documents d + ON d.id=r.document_id WHERE d.source_slug=? AND r.revision=?`).get(sourceSlug, revision) as { payload: string } | undefined; + return row ? JSON.parse(row.payload) as Doc : null; + } finally { db.close(); } +} diff --git a/apps/docs/src/lib/document-keys.ts b/apps/docs/src/lib/document-keys.ts new file mode 100644 index 0000000..5f8b16a --- /dev/null +++ b/apps/docs/src/lib/document-keys.ts @@ -0,0 +1,56 @@ +import { createHash, randomBytes, randomUUID } from "node:crypto"; +import { openSiteDatabase } from "./site-database"; + +export type DocumentKeyScope = "read" | "read-write"; +export type DocumentKey = { id: string; name: string; scope: DocumentKeyScope; + createdAt: number; expiresAt: number; revokedAt: number | null; lastUsedAt: number | null }; +function database() { + const db = openSiteDatabase(); + db.exec(`CREATE TABLE IF NOT EXISTS document_api_keys ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, digest TEXT NOT NULL UNIQUE, + scope TEXT NOT NULL CHECK(scope IN ('read','read-write')), created_at INTEGER NOT NULL, + expires_at INTEGER NOT NULL, revoked_at INTEGER, last_used_at INTEGER + );`); + return db; +} +export function createDocumentKey(name: string, scope: DocumentKeyScope = "read-write", days = 90) { + if (!name.trim() || name.length > 100 || !["read", "read-write"].includes(scope) || + !Number.isSafeInteger(days) || days < 1 || days > 365) throw Error("Invalid key name, scope or lifetime"); + const id = randomUUID(); + const token = "tdoc_" + randomBytes(32).toString("base64url"); + const createdAt = Date.now(); + const expiresAt = createdAt + days * 86400000; + const db = database(); + try { + db.prepare("INSERT INTO document_api_keys(id,name,digest,scope,created_at,expires_at) VALUES (?,?,?,?,?,?)") + .run(id, name.trim(), createHash("sha256").update(token).digest("hex"), scope, createdAt, expiresAt); + return { id, token, name: name.trim(), scope, createdAt, expiresAt }; + } finally { db.close(); } +} +export function authenticateDocumentKey(token: string | undefined, now = Date.now()) { + if (!token || !/^tdoc_[A-Za-z0-9_-]{43}$/.test(token)) return null; + const db = database(); + try { + const row = db.prepare("SELECT id,scope,last_used_at AS lastUsedAt FROM document_api_keys WHERE digest=? AND revoked_at IS NULL AND expires_at>?") + .get(createHash("sha256").update(token).digest("hex"), now) as { id: string; scope: DocumentKeyScope; lastUsedAt: number | null } | undefined; + if (!row) return null; + if (row.lastUsedAt === null || row.lastUsedAt < now - 60000) + db.prepare("UPDATE document_api_keys SET last_used_at=? WHERE id=? AND (last_used_at IS NULL OR last_used_at; + const fields = ["slug", "sourceSlug", "title", "category", "collection", "group", "description", "draft", "featured", "content", "revision"]; + if (Object.keys(row).some((key) => !fields.includes(key))) throw new DocumentValidationError("Unknown document field"); + const doc = Object.fromEntries(fields.map((key) => [key, row[key]])) as Doc; + validateDocument(doc); + if (!Number.isSafeInteger(doc.revision) || doc.revision < 0) throw new DocumentValidationError("Provide expected revision"); + return doc; +} +export function pagination(request: NextApiRequest) { + const number = (value: string | string[] | undefined, fallback: number) => { + if (value === undefined) return fallback; + if (typeof value !== "string" || !/^\d+$/.test(value)) throw new DocumentValidationError("Invalid pagination"); + return Number(value); + }; + return { limit: number(request.query.limit, 50), offset: number(request.query.offset, 0) }; +} +export function idempotency(request: NextApiRequest, namespace: string) { + const id = request.headers["idempotency-key"]; + if (id === undefined) return undefined; + if (typeof id !== "string" || !/^[A-Za-z0-9._:-]{1,128}$/.test(id)) throw new DocumentValidationError("Invalid Idempotency-Key"); + return { namespace, id }; +} diff --git a/apps/docs/src/pages/api/documents/batch.ts b/apps/docs/src/pages/api/documents/batch.ts new file mode 100644 index 0000000..2f90622 --- /dev/null +++ b/apps/docs/src/pages/api/documents/batch.ts @@ -0,0 +1,22 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { DocumentValidationError, saveDocumentBatch } from "@/lib/docs"; +import { authorizeDocuments, documentApiError, idempotency, parseDocument } from "@/lib/documents-api"; + +export const config = { api: { bodyParser: { sizeLimit: "8mb" } } }; +export default function handler(request: NextApiRequest, response: NextApiResponse) { + try { + const key = authorizeDocuments(request, response, true); + if (!key) return; + if (request.method !== "POST") { + response.setHeader("Allow", "POST"); + return response.status(405).end(); + } + const body = request.body; + if (!body || typeof body !== "object" || Array.isArray(body) || + Object.keys(body).length !== 1 || !Array.isArray(body.documents) || + !body.documents.length || body.documents.length > 100) + throw new DocumentValidationError("Expected { documents: [...] } with 1 to 100 documents"); + const documents = body.documents.map((row: unknown) => parseDocument(row)); + return response.status(200).json(saveDocumentBatch(documents, idempotency(request, key.id))); + } catch (error) { return documentApiError(response, error); } +} diff --git a/apps/docs/src/pages/api/documents/index.ts b/apps/docs/src/pages/api/documents/index.ts new file mode 100644 index 0000000..4679e3e --- /dev/null +++ b/apps/docs/src/pages/api/documents/index.ts @@ -0,0 +1,21 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { documentPage, saveDocumentBatch } from "@/lib/docs"; +import { authorizeDocuments, documentApiError, idempotency, pagination, parseDocument } from "@/lib/documents-api"; + +export const config = { api: { bodyParser: { sizeLimit: "8mb" } } }; +export default function handler(request: NextApiRequest, response: NextApiResponse) { + try { + const key = authorizeDocuments(request, response, request.method === "POST"); + if (!key) return; + if (request.method === "GET") { + const { limit, offset } = pagination(request); + return response.status(200).json(documentPage(limit, offset)); + } + if (request.method === "POST") { + const result = saveDocumentBatch([parseDocument(request.body)], idempotency(request, key.id)); + return response.status(200).json({ document: result.documents[0], replayed: result.replayed }); + } + response.setHeader("Allow", "GET, POST"); + return response.status(405).end(); + } catch (error) { return documentApiError(response, error); } +} diff --git a/apps/docs/src/pages/api/documents/item/[...slug].ts b/apps/docs/src/pages/api/documents/item/[...slug].ts new file mode 100644 index 0000000..fa04bca --- /dev/null +++ b/apps/docs/src/pages/api/documents/item/[...slug].ts @@ -0,0 +1,30 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { DocumentValidationError, documentHistoryPage, documentRevision, readDocument } from "@/lib/docs"; +import { authorizeDocuments, documentApiError, pagination } from "@/lib/documents-api"; + +export default function handler(request: NextApiRequest, response: NextApiResponse) { + try { + if (!authorizeDocuments(request, response)) return; + if (request.method !== "GET") { + response.setHeader("Allow", "GET"); + return response.status(405).end(); + } + const slug = request.query.slug; + if (!Array.isArray(slug)) throw new DocumentValidationError("Invalid document slug"); + const doc = readDocument(slug.join("/")); + if (!doc) return response.status(404).json({ error: "文档不存在", code: "not_found" }); + if (request.query.revision !== undefined) { + if (request.query.history !== undefined || typeof request.query.revision !== "string" || !/^\d+$/.test(request.query.revision)) + throw new DocumentValidationError("Invalid revision query"); + const prior = documentRevision(doc.sourceSlug, Number(request.query.revision)); + if (!prior) return response.status(404).json({ error: "修订不存在", code: "not_found" }); + return response.status(200).json(prior); + } + if (request.query.history !== undefined) { + if (request.query.history !== "1") throw new DocumentValidationError("Use history=1"); + const { limit, offset } = pagination(request); + return response.status(200).json(documentHistoryPage(doc.sourceSlug, limit, offset)); + } + return response.status(200).json(doc); + } catch (error) { return documentApiError(response, error); } +} diff --git a/apps/docs/src/proxy.ts b/apps/docs/src/proxy.ts index 2f4980d..77cc4e0 100644 --- a/apps/docs/src/proxy.ts +++ b/apps/docs/src/proxy.ts @@ -1,5 +1,6 @@ import { NextResponse, type NextRequest } from "next/server"; import { authConfig, SESSION_COOKIE, validSession } from "./lib/auth"; +import { authenticateDocumentKey } from "./lib/document-keys"; export function proxy(request: NextRequest) { const path = request.nextUrl.pathname; const headers = { @@ -31,6 +32,14 @@ export function proxy(request: NextRequest) { ? authorization.slice(7) : undefined; const token = bearer ?? request.cookies.get(SESSION_COOKIE)?.value; + // Document keys are scoped to the document API; they never authenticate other pages or APIs. + if (bearer && (path === "/api/documents" || path.startsWith("/api/documents/"))) { + try { + if (authenticateDocumentKey(bearer)) return NextResponse.next({ headers }); + } catch { + return NextResponse.json({ error: "文档认证暂时不可用" }, { status: 503, headers }); + } + } if (validSession(token, config)) { if (!["GET", "HEAD", "OPTIONS"].includes(request.method) && !bearer) { try { diff --git a/apps/docs/tests/documents-api.test.ts b/apps/docs/tests/documents-api.test.ts new file mode 100644 index 0000000..1c28528 --- /dev/null +++ b/apps/docs/tests/documents-api.test.ts @@ -0,0 +1,136 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { mkdtempSync, rmSync, writeFileSync, readFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { NextApiRequest, NextApiResponse } from "next"; +import documents from "../src/pages/api/documents/index"; +import batch from "../src/pages/api/documents/batch"; +import item from "../src/pages/api/documents/item/[...slug]"; +import { createDocumentKey, authenticateDocumentKey, revokeDocumentKey, listDocumentKeys } from "../src/lib/document-keys"; +import { allDocuments, documentHistory, readDocument, saveDocument, type Doc } from "../src/lib/docs"; +import { newAuthConfig, signSession } from "../src/lib/auth"; + +const input: Doc = { slug: "endor/example", sourceSlug: "00-example", title: "Example", category: "Endor", + collection: "Endor 文档", group: "讨论", description: "Description", draft: true, featured: false, + content: "# Example\n\nOriginal body", revision: 0 }; +type Options = { method?: string; body?: unknown; headers?: Record; + query?: Record }; +function call(handler: (req: NextApiRequest, res: NextApiResponse) => unknown, options: Options = {}) { + let status = 200; + let body: unknown; + const headers: Record = {}; + const response = { + status(value: number) { status = value; return this; }, + setHeader(name: string, value: string) { headers[name.toLowerCase()] = value; return this; }, + json(value: unknown) { body = value; return this; }, + end() { return this; }, + }; + handler({ method: options.method ?? "GET", body: options.body, headers: options.headers ?? {}, + query: options.query ?? {} } as unknown as NextApiRequest, response as unknown as NextApiResponse); + return { status, body, headers }; +} +function isolated(run: (context: { directory: string; headers: Record }) => void) { + const directory = mkdtempSync(join(tmpdir(), "tabularium-doc-api-")); + const previous = process.env.AUTH_CONFIG_FILE; + process.env.AUTH_CONFIG_FILE = join(directory, "config.json"); + writeFileSync(process.env.AUTH_CONFIG_FILE, JSON.stringify(newAuthConfig()), { mode: 0o600 }); + try { + const key = createDocumentKey("test uploader"); + run({ directory, headers: { authorization: "Bearer " + key.token } }); + } finally { + if (previous === undefined) delete process.env.AUTH_CONFIG_FILE; + else process.env.AUTH_CONFIG_FILE = previous; + rmSync(directory, { recursive: true, force: true }); + } +} + +test("new API requires document keys, rejects browser cookies and broad web sessions", () => isolated(() => { + assert.equal(call(documents).status, 401); + assert.equal(call(documents, { headers: { authorization: "Bearer invalid" } }).status, 401); + const config = JSON.parse(readFileSync(process.env.AUTH_CONFIG_FILE!, "utf8")); + const token = signSession(config); + assert.equal(call(documents, { headers: { authorization: "Bearer " + token } }).status, 401); + assert.equal(call(documents, { headers: { cookie: "tabularium_session=" + token } }).status, 401); + assert.deepEqual(allDocuments(), []); +})); + +test("key scopes, revocation, expiry and secret-free listing are persistent", () => isolated(({ directory }) => { + const key = createDocumentKey("reader", "read", 1); + const headers = { authorization: "Bearer " + key.token }; + assert.equal(call(documents, { headers }).status, 200); + assert.equal(call(documents, { method: "POST", headers, body: input }).status, 403); + assert.equal(call(batch, { method: "POST", headers, body: { documents: [input] } }).status, 403); + assert.ok(!JSON.stringify(listDocumentKeys()).includes(key.token)); + assert.ok(!readFileSync(join(directory, "login-state.sqlite")).includes(Buffer.from(key.token))); + assert.equal(authenticateDocumentKey(key.token, key.expiresAt), null); + revokeDocumentKey(key.id); + assert.equal(call(documents, { headers }).status, 401); + const revoked = listDocumentKeys().find((row) => row.id === key.id); + assert.ok(revoked); assert.equal(typeof revoked.revokedAt, "number"); +})); + +test("single save, aliases, metadata and historical content are visible without rebuilding", () => isolated(({ headers }) => { + assert.equal(call(documents, { method: "POST", headers, body: input }).status, 200); + assert.equal((call(item, { headers, query: { slug: input.slug.split("/") } }).body as Doc).revision, 1); + const changed = { ...input, revision: 1, slug: "endor/renamed", content: "# Changed", collection: "新合集" }; + assert.equal(call(documents, { method: "POST", headers, body: changed }).status, 200); + assert.equal((call(item, { headers, query: { slug: input.slug.split("/") } }).body as Doc).slug, changed.slug); + assert.equal((call(item, { headers, query: { slug: [input.sourceSlug], revision: "1" } }).body as Doc).content, input.content); + const history = call(item, { headers, query: { slug: [input.sourceSlug], history: "1", limit: "1" } }); + const page = history.body as { revisions: Doc[]; total: number; nextOffset: number }; + assert.equal(page.total, 2); + assert.equal(page.nextOffset, 1); + assert.equal(page.revisions[0].revision, 2); + assert.equal(page.revisions[0].content, undefined); + assert.equal(call(item, { headers, query: { slug: [input.sourceSlug], revision: "99" } }).status, 404); + assert.equal(call(item, { headers, query: { slug: [input.sourceSlug], revision: "0" } }).status, 400); +})); + +test("pagination returns metadata only with strict bounds", () => isolated(({ headers }) => { + const second = { ...input, slug: "endor/second", sourceSlug: "01-second" }; + assert.equal(call(batch, { method: "POST", headers, body: { documents: [input, second] } }).status, 200); + const result = call(documents, { headers, query: { limit: "1" } }); + const page = result.body as { documents: Doc[]; total: number; nextOffset: number }; + assert.equal(page.total, 2); assert.equal(page.nextOffset, 1); + assert.equal(page.documents[0].content, undefined); + assert.equal((call(documents, { headers, query: { limit: "1", offset: "1" } }).body as typeof page).documents[0].slug, second.slug); + const invalidQueries: Record[] = [ + { limit: "101" }, { offset: "-1" }, { limit: ["1", "2"] }, { limit: "1e2" }, + ]; + for (const query of invalidQueries) assert.equal(call(documents, { headers, query }).status, 400); + assert.equal(result.headers["cache-control"], "private, no-store"); +})); + +test("late batch conflict rolls back earlier rows and their revision history", () => isolated(({ headers }) => { + saveDocument({ ...input, slug: "endor/existing", sourceSlug: "existing" }, 0); + const conflict = { ...input, slug: "endor/existing", sourceSlug: "existing", revision: 0 }; + assert.equal(call(batch, { method: "POST", headers, body: { documents: [input, conflict] } }).status, 409); + assert.equal(readDocument(input.slug), null); + assert.equal(documentHistory(input.sourceSlug).length, 0); + assert.equal(documentHistory("existing").length, 1); + assert.equal(allDocuments().length, 1); +})); + +test("idempotent retries return the original receipt without creating another revision", () => isolated(({ headers }) => { + const keyed = { ...headers, "idempotency-key": "upload-1" }; + const first = call(documents, { method: "POST", headers: keyed, body: input }); + const second = call(documents, { method: "POST", headers: keyed, body: input }); + assert.equal(first.status, 200); assert.equal(second.status, 200); + assert.equal((second.body as { replayed: boolean }).replayed, true); + assert.equal(documentHistory(input.sourceSlug).length, 1); + assert.equal(call(documents, { method: "POST", headers: keyed, body: { ...input, content: "Different" } }).status, 409); + assert.equal(call(documents, { method: "POST", headers, body: input }).status, 409); +})); + +test("invalid batch members, duplicate identifiers, raw images and malformed fields never write", () => isolated(({ headers }) => { + for (const body of [{ documents: [] }, { documents: [input, input] }, + { documents: [input], unexpected: true }, { documents: Array.from({ length: 101 }, () => input) }]) + assert.equal(call(batch, { method: "POST", headers, body }).status, 400); + for (const body of [{ ...input, revision: undefined }, { ...input, typo: true }, + { ...input, slug: "../private" }, { ...input, content: "![raw](/figures/original.png)" }, null]) + assert.equal(call(documents, { method: "POST", headers, body }).status, 400); + assert.deepEqual(allDocuments(), []); + assert.equal(call(documents, { method: "DELETE", headers }).status, 405); + assert.equal(call(item, { headers, query: { slug: ["missing"] } }).status, 404); +})); diff --git a/scripts/docs-admin.sh b/scripts/docs-admin.sh index 8bc4eeb..b2309fe 100644 --- a/scripts/docs-admin.sh +++ b/scripts/docs-admin.sh @@ -3,7 +3,7 @@ set -eu cd /opt/endorphin-tabularium-docs/current case "${1:-}" in - list|get|save|history|export|backup|import) + list|get|save|history|export|backup|import|key-create|key-list|key-revoke) docker compose -p endorphin-tabularium-docs --env-file /etc/endorphin-tabularium-docs/production.env --env-file release.env -f compose.yml exec -T docs node apps/docs/docs-cli.cjs "$@" ;; - *) echo 'Usage: docs list | get | save < JSON | history | export | backup | import < JSON' >&2; exit 1 ;; + *) echo 'Usage: docs key-create [read|read-write] [days] | key-list | key-revoke | list | get | save < JSON | history | export | backup | import < JSON' >&2; exit 1 ;; esac diff --git a/scripts/documents-client.py b/scripts/documents-client.py new file mode 100644 index 0000000..fc287be --- /dev/null +++ b/scripts/documents-client.py @@ -0,0 +1,167 @@ +"""Upload documents through the database API; credentials stay in a private file.""" +import argparse +import json +import os +from pathlib import Path +import stat +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid + + +class DocumentClientError(ValueError): + pass + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # Never forward a bearer credential to a redirected host or protocol. + return None + + +def server_url(value): + parsed = urllib.parse.urlsplit(value) + if (parsed.username or parsed.password or parsed.query or parsed.fragment + or parsed.path not in ('', '/') or not parsed.hostname): + raise ValueError('Provide a server origin without credentials, path, query or fragment') + if parsed.scheme != 'https' and not ( + parsed.scheme == 'http' and parsed.hostname in ('localhost', '127.0.0.1', '::1')): + raise ValueError('Use HTTPS or an HTTP loopback connection through a trusted SSH tunnel') + return value.rstrip('/') + + +def read_token(filename): + path = Path(filename) + if stat.S_IMODE(path.stat().st_mode) & 0o077: + raise ValueError('API key file must be private (chmod 600)') + value = path.read_text().strip() + if value.startswith('{'): + value = json.loads(value)['token'] + if not isinstance(value, str) or not value.startswith('tdoc_') or len(value) != 48: + raise ValueError('Invalid document API key file') + return value + + +def request(base, token, method, path, payload=None, request_id=None): + data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode() + if data is not None and len(data) > 8 * 1024 * 1024: + raise ValueError('Request exceeds 8 MiB; split into smaller atomic batches') + headers = {'Authorization': 'Bearer ' + token, 'Accept': 'application/json'} + if data is not None: + headers['Content-Type'] = 'application/json' + headers['Idempotency-Key'] = request_id + opener = urllib.request.build_opener(NoRedirect()) + for attempt in range(2): + req = urllib.request.Request(base + path, data=data, method=method, headers=headers) + try: + with opener.open(req, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + if error.code in (502, 503, 504) and attempt == 0: + error.close() + time.sleep(1) + continue + status = error.code + error.close() + explanations = {400: 'Invalid document or pagination', 401: 'Key missing, expired or revoked', + 403: 'Read-only key', 404: 'Document or revision not found', + 409: 'Revision or request-id conflict; read current state before retrying', + 413: 'Request too large', 503: 'Database unavailable'} + raise DocumentClientError(f'HTTP {status}: ' + explanations.get(status, 'Request failed')) from None + except (urllib.error.URLError, TimeoutError): + if attempt == 0: + time.sleep(1) + continue + raise DocumentClientError('Connection failed; retry using the same request id and unchanged input') from None + + +def document_batches(documents, max_bytes=8 * 1024 * 1024): + if not isinstance(documents, list) or not documents: + raise DocumentClientError('Provide a nonempty documents array') + batches, chunk, size, seen = [], [], 20, set() + for document in documents: + if not isinstance(document, dict) or not isinstance(document.get('sourceSlug'), str): + raise DocumentClientError('Every document needs a sourceSlug') + if document['sourceSlug'] in seen: + raise DocumentClientError('Duplicate sourceSlug in upload') + seen.add(document['sourceSlug']) + encoded = len(json.dumps(document, ensure_ascii=False).encode()) + 2 + if encoded + 20 > max_bytes: + raise DocumentClientError('A document exceeds the request limit') + if chunk and (len(chunk) == 100 or size + encoded > max_bytes): + batches.append(chunk) + chunk, size = [], 20 + chunk.append(document) + size += encoded + batches.append(chunk) + return batches + + +def main(): + os.umask(0o077) + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--url', default=os.environ.get('TABULARIUM_DOCS_URL')) + parser.add_argument('--key-file', default=os.environ.get('TABULARIUM_DOCS_KEY_FILE')) + sub = parser.add_subparsers(dest='command', required=True) + listing = sub.add_parser('list') + listing.add_argument('--limit', type=int, default=50) + listing.add_argument('--offset', type=int, default=0) + reading = sub.add_parser('get') + reading.add_argument('slug') + mode = reading.add_mutually_exclusive_group() + mode.add_argument('--history', action='store_true') + mode.add_argument('--revision', type=int) + reading.add_argument('--limit', type=int, default=50) + reading.add_argument('--offset', type=int, default=0) + saving = sub.add_parser('save') + saving.add_argument('file', help='Private JSON file or - for stdin; one document or {documents: [...]}') + saving.add_argument('--request-id', help='Reuse this id only with the same input when retrying') + args = parser.parse_args() + if not args.url or not args.key_file: + parser.error('Set --url and --key-file or TABULARIUM_DOCS_URL and TABULARIUM_DOCS_KEY_FILE') + try: + base = server_url(args.url) + token = read_token(args.key_file) + if args.command == 'list': + result = request(base, token, 'GET', '/api/documents?' + urllib.parse.urlencode( + {'limit': args.limit, 'offset': args.offset})) + elif args.command == 'get': + path = '/api/documents/item/' + urllib.parse.quote(args.slug, safe='/') + if args.history: + path += '?' + urllib.parse.urlencode({'history': 1, 'limit': args.limit, 'offset': args.offset}) + elif args.revision is not None: + path += '?' + urllib.parse.urlencode({'revision': args.revision}) + result = request(base, token, 'GET', path) + else: + payload = json.load(sys.stdin) if args.file == '-' else json.loads(Path(args.file).read_text()) + request_id = args.request_id or str(uuid.uuid4()) + if not request_id or len(request_id) > 100 or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:-' for c in request_id): + raise ValueError('Invalid request id') + print('Request id: ' + request_id, file=sys.stderr) + if isinstance(payload, dict) and 'documents' in payload: + if set(payload) != {'documents'}: + raise DocumentClientError('Batch payload must contain only documents') + chunks = document_batches(payload['documents']) + receipts, replayed = [], 0 + for index, chunk in enumerate(chunks, 1): + batch_id = f'{request_id}:{index}' + saved = request(base, token, 'POST', '/api/documents/batch', {'documents': chunk}, batch_id) + receipts.extend(saved['documents']) + replayed += int(saved['replayed']) + print(f'Batch {index}/{len(chunks)} committed; {len(receipts)} documents total; request id: {batch_id}', file=sys.stderr) + result = {'documents': receipts, 'batches': len(chunks), 'replayedBatches': replayed, 'requestId': request_id} + else: + result = request(base, token, 'POST', '/api/documents', payload, request_id) + print(json.dumps(result, ensure_ascii=False, indent=2)) + except DocumentClientError as error: + raise SystemExit(str(error)) from None + except (ValueError, OSError, KeyError, TypeError): + # Never print exception objects that could contain credentials or document data. + raise SystemExit('Document request failed. Check the URL, private key file, JSON schema and server response; credentials omitted.') from None + + +if __name__ == '__main__': + main() diff --git a/scripts/test_documents_client.py b/scripts/test_documents_client.py new file mode 100644 index 0000000..11e1267 --- /dev/null +++ b/scripts/test_documents_client.py @@ -0,0 +1,85 @@ +"""Lightweight client tests: no HTTP server, compilation or real credentials.""" +import importlib.util +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import urllib.error +import urllib.request + +spec = importlib.util.spec_from_file_location('documents_client', Path(__file__).with_name('documents-client.py')) +client = importlib.util.module_from_spec(spec) +spec.loader.exec_module(client) + + +class ClientTests(unittest.TestCase): + def test_only_tls_or_loopback_origins_are_accepted(self): + self.assertEqual(client.server_url('https://docs.example/'), 'https://docs.example') + self.assertEqual(client.server_url('http://127.0.0.1:18135'), 'http://127.0.0.1:18135') + for url in ['http://docs.example', 'https://user:secret@docs.example', 'file:///tmp/data', + 'https://docs.example/api', 'https://docs.example/?token=secret']: + with self.assertRaises(ValueError): + client.server_url(url) + + def test_key_file_permissions_and_formats(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'key.json' + token = 'tdoc_' + 'x' * 43 + path.write_text(json.dumps({'token': token})) + path.chmod(0o600) + self.assertEqual(client.read_token(path), token) + path.chmod(0o644) + with self.assertRaises(ValueError): + client.read_token(path) + + def test_redirects_never_forward_bearer_credentials(self): + request = urllib.request.Request('https://docs.example', headers={'Authorization': 'Bearer test-only'}) + self.assertIsNone(client.NoRedirect().redirect_request(request, None, 302, '', {}, 'https://other.example')) + + def test_network_retry_preserves_idempotency_and_body(self): + calls = [] + + class Opener: + def open(self, request, timeout): + calls.append(request) + if len(calls) == 1: + raise urllib.error.URLError('temporary network failure') + return io.BytesIO(b'{"replayed":true}') + + with patch.object(client.urllib.request, 'build_opener', return_value=Opener()), patch.object(client.time, 'sleep'): + result = client.request('https://docs.example', 'test-token', 'POST', '/api/documents', {'revision': 0}, 'request-1') + self.assertTrue(result['replayed']) + self.assertEqual(calls[0].data, calls[1].data) + self.assertEqual(calls[0].get_header('Idempotency-key'), 'request-1') + self.assertEqual(calls[1].get_header('Idempotency-key'), 'request-1') + + def test_large_uploads_are_split_by_count_and_bytes(self): + documents = [{'sourceSlug': str(i), 'content': 'a' * 20} for i in range(201)] + batches = client.document_batches(documents) + self.assertEqual([len(batch) for batch in batches], [100, 100, 1]) + self.assertEqual([doc for batch in batches for doc in batch], documents) + bounded = client.document_batches(documents[:5], max_bytes=150) + self.assertTrue(all(len(json.dumps({'documents': batch}, ensure_ascii=False).encode()) <= 150 for batch in bounded)) + + def test_duplicate_identifiers_and_oversized_items_fail_before_upload(self): + with self.assertRaises(ValueError): + client.document_batches([{'sourceSlug': 'same'}, {'sourceSlug': 'same'}]) + with self.assertRaises(ValueError): + client.document_batches([{'sourceSlug': 'one', 'content': 'a' * 500}], max_bytes=100) + + def test_conflict_is_reported_without_printing_server_content_or_secret(self): + class Opener: + def open(self, request, timeout): + raise urllib.error.HTTPError(request.full_url, 409, 'Conflict', {}, io.BytesIO(b'{"error":"private-content"}')) + + with patch.object(client.urllib.request, 'build_opener', return_value=Opener()): + with self.assertRaisesRegex(client.DocumentClientError, 'HTTP 409') as result: + client.request('https://docs.example', 'private-token', 'POST', '/api/documents', {}, 'request-1') + self.assertNotIn('private-token', str(result.exception)) + self.assertNotIn('private-content', str(result.exception)) + + +if __name__ == '__main__': + unittest.main()