diff --git a/AGENTS.md b/AGENTS.md index e6e5432..80bc2b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,6 +4,7 @@ - Web 技术栈为 TypeScript、React、Tailwind CSS、Next.js Pages Router；不要切换到 App Router。 - 使用 pnpm 管理 JavaScript workspace，使用 uv 管理 `tools/figures` 的 Python 项目，保留两套锁文件。 - 文档正文、目录元数据与修订记录以站点现有 SQLite 数据库为唯一权威源，使用文档 API 或 docs 管理工具直接更新；不得把新正文、目录 JSON、数据库文件或日常内容快照提交到仓库。历史 Git 文档只用于显式的一次性迁移，不作为运行时种子。Python 图形源码放在 `tools/figures/src/endor_figures`，可复现的 SVG 输出放在 `apps/docs/public/figures`。 +- 编辑已有文章时优先使用章节 API：先读取章节目录与内部摘要，据此选择需要读取正文的章节，再提交单章或多章操作。摘要缺失时先读对应正文并补充；修改观点前必须核对该章原文，检查读取版本与提交 revision 一致。摘要只写入内部章节数据，不放入网页正文、目录摘要或 Git；改写正文时同步更新摘要。完整操作与 Python 客户端用法见 README 的「先读摘要，再按章节编辑」。 - 未确认的架构提议明确标注为草案，不能表述为现有实现。 - 本地开发与预览只监听 `127.0.0.1`。部署容器监听 `0.0.0.0:3000`，服务器入口默认 18135，可通过独立 production.env 配置。 - 对 gitea.entroia.com 仅使用 codex-bot，提交身份为 Codex Bot 。SSH 使用隔离配置：`ssh -F /dev/null -o IdentitiesOnly=yes -i /Users/zzyang/.codex/credentials/gitea-entroia/codex_entroia_ed25519 -p 22222`。 diff --git a/README.md b/README.md index 7eb67af..eb775e3 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,10 @@ sudo /opt/endorphin-tabularium-docs/docs save < "$doc_work_dir/document.json" | `GET /api/documents/item/?revision=1` | 读取指定修订的完整正文 | | `POST /api/documents` | 保存一个完整文档 JSON；字段与 `docs save` 相同 | | `POST /api/documents/batch` | 保存 `{ "documents": [...] }`，每个请求 1—100 篇 | +| `GET /api/documents/sections/` | 读取章节 ID、标题和内部摘要，不返回正文 | +| `GET /api/documents/sections/?section=` | 读取选中章节正文；重复 `section` 可选多章 | +| `GET /api/documents/sections/?revision=2` | 读取对应修订的章节目录；也可搭配 `section` | +| `POST /api/documents/sections/` | 原子执行章节新增、更新、删除和移动 | 写入使用 `Authorization: Bearer <文档 API Key>`。新建传 `revision: 0`；更新先读取当前全文，再使用读到的 revision。每篇正文最多 2,000,000 个 JavaScript 字符单位；每次请求最多 8 MiB。目录和修订分页的 `limit` 范围为 1—100，响应包含 `total` 和 `nextOffset`。 @@ -82,6 +86,52 @@ sudo /opt/endorphin-tabularium-docs/docs save < "$doc_work_dir/document.json" 建议写入时提供 `Idempotency-Key`，由 1—128 个字母、数字或 `._:-` 组成。同一个 Key、相同请求 ID 和相同输入重试时，返回原回执并标记 `replayed: true`，不追加修订；相同 ID 改换输入会冲突。回执描述原请求，不代表之后没有其他修改；需要当前内容时重新读取文档。改过正文或 revision 后，应使用新的请求 ID。 +### 先读摘要，再按章节编辑 + +章节接口使用相同的文档 Key 和读写权限。编辑流程为：先读章节目录及摘要，选择需要深入的章节，再读取这些章节的正文，最后提交一组操作。不要只凭摘要修改未读正文；目录与正文响应的 `revision` 应一致，否则重新读取。接口不调用模型生成摘要：由编辑 Agent 阅读正文、撰写摘要，并在编辑请求中保存。 + +章节按行首 Markdown ATX 二级标题（`##`）划分，三级至六级标题随所属章节一起编辑；代码围栏内的标题不会切章。二级标题之前的文档标题及导言属于固定 ID `preamble`，可以修改但不能删除或移动。Setext 标题、带缩进或引用、列表标记的标题不作为章节边界。正文无行首二级标题时，全文作为导言处理。拆分及组合保持换行和未编辑内容原样，不自动格式化 Markdown。 + +目录中的每章包含 `id`、`title`、`position`、`summary`、`summaryStatus` 和 `contentHash`。摘要为编辑内部数据，单独保存在 SQLite 的 `document_sections` 和 `document_section_revisions` 中，不加入普通文档响应、网页 props、搜索索引或页面正文。完整数据库备份包含这些表；`docs export` 仍只导出当前文档，不包含内部摘要和修订。 + +旧文章无需重写，首次读取即可按原文返回目录，初始摘要为空，`summaryStatus` 为 `missing`。Agent 应按需读取这些章节，生成摘要并通过 `update` 保存；保存后状态为 `current`。这里的 `current` 仅表示摘要与该版正文一起保存，不保证模型的总结正确。全文接口仍兼容；未变化章节保留摘要与 ID，正文变化的章节清空摘要，等待更新。全文改名且改写章节时可能产生新 ID，因此全文保存后应重新读取目录。 + +章节编辑请求示例（示例 ID 需要替换为目录实际返回的值）： + +```json +{ + "revision": 3, + "operations": [ + { + "op": "update", + "id": "s_existing", + "content": "## 原有章节\n\n修改后的正文。\n\n", + "summary": "本章说明修改后的核心观点与边界。" + }, + { + "op": "insert", + "after": "s_existing", + "content": "## 新章节\n\n新增论证。\n\n", + "summary": "补充上一章尚未覆盖的论证。" + } + ] +} +``` + +`update` 可只改 `summary`，也可改正文；正文改变时若不提供新摘要，旧摘要会清空。`insert` 必须提供正文和非空摘要；摘要最多 8000 个 JavaScript 字符单位。`delete` 使用 `{ "op": "delete", "id": "s_existing" }`；`move` 使用 `{ "op": "move", "id": "s_existing", "after": "preamble" }`。按数组顺序执行，最多 100 个操作；新章 ID 由服务器生成，在回执目录中返回。章正文应从自己的 `##` 标题开始，只包含一个二级标题，并保留与下一章之间的换行；跨章代码围栏或章节边界改变会被拒绝，拆章请使用多个操作。 + +章节变更、内部摘要、组合后的全文和历史快照在同一事务内提交。每批操作只增加一次文章 `revision`；摘要单独更新也增加版本号。后续操作失败或全文版本过期时全部回滚。写入沿用幂等规则，回执包含新版本、章节目录和 `replayed`；章节接口与全文接口的幂等记录分开。修改章节不会改变其 ID，移动亦保留 ID；删除后的正文和摘要仍可从对应的旧修订读取。启用前的历史修订只有原文，摘要显示缺失。 + +Python 客户端对应命令： + +```sh +python3 scripts/documents-client.py outline endor/endor-purpose-draft +python3 scripts/documents-client.py sections endor/endor-purpose-draft preamble s_existing +python3 scripts/documents-client.py edit-sections endor/endor-purpose-draft /private/path/edit.json --request-id chapter-edit-1 +``` + +`outline` 和 `sections` 支持 `--revision`；请求文件只包含 `revision` 和 `operations`，不要把内部摘要文件提交到 Git。 + ### 创建、查看与撤销 Key 管理员只需通过已验证的服务器连接进行 Key 管理： diff --git a/apps/docs/src/lib/docs.ts b/apps/docs/src/lib/docs.ts index 5e16128..ced0e1c 100644 --- a/apps/docs/src/lib/docs.ts +++ b/apps/docs/src/lib/docs.ts @@ -1,5 +1,9 @@ import { randomUUID, createHash } from "node:crypto"; import { openSiteDatabase } from "./site-database"; +import { DocumentValidationError, DocumentConflictError } from "./document-errors"; +import { sectionSchema, loadSections, reconcileSections, persistSections, splitSections, outline, + applySectionOperations, parseSectionEdit, type DocumentSection } from "./document-sections"; +export { DocumentValidationError, DocumentConflictError } from "./document-errors"; export type DocLink = { slug: string; sourceSlug: string; category: string; title: string; searchText?: string; @@ -9,8 +13,6 @@ export type DocLink = { export type Doc = DocLink & { content: string }; export type DocInput = Omit; export type DocumentExport = { format: "tabularium-documents-v1"; documents: Doc[] }; -export class DocumentValidationError extends Error {} -export class DocumentConflictError extends Error {} export type DocumentReceipt = Pick; export const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/; @@ -36,6 +38,7 @@ function database() { digest TEXT PRIMARY KEY, document_count INTEGER NOT NULL, created_at TEXT NOT NULL ); `); + sectionSchema(db); return db; } @@ -57,12 +60,13 @@ export function validateDocument(input: DocInput): void { throw new DocumentValidationError("Use compressed WebP images instead of original rasters"); } -function write(db: ReturnType, input: DocInput, expectedRevision: number): Doc { +function write(db: ReturnType, input: DocInput, expectedRevision: number, editedSections?: DocumentSection[]): Doc { validateDocument(input); - const existing = db.prepare("SELECT id, revision FROM documents WHERE source_slug=?") - .get(input.sourceSlug) as { id: string; revision: number } | undefined; + const existing = db.prepare("SELECT id, revision, payload FROM documents WHERE source_slug=?") + .get(input.sourceSlug) as { id: string; revision: number; payload: string } | undefined; if ((existing?.revision ?? 0) !== expectedRevision) throw new DocumentConflictError("Document revision conflict"); const id = existing?.id ?? randomUUID(); + const previous = existing ? loadSections(db, id, (JSON.parse(existing.payload) as Doc).content) : []; const doc: Doc = { slug: input.slug, sourceSlug: input.sourceSlug, category: input.category, title: input.title, group: input.group, collection: input.collection, description: input.description, draft: input.draft, featured: input.featured, content: input.content, revision: expectedRevision + 1 }; @@ -80,9 +84,75 @@ function write(db: ReturnType, input: DocInput, expectedRevisio db.prepare("INSERT OR IGNORE INTO document_aliases VALUES (?,?)").run(alias, id); db.prepare("INSERT INTO document_revisions VALUES (?,?,?,?)") .run(id, doc.revision, payload, new Date().toISOString()); + persistSections(db, id, doc.revision, editedSections ?? reconcileSections(doc.content, previous)); return doc; } +// Internal summaries never enter Doc, catalog, search or browser page props. +export function readDocumentSections(slug: string, revision?: number) { + if (!validSlug.test(slug)) throw new DocumentValidationError("Invalid document slug"); + if (revision !== undefined && (!Number.isSafeInteger(revision) || revision < 1)) throw new DocumentValidationError("Invalid revision"); + const db = database(); + try { + db.exec("BEGIN"); + const row = db.prepare(`SELECT d.id,d.payload FROM documents d JOIN document_aliases a + ON a.document_id=d.id WHERE a.alias=?`).get(slug) as { id: string; payload: string } | undefined; + if (!row) { db.exec("COMMIT"); return null; } + let doc = JSON.parse(row.payload) as Doc; + let sections: DocumentSection[]; + if (revision === undefined || revision === doc.revision) sections = loadSections(db, row.id, doc.content); + else { + const prior = db.prepare("SELECT payload FROM document_revisions WHERE document_id=? AND revision=?") + .get(row.id, revision) as { payload: string } | undefined; + if (!prior) { db.exec("COMMIT"); return null; } + doc = JSON.parse(prior.payload) as Doc; + const snapshot = db.prepare("SELECT payload FROM document_section_revisions WHERE document_id=? AND revision=?") + .get(row.id, revision) as { payload: string } | undefined; + sections = snapshot ? JSON.parse(snapshot.payload) as DocumentSection[] : splitSections(doc.content); + } + db.exec("COMMIT"); + return { slug: doc.slug, sourceSlug: doc.sourceSlug, title: doc.title, revision: doc.revision, sections }; + } finally { db.close(); } +} + +export function editDocumentSections(slug: string, value: unknown, request?: { namespace: string; id: string }) { + if (!validSlug.test(slug)) throw new DocumentValidationError("Invalid document slug"); + const edit = parseSectionEdit(value); + if (request && (!request.namespace || !/^[A-Za-z0-9._:-]{1,128}$/.test(request.id))) throw new DocumentValidationError("Invalid Idempotency-Key"); + const digest = createHash("sha256").update(JSON.stringify({ slug, edit })).digest("hex"); + const namespace = request ? request.namespace + ":sections" : undefined; + const db = database(); + try { + db.exec("BEGIN IMMEDIATE"); + try { + if (request) { + const prior = db.prepare("SELECT digest,response FROM document_write_requests WHERE namespace=? AND request_id=?") + .get(namespace!, request.id) as { digest: string; response: string } | undefined; + if (prior) { + if (prior.digest !== digest) throw new DocumentConflictError("Idempotency-Key was used with different content"); + db.exec("COMMIT"); + return { ...JSON.parse(prior.response) as { slug: string; sourceSlug: string; revision: number; sections: ReturnType }, replayed: true }; + } + } + const row = db.prepare(`SELECT d.id,d.payload FROM documents d JOIN document_aliases a + ON a.document_id=d.id WHERE a.alias=?`).get(slug) as { id: string; payload: string } | undefined; + if (!row) { db.exec("COMMIT"); return null; } + const doc = JSON.parse(row.payload) as Doc; + if (doc.revision !== edit.revision) throw new DocumentConflictError("Document revision conflict"); + const previous = loadSections(db, row.id, doc.content); + const sections = applySectionOperations(previous, edit.operations); + // Preserve the initial outline when upgrading an existing document for the first time. + db.prepare("INSERT OR IGNORE INTO document_section_revisions VALUES (?,?,?)").run(row.id, doc.revision, JSON.stringify(previous)); + const saved = write(db, { ...doc, content: sections.map((s) => s.content).join("") }, doc.revision, sections); + const receipt = { slug: saved.slug, sourceSlug: saved.sourceSlug, revision: saved.revision, sections: outline(sections) }; + if (request) db.prepare("INSERT INTO document_write_requests VALUES (?,?,?,?)") + .run(namespace!, request.id, digest, JSON.stringify(receipt)); + db.exec("COMMIT"); + return { ...receipt, replayed: false }; + } catch (error) { db.exec("ROLLBACK"); throw error; } + } finally { db.close(); } +} + export function saveDocument(input: DocInput, expectedRevision: number): Doc { if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) throw new DocumentValidationError("Provide expected revision"); const db = database(); diff --git a/apps/docs/src/lib/document-errors.ts b/apps/docs/src/lib/document-errors.ts new file mode 100644 index 0000000..90ab801 --- /dev/null +++ b/apps/docs/src/lib/document-errors.ts @@ -0,0 +1,2 @@ +export class DocumentValidationError extends Error {} +export class DocumentConflictError extends Error {} diff --git a/apps/docs/src/lib/document-sections.ts b/apps/docs/src/lib/document-sections.ts new file mode 100644 index 0000000..102a689 --- /dev/null +++ b/apps/docs/src/lib/document-sections.ts @@ -0,0 +1,144 @@ +import { createHash, randomUUID } from "node:crypto"; +import type { DatabaseSync } from "node:sqlite"; +import { DocumentValidationError } from "./document-errors"; + +export type DocumentSection = { + id: string; title: string; content: string; summary: string; +}; +export type SectionOperation = + | { op: "update"; id: string; content?: string; summary?: string } + | { op: "insert"; after: string; content: string; summary: string } + | { op: "delete"; id: string } + | { op: "move"; id: string; after: string }; +export type SectionEdit = { revision: number; operations: SectionOperation[] }; +const hash = (text: string) => createHash("sha256").update(text).digest("hex"); + +// Preserve every byte, including CRLF and separators. H3-H6 belong to their H2 chapter. +export function splitSections(content: string): DocumentSection[] { + const starts: { offset: number; title: string }[] = [{ offset: 0, title: "导言" }]; + let offset = 0; + let fence: { marker: string; length: number } | undefined; + for (const line of content.match(/[^\n]*\n|[^\n]+$/g) ?? []) { + const opening = line.match(/^ {0,3}(`{3,}|~{3,})(.*?)(?:\r?\n)?$/); + if (fence) { + if (new RegExp(`^ {0,3}${fence.marker}{${fence.length},}[ \\t]*\\r?\\n?$`).test(line)) fence = undefined; + } else if (opening && !(opening[1][0] === "`" && opening[2].includes("`"))) { + fence = { marker: opening[1][0], length: opening[1].length }; + } else { + const heading = line.match(/^##(?:[ \t]+(.*?))?[ \t]*\r?\n?$/); + if (heading) starts.push({ offset, title: (heading[1] ?? "").replace(/[ \t]+#+[ \t]*$/, "").trim() }); + } + offset += line.length; + } + const seen = new Map(); + return starts.map((start, index) => { + const occurrence = seen.get(start.title) ?? 0; + seen.set(start.title, occurrence + 1); + return { id: index === 0 ? "preamble" : "s_" + hash(`${start.title}\0${occurrence}`).slice(0, 24), + title: start.title, content: content.slice(start.offset, starts[index + 1]?.offset), summary: "" }; + }); +} + +export function sectionSchema(db: DatabaseSync) { + db.exec(`CREATE TABLE IF NOT EXISTS document_sections ( + document_id TEXT PRIMARY KEY REFERENCES documents(id), payload TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS document_section_revisions ( + document_id TEXT NOT NULL REFERENCES documents(id), revision INTEGER NOT NULL, + payload TEXT NOT NULL, PRIMARY KEY(document_id, revision) + );`); +} + +export function loadSections(db: DatabaseSync, documentId: string, content: string): DocumentSection[] { + const row = db.prepare("SELECT payload FROM document_sections WHERE document_id=?") + .get(documentId) as { payload: string } | undefined; + if (!row) return splitSections(content); + const sections = JSON.parse(row.payload) as DocumentSection[]; + // A legacy writer must not leave an apparently current outline over different Markdown. + return sections.map((s) => s.content).join("") === content ? sections : reconcileSections(content, sections); +} + +export function reconcileSections(content: string, previous: DocumentSection[]): DocumentSection[] { + const unused = new Set(previous); + return splitSections(content).map((section) => { + const prior = section.id === "preamble" ? previous.find((p) => p.id === "preamble") : + previous.find((p) => p.id !== "preamble" && unused.has(p) && p.content === section.content) ?? + previous.find((p) => p.id !== "preamble" && unused.has(p) && p.title === section.title); + if (prior) unused.delete(prior); + return { ...section, id: prior?.id ?? (section.id === "preamble" ? "preamble" : "s_" + randomUUID()), + summary: prior?.content === section.content ? prior.summary : "" }; + }); +} + +export function persistSections(db: DatabaseSync, documentId: string, revision: number, sections: DocumentSection[]) { + const payload = JSON.stringify(sections); + db.prepare(`INSERT INTO document_sections VALUES (?,?) ON CONFLICT(document_id) + DO UPDATE SET payload=excluded.payload`).run(documentId, payload); + db.prepare("INSERT INTO document_section_revisions VALUES (?,?,?)").run(documentId, revision, payload); +} + +export function outline(sections: DocumentSection[]) { + return sections.map(({ content, ...section }, position) => ({ ...section, position, + contentHash: hash(content), summaryStatus: section.summary.trim() ? "current" : "missing" })); +} + +export function parseSectionEdit(value: unknown): SectionEdit { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new DocumentValidationError("Expected section edit object"); + const row = value as Record; + if (Object.keys(row).some((key) => !["revision", "operations"].includes(key)) || + !Number.isSafeInteger(row.revision) || (row.revision as number) < 1 || !Array.isArray(row.operations) || + !row.operations.length || row.operations.length > 100) throw new DocumentValidationError("Provide revision and 1 to 100 operations"); + for (const raw of row.operations) { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) throw new DocumentValidationError("Invalid operation"); + const op = raw as Record; + const fields: Record = { update: ["op", "id", "content", "summary"], + insert: ["op", "after", "content", "summary"], delete: ["op", "id"], move: ["op", "id", "after"] }; + if (typeof op.op !== "string" || !Object.hasOwn(fields, op.op) || Object.keys(op).some((key) => !fields[op.op as string].includes(key))) + throw new DocumentValidationError("Unknown operation or field"); + for (const key of fields[op.op].filter((key) => !["content", "summary", "op"].includes(key))) + if (typeof op[key] !== "string" || !(op[key] as string).length) throw new DocumentValidationError("Missing section identifier"); + for (const key of ["content", "summary"]) + if (op[key] !== undefined && typeof op[key] !== "string") throw new DocumentValidationError(`Invalid ${key}`); + if (op.op === "update" && op.content === undefined && op.summary === undefined) throw new DocumentValidationError("Empty update"); + if (op.op === "insert" && (typeof op.content !== "string" || typeof op.summary !== "string" || !op.summary.trim())) + throw new DocumentValidationError("Insert requires content and summary"); + if (typeof op.summary === "string" && (!op.summary.trim() || op.summary.length > 8000)) + throw new DocumentValidationError("Summary must contain 1 to 8000 characters"); + } + return { revision: row.revision as number, operations: row.operations as SectionOperation[] }; +} + +export function applySectionOperations(previous: DocumentSection[], operations: SectionOperation[]) { + const sections = previous.map((section) => ({ ...section })); + function index(id: string) { + const result = sections.findIndex((s) => s.id === id); + if (result < 0) throw new DocumentValidationError("Unknown section id"); + return result; + } + for (const op of operations) { + if (op.op === "insert") { + sections.splice(index(op.after) + 1, 0, { id: "s_" + randomUUID(), title: "", content: op.content, summary: op.summary }); + } else { + const at = index(op.id); + if (op.op === "update") { + if (op.content !== undefined && op.content !== sections[at].content) sections[at].summary = ""; + if (op.content !== undefined) sections[at].content = op.content; + if (op.summary !== undefined) sections[at].summary = op.summary; + } else { + if (op.id === "preamble") throw new DocumentValidationError("Cannot delete or move the preamble"); + if (op.op === "delete") sections.splice(at, 1); + else { + if (op.id === op.after) throw new DocumentValidationError("Cannot move section after itself"); + index(op.after); + const [section] = sections.splice(at, 1); + sections.splice(index(op.after) + 1, 0, section); + } + } + } + } + const content = sections.map((s) => s.content).join(""); + const parsed = splitSections(content); + if (parsed.length !== sections.length || parsed.some((s, i) => s.content !== sections[i].content)) + throw new DocumentValidationError("Each chapter must start with one H2 heading; preserve newline boundaries and balanced fences"); + return sections.map((s, i) => ({ ...s, title: parsed[i].title })); +} diff --git a/apps/docs/src/pages/api/documents/sections/[...slug].ts b/apps/docs/src/pages/api/documents/sections/[...slug].ts new file mode 100644 index 0000000..6d8c561 --- /dev/null +++ b/apps/docs/src/pages/api/documents/sections/[...slug].ts @@ -0,0 +1,37 @@ +import type { NextApiRequest, NextApiResponse } from "next"; +import { DocumentValidationError, readDocumentSections, editDocumentSections } from "@/lib/docs"; +import { outline } from "@/lib/document-sections"; +import { authorizeDocuments, documentApiError, idempotency } from "@/lib/documents-api"; + +export const config = { api: { bodyParser: { sizeLimit: "8mb" } } }; +export default function handler(request: NextApiRequest, response: NextApiResponse) { + try { + const key = authorizeDocuments(request, response, request.method === "POST"); + if (!key) return; + if (request.method !== "GET" && request.method !== "POST") { + response.setHeader("Allow", "GET, POST"); + return response.status(405).end(); + } + if (!Array.isArray(request.query.slug)) throw new DocumentValidationError("Invalid document slug"); + const slug = request.query.slug.join("/"); + if (Object.keys(request.query).some((k) => !["slug", "section", "revision"].includes(k))) + throw new DocumentValidationError("Unknown section query"); + if (request.method === "POST") { + if (request.query.section !== undefined || request.query.revision !== undefined) + throw new DocumentValidationError("POST uses revision and operations in the body"); + const saved = editDocumentSections(slug, request.body, idempotency(request, key.id)); + return saved ? response.status(200).json(saved) : response.status(404).json({ code: "not_found" }); + } + const revision = request.query.revision; + if (revision !== undefined && (typeof revision !== "string" || !/^[1-9]\d*$/.test(revision))) + throw new DocumentValidationError("Invalid revision"); + const doc = readDocumentSections(slug, revision === undefined ? undefined : Number(revision)); + if (!doc) return response.status(404).json({ code: "not_found" }); + const selected = request.query.section; + if (selected === undefined) return response.status(200).json({ ...doc, sections: outline(doc.sections) }); + const ids = typeof selected === "string" ? [selected] : selected; + if (!ids.length || ids.length > 100 || ids.some((id) => !id)) throw new DocumentValidationError("Select 1 to 100 sections"); + if (ids.some((id) => !doc.sections.some((s) => s.id === id))) return response.status(404).json({ code: "not_found" }); + return response.status(200).json({ ...doc, sections: doc.sections.filter((s) => ids.includes(s.id)) }); + } catch (error) { return documentApiError(response, error); } +} diff --git a/apps/docs/tests/documents-api.test.ts b/apps/docs/tests/documents-api.test.ts index 1c28528..12374d5 100644 --- a/apps/docs/tests/documents-api.test.ts +++ b/apps/docs/tests/documents-api.test.ts @@ -7,8 +7,11 @@ import type { NextApiRequest, NextApiResponse } from "next"; import documents from "../src/pages/api/documents/index"; import batch from "../src/pages/api/documents/batch"; import item from "../src/pages/api/documents/item/[...slug]"; +import sections from "../src/pages/api/documents/sections/[...slug]"; +import { splitSections } from "../src/lib/document-sections"; +import { openSiteDatabase } from "../src/lib/site-database"; import { createDocumentKey, authenticateDocumentKey, revokeDocumentKey, listDocumentKeys } from "../src/lib/document-keys"; -import { allDocuments, documentHistory, readDocument, saveDocument, type Doc } from "../src/lib/docs"; +import { allDocuments, documentHistory, readDocument, saveDocument, readDocumentSections, type Doc } from "../src/lib/docs"; import { newAuthConfig, signSession } from "../src/lib/auth"; const input: Doc = { slug: "endor/example", sourceSlug: "00-example", title: "Example", category: "Endor", @@ -134,3 +137,122 @@ test("invalid batch members, duplicate identifiers, raw images and malformed fie assert.equal(call(documents, { method: "DELETE", headers }).status, 405); assert.equal(call(item, { headers, query: { slug: ["missing"] } }).status, 404); })); + +const chapterContent = "# Example\n\nIntroduction.\n\n## First\n\nFirst body.\n\n### Detail\n\nChild stays in chapter.\n\n## Second\n\nSecond body.\n"; +type SectionResponse = { revision: number; replayed?: boolean; + sections: { id: string; title: string; summary: string; summaryStatus?: string; content?: string }[] }; +const chapterQuery = { slug: input.slug.split("/") }; + +test("chapter parser preserves bytes, fences, duplicate headings and CRLF", () => { + const content = "# Title\r\n\r\n## Repeated\r\n~~~text\r\n## Inside fence\r\n~~~\r\n### Nested\r\n```js\r\n## Also inside\r\n````\r\n\r\n## Repeated\r\nTail"; + const parts = splitSections(content); + assert.equal(parts.length, 3); + assert.notEqual(parts[1].id, parts[2].id); + assert.equal(parts.map((p) => p.content).join(""), content); + assert.equal(parts[1].title, "Repeated"); + assert.ok(parts[1].content.includes("### Nested")); + assert.equal(splitSections("").length, 1); + assert.equal(splitSections("# Title\n\n- Item\n ## List detail\n\n> ## Quoted\n").length, 1); +}); + +test("outline reads existing Markdown without changing it and returns only selected bodies", () => isolated(({ headers }) => { + saveDocument({ ...input, content: chapterContent }, 0); + const db = openSiteDatabase(); + db.exec("DELETE FROM document_sections; DELETE FROM document_section_revisions;"); db.close(); + const result = call(sections, { headers, query: chapterQuery }); + assert.equal(result.status, 200); + const outline = result.body as SectionResponse; + assert.equal(outline.revision, 1); assert.equal(outline.sections.length, 3); + assert.deepEqual(outline.sections.map((s) => s.title), ["导言", "First", "Second"]); + assert.ok(outline.sections.every((s) => s.content === undefined && s.summaryStatus === "missing")); + const selected = call(sections, { headers, query: { ...chapterQuery, section: outline.sections[1].id } }).body as SectionResponse; + assert.equal(selected.sections.length, 1); + assert.ok(selected.sections[0].content!.includes("### Detail")); + assert.equal(readDocument(input.slug)!.content, chapterContent); + assert.equal(call(sections, { headers, query: { ...chapterQuery, section: "missing" } }).status, 404); + assert.equal(call(sections, { headers, query: { ...chapterQuery, revision: "0" } }).status, 400); +})); + +test("internal summaries persist separately and chapter updates leave other bytes untouched", () => isolated(({ headers }) => { + saveDocument({ ...input, content: chapterContent }, 0); + const before = readDocumentSections(input.slug)!; + const first = before.sections[1]; const second = before.sections[2]; + const changed = first.content.replace("First body.", "Revised body."); + const edited = call(sections, { method: "POST", headers, query: chapterQuery, body: { revision: 1, operations: [ + { op: "update", id: first.id, content: changed, summary: "INTERNAL_SUMMARY_FIRST" }, + { op: "update", id: second.id, summary: "INTERNAL_SUMMARY_SECOND" }, + ] } }); + assert.equal(edited.status, 200); + const after = readDocumentSections(input.slug)!; + assert.equal(after.revision, 2); assert.equal(after.sections[1].id, first.id); + assert.equal(after.sections[0].content, before.sections[0].content); + assert.equal(after.sections[2].content, before.sections[2].content); + assert.equal(after.sections[1].summary, "INTERNAL_SUMMARY_FIRST"); + assert.equal(readDocument(input.slug)!.content, chapterContent.replace("First body.", "Revised body.")); + assert.ok(!JSON.stringify(allDocuments()).includes("INTERNAL_SUMMARY")); + assert.ok(!JSON.stringify(call(item, { headers, query: chapterQuery }).body).includes("INTERNAL_SUMMARY")); + assert.ok(!JSON.stringify(call(documents, { headers }).body).includes("INTERNAL_SUMMARY")); + const history = readDocumentSections(input.slug, 1)!; + assert.equal(history.sections[1].content, first.content); assert.equal(history.sections[1].summary, ""); + assert.equal(readDocumentSections(input.slug, 2)!.sections[1].summary, "INTERNAL_SUMMARY_FIRST"); +})); + +test("insert, move and delete chapters commit together with replay and conflict protection", () => isolated(({ headers }) => { + saveDocument({ ...input, content: chapterContent }, 0); + const before = readDocumentSections(input.slug)!; + const first = before.sections[1]; const second = before.sections[2]; + const body = { revision: 1, operations: [ + { op: "move", id: second.id, after: "preamble" }, + { op: "delete", id: first.id }, + { op: "insert", after: second.id, content: "## New chapter\n\nNew body.\n", summary: "New material" }, + ] }; + const options = { method: "POST", headers: { ...headers, "idempotency-key": "chapters-1" }, query: chapterQuery, body }; + assert.equal(call(sections, options).status, 200); + const repeated = call(sections, options); + assert.equal(repeated.status, 200); assert.equal((repeated.body as SectionResponse).replayed, true); + const after = readDocumentSections(input.slug)!; + assert.deepEqual(after.sections.map((s) => s.title), ["导言", "Second", "New chapter"]); + assert.equal(after.sections[1].content, second.content); + assert.equal(after.revision, 2); assert.equal(documentHistory(input.sourceSlug).length, 2); + assert.equal(call(sections, { ...options, body: { ...body, revision: 2 } }).status, 409); + assert.equal(call(sections, { ...options, headers }).status, 409); + assert.equal(saveDocument({ ...readDocument(input.slug)!, description: "Changed metadata" }, 2).revision, 3); + assert.equal(readDocumentSections(input.slug)!.sections[2].summary, "New material"); +})); + +test("invalid late operation rolls back content, summary and history; read-only keys cannot edit", () => isolated(({ headers }) => { + saveDocument({ ...input, content: chapterContent }, 0); + const before = readDocumentSections(input.slug)!; + const invalid = [ + [{ op: "update", id: before.sections[1].id, summary: "Should roll back" }, { op: "delete", id: "missing" }], + [{ op: "delete", id: "preamble" }], + [{ op: "move", id: before.sections[1].id, after: before.sections[1].id }], + [{ op: "update", id: before.sections[1].id, content: "## First\n\n## Extra\n" }], + [{ op: "update", id: before.sections[1].id, content: "## First\n![raw](/figures/raw.png)\n" }], + [{ op: "insert", after: "preamble", content: "## Missing summary\n" }], + [{ op: "delete", id: before.sections[1].id, unexpected: true }], + ]; + for (const operations of invalid) { + assert.equal(call(sections, { method: "POST", headers, query: chapterQuery, body: { revision: 1, operations } }).status, 400); + assert.deepEqual(readDocumentSections(input.slug), before); + assert.equal(documentHistory(input.sourceSlug).length, 1); + } + assert.equal(call(sections, { query: chapterQuery }).status, 401); + const reader = createDocumentKey("chapter reader", "read"); + const readHeaders = { authorization: "Bearer " + reader.token }; + assert.equal(call(sections, { headers: readHeaders, query: chapterQuery }).status, 200); + assert.equal(call(sections, { method: "POST", headers: readHeaders, query: chapterQuery, body: { revision: 1, operations: [] } }).status, 403); +})); + +test("full-document saves invalidate changed chapter summaries and preserve unaffected summaries and ids", () => isolated(({ headers }) => { + saveDocument({ ...input, content: chapterContent }, 0); + const before = readDocumentSections(input.slug)!; + assert.equal(call(sections, { method: "POST", headers, query: chapterQuery, body: { revision: 1, operations: + before.sections.map((s) => ({ op: "update", id: s.id, summary: "Summary of " + s.title })) } }).status, 200); + saveDocument({ ...readDocument(input.slug)!, content: chapterContent.replace("First body.", "Changed externally.") }, 2); + const after = readDocumentSections(input.slug)!; + assert.equal(after.sections[1].id, before.sections[1].id); + assert.equal(after.sections[1].summary, ""); + assert.equal(after.sections[2].summary, "Summary of Second"); + assert.equal(readDocumentSections(input.slug, 2)!.sections[1].summary, "Summary of First"); +})); diff --git a/scripts/documents-client.py b/scripts/documents-client.py index fc287be..1c3b679 100644 --- a/scripts/documents-client.py +++ b/scripts/documents-client.py @@ -116,6 +116,17 @@ def main(): mode.add_argument('--revision', type=int) reading.add_argument('--limit', type=int, default=50) reading.add_argument('--offset', type=int, default=0) + sections = sub.add_parser('outline', help='Read chapter titles and internal summaries without bodies') + sections.add_argument('slug') + sections.add_argument('--revision', type=int) + section = sub.add_parser('sections', help='Read selected chapter bodies') + section.add_argument('slug') + section.add_argument('ids', nargs='+') + section.add_argument('--revision', type=int) + editing = sub.add_parser('edit-sections', help='Atomically apply chapter operations from private JSON') + editing.add_argument('slug') + editing.add_argument('file') + editing.add_argument('--request-id') saving = sub.add_parser('save') saving.add_argument('file', help='Private JSON file or - for stdin; one document or {documents: [...]}') saving.add_argument('--request-id', help='Reuse this id only with the same input when retrying') @@ -135,6 +146,24 @@ def main(): elif args.revision is not None: path += '?' + urllib.parse.urlencode({'revision': args.revision}) result = request(base, token, 'GET', path) + elif args.command in ('outline', 'sections', 'edit-sections'): + path = '/api/documents/sections/' + urllib.parse.quote(args.slug, safe='/') + if args.command == 'edit-sections': + payload = json.load(sys.stdin) if args.file == '-' else json.loads(Path(args.file).read_text()) + request_id = args.request_id or str(uuid.uuid4()) + if not request_id or len(request_id) > 100 or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:-' for c in request_id): + raise ValueError('Invalid request id') + print('Request id: ' + request_id, file=sys.stderr) + result = request(base, token, 'POST', path, payload, request_id) + else: + query = [] + if args.revision is not None: + query.append(('revision', args.revision)) + if args.command == 'sections': + query.extend(('section', section_id) for section_id in args.ids) + if query: + path += '?' + urllib.parse.urlencode(query) + result = request(base, token, 'GET', path) else: payload = json.load(sys.stdin) if args.file == '-' else json.loads(Path(args.file).read_text()) request_id = args.request_id or str(uuid.uuid4()) diff --git a/scripts/test_documents_client.py b/scripts/test_documents_client.py index 11e1267..cf10e93 100644 --- a/scripts/test_documents_client.py +++ b/scripts/test_documents_client.py @@ -4,6 +4,7 @@ import io import json from pathlib import Path import tempfile +import sys import unittest from unittest.mock import patch import urllib.error @@ -15,6 +16,30 @@ spec.loader.exec_module(client) class ClientTests(unittest.TestCase): + def test_section_reads_select_bodies_and_revision_without_full_document_fetch(self): + argv = ['client', '--url', 'https://docs.example', '--key-file', '/private/key', + 'sections', 'endor/example', 'preamble', 's_one', '--revision', '3'] + with patch.object(sys, 'argv', argv), patch.object(client, 'read_token', return_value='test-token'), \ + patch.object(client, 'request', return_value={'revision': 3, 'sections': []}) as request, \ + patch.object(sys, 'stdout', io.StringIO()): + client.main() + request.assert_called_once_with('https://docs.example', 'test-token', 'GET', + '/api/documents/sections/endor/example?revision=3§ion=preamble§ion=s_one') + + def test_section_edit_keeps_payload_and_retry_id(self): + payload = {'revision': 3, 'operations': [{'op': 'update', 'id': 's_one', 'summary': 'Internal summary'}]} + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'edit.json' + path.write_text(json.dumps(payload)) + argv = ['client', '--url', 'https://docs.example', '--key-file', '/private/key', + 'edit-sections', 'endor/example', str(path), '--request-id', 'edit-1'] + with patch.object(sys, 'argv', argv), patch.object(client, 'read_token', return_value='test-token'), \ + patch.object(client, 'request', return_value={'revision': 4}) as request, \ + patch.object(sys, 'stdout', io.StringIO()), patch.object(sys, 'stderr', io.StringIO()): + client.main() + request.assert_called_once_with('https://docs.example', 'test-token', 'POST', + '/api/documents/sections/endor/example', payload, 'edit-1') + def test_only_tls_or_loopback_origins_are_accepted(self): self.assertEqual(client.server_url('https://docs.example/'), 'https://docs.example') self.assertEqual(client.server_url('http://127.0.0.1:18135'), 'http://127.0.0.1:18135')