feat: make the site database authoritative for documents and catalog
Verify database-backed documents / verify (pull_request) Failing after 6m6s
Verify database-backed documents / verify (pull_request) Failing after 6m6s
This commit is contained in:
+135
-38
@@ -1,50 +1,147 @@
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import catalog from "../../../../content/catalog.json";
|
||||
import { randomUUID, createHash } from "node:crypto";
|
||||
import { openSiteDatabase } from "./site-database";
|
||||
|
||||
const directory = path.resolve(process.cwd(), "../../content/docs");
|
||||
const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/;
|
||||
type Metadata = { slug?: string; category?: string; collection?: string; group?: string; description?: string; draft?: boolean; featured?: boolean };
|
||||
export type DocLink = {
|
||||
slug: string; sourceSlug: string; category: string; title: string; searchText?: string;
|
||||
group: string; collection: string; description: string; draft: boolean; featured: boolean;
|
||||
revision: number;
|
||||
};
|
||||
export type Doc = DocLink & { content: string };
|
||||
export type DocInput = Omit<Doc, "revision" | "searchText">;
|
||||
export type DocumentExport = { format: "tabularium-documents-v1"; documents: Doc[] };
|
||||
export const validSlug = /^[a-z0-9]+(?:-[a-z0-9]+)*(?:\/[a-z0-9]+(?:-[a-z0-9]+)*)*$/;
|
||||
|
||||
function database() {
|
||||
const db = openSiteDatabase();
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS documents (
|
||||
id TEXT PRIMARY KEY, slug TEXT NOT NULL UNIQUE, source_slug TEXT NOT NULL UNIQUE,
|
||||
revision INTEGER NOT NULL, payload TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS document_aliases (
|
||||
alias TEXT PRIMARY KEY, document_id TEXT NOT NULL REFERENCES documents(id)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS document_revisions (
|
||||
document_id TEXT NOT NULL REFERENCES documents(id), revision INTEGER NOT NULL,
|
||||
payload TEXT NOT NULL, created_at TEXT NOT NULL, PRIMARY KEY(document_id, revision)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS document_imports (
|
||||
digest TEXT PRIMARY KEY, document_count INTEGER NOT NULL, created_at TEXT NOT NULL
|
||||
);
|
||||
`);
|
||||
return db;
|
||||
}
|
||||
|
||||
export function validateDocument(input: DocInput): void {
|
||||
if (!input || typeof input !== "object") throw Error("Invalid document");
|
||||
for (const key of ["slug", "sourceSlug"] as const)
|
||||
if (typeof input[key] !== "string" || !validSlug.test(input[key])) throw Error(`Invalid ${key}`);
|
||||
if (input.slug.split("/").some((part) => /^\d+-/.test(part))) throw Error("Use a semantic public slug");
|
||||
for (const key of ["title", "category", "collection", "group"] as const)
|
||||
if (typeof input[key] !== "string" || !input[key].trim()) throw Error(`Missing ${key}`);
|
||||
if (typeof input.content !== "string" || typeof input.description !== "string" ||
|
||||
typeof input.draft !== "boolean" || typeof input.featured !== "boolean") throw Error("Invalid document fields");
|
||||
if (input.content.length > 2_000_000) throw Error("Document exceeds content limit");
|
||||
const images = [
|
||||
...input.content.matchAll(/!\[[^\]]*\]\(<?([^\s)>]+)/g),
|
||||
...input.content.matchAll(/<img\b[^>]*\bsrc=["']([^"']+)/gi),
|
||||
];
|
||||
if (images.some(([, source]) => /\.(?:png|jpe?g|tiff?|bmp)(?:[?#]|$)/i.test(source)))
|
||||
throw Error("Use compressed WebP images instead of original rasters");
|
||||
}
|
||||
|
||||
function write(db: ReturnType<typeof database>, input: DocInput, expectedRevision: number): Doc {
|
||||
validateDocument(input);
|
||||
const existing = db.prepare("SELECT id, revision FROM documents WHERE source_slug=?")
|
||||
.get(input.sourceSlug) as { id: string; revision: number } | undefined;
|
||||
if ((existing?.revision ?? 0) !== expectedRevision) throw Error("Document revision conflict");
|
||||
const id = existing?.id ?? randomUUID();
|
||||
const doc: Doc = { slug: input.slug, sourceSlug: input.sourceSlug, category: input.category,
|
||||
title: input.title, group: input.group, collection: input.collection, description: input.description,
|
||||
draft: input.draft, featured: input.featured, content: input.content, revision: expectedRevision + 1 };
|
||||
const payload = JSON.stringify(doc);
|
||||
// Aliases remain reserved after renaming; collision rolls back the whole write.
|
||||
db.prepare(`INSERT INTO documents VALUES (?,?,?,?,?) ON CONFLICT(id) DO UPDATE SET
|
||||
slug=excluded.slug, revision=excluded.revision, payload=excluded.payload`)
|
||||
.run(id, doc.slug, doc.sourceSlug, doc.revision, payload);
|
||||
for (const alias of new Set([doc.slug, doc.sourceSlug])) {
|
||||
const prior = db.prepare("SELECT document_id FROM document_aliases WHERE alias=?")
|
||||
.get(alias) as { document_id: string } | undefined;
|
||||
if (prior && prior.document_id !== id) throw Error(`Document alias collision: ${alias}`);
|
||||
db.prepare("INSERT OR IGNORE INTO document_aliases VALUES (?,?)").run(alias, id);
|
||||
}
|
||||
db.prepare("INSERT INTO document_revisions VALUES (?,?,?,?)")
|
||||
.run(id, doc.revision, payload, new Date().toISOString());
|
||||
return doc;
|
||||
}
|
||||
|
||||
export function saveDocument(input: DocInput, expectedRevision: number): Doc {
|
||||
if (!Number.isSafeInteger(expectedRevision) || expectedRevision < 0) throw Error("Provide expected revision");
|
||||
const db = database();
|
||||
try {
|
||||
db.exec("BEGIN IMMEDIATE");
|
||||
try { const doc = write(db, input, expectedRevision); db.exec("COMMIT"); return doc; }
|
||||
catch (error) { db.exec("ROLLBACK"); throw error; }
|
||||
} finally { db.close(); }
|
||||
}
|
||||
|
||||
export function readDocument(slug: string): Doc | null {
|
||||
if (!validSlug.test(slug)) return null;
|
||||
const db = database();
|
||||
try {
|
||||
const row = db.prepare(`SELECT d.payload FROM documents d JOIN document_aliases a
|
||||
ON a.document_id=d.id WHERE a.alias=?`).get(slug) as { payload: string } | undefined;
|
||||
return row ? JSON.parse(row.payload) as Doc : null;
|
||||
} finally { db.close(); }
|
||||
}
|
||||
|
||||
export function allDocuments(): Doc[] {
|
||||
const db = database();
|
||||
try {
|
||||
return (db.prepare("SELECT payload FROM documents ORDER BY source_slug").all() as { payload: string }[])
|
||||
.map((row) => JSON.parse(row.payload) as Doc);
|
||||
} finally { db.close(); }
|
||||
}
|
||||
|
||||
export async function readDoc(slug: string): Promise<Doc> {
|
||||
if (!validSlug.test(slug)) throw new Error("Invalid document slug");
|
||||
const sourceSlug = Object.entries(catalog.documents).find(([, metadata]) => metadata.slug === slug)?.[0] ?? slug;
|
||||
const content = await fs.readFile(path.join(directory, `${sourceSlug}.md`), "utf8");
|
||||
const title = content.match(/^# (.+)$/m)?.[1]?.trim() ?? slug;
|
||||
const collection = catalog.collections.find((c) => !c.path || sourceSlug.startsWith(`${c.path}/`))!;
|
||||
const metadata: Metadata = (catalog.documents as Record<string, Metadata>)[sourceSlug] ?? {};
|
||||
if (!metadata.slug || !metadata.category || !validSlug.test(metadata.slug) || metadata.slug.split("/").some((part) => /^\d+-/.test(part))) {
|
||||
throw new Error(`Register a semantic slug and category in content/catalog.json: ${sourceSlug}`);
|
||||
}
|
||||
return { slug: metadata.slug, sourceSlug, category: metadata.category, title, content, searchText: content,
|
||||
group: metadata.group ?? collection.title, collection: metadata.collection ?? collection.title,
|
||||
description: metadata.description ?? collection.description,
|
||||
draft: metadata.draft ?? false, featured: metadata.featured ?? false,
|
||||
};
|
||||
const doc = readDocument(slug);
|
||||
if (!doc) throw Error("Document not found");
|
||||
return doc;
|
||||
}
|
||||
|
||||
export async function listDocs(): Promise<DocLink[]> {
|
||||
async function walk(folder: string): Promise<string[]> {
|
||||
const entries = await fs.readdir(path.join(directory, folder), { withFileTypes: true });
|
||||
const files = await Promise.all(entries.map(async (entry) => {
|
||||
const relative = folder ? `${folder}/${entry.name}` : entry.name;
|
||||
if (entry.isDirectory()) return walk(relative);
|
||||
if (entry.isFile() && relative.endsWith(".md") && validSlug.test(relative.slice(0, -3)))
|
||||
return [relative.slice(0, -3)];
|
||||
return [];
|
||||
}));
|
||||
return files.flat();
|
||||
}
|
||||
const docs = await Promise.all((await walk("")).sort().map(async (slug) => {
|
||||
const { content: _, searchText: __, ...doc } = await readDoc(slug);
|
||||
void _; void __;
|
||||
return doc;
|
||||
}));
|
||||
if (new Set(docs.map((doc) => doc.slug)).size !== docs.length) throw new Error("Duplicate public document slug");
|
||||
return docs;
|
||||
return allDocuments().map(({ content: _, ...doc }) => { void _; return doc; });
|
||||
}
|
||||
|
||||
// Explicit, atomic bootstrap only. Runtime reads never import repository data.
|
||||
export function importDocuments(bundle: DocumentExport, onlyIfEmpty = false): number {
|
||||
if (bundle?.format !== "tabularium-documents-v1" || !Array.isArray(bundle.documents) || !bundle.documents.length)
|
||||
throw Error("Invalid or empty document export");
|
||||
bundle.documents.forEach(validateDocument);
|
||||
const digest = createHash("sha256").update(JSON.stringify(bundle)).digest("hex");
|
||||
const db = database();
|
||||
try {
|
||||
db.exec("BEGIN IMMEDIATE");
|
||||
try {
|
||||
const imported = db.prepare("SELECT digest FROM document_imports WHERE digest=?").get(digest);
|
||||
if (imported) { db.exec("COMMIT"); return 0; }
|
||||
const count = (db.prepare("SELECT COUNT(*) AS n FROM documents").get() as { n: number }).n;
|
||||
if (count) {
|
||||
if (!onlyIfEmpty) throw Error("Bootstrap requires an empty document database");
|
||||
db.exec("COMMIT"); return 0;
|
||||
}
|
||||
for (const doc of bundle.documents) write(db, doc, 0);
|
||||
db.prepare("INSERT INTO document_imports VALUES (?,?,?)").run(digest, bundle.documents.length, new Date().toISOString());
|
||||
db.exec("COMMIT"); return bundle.documents.length;
|
||||
} catch (error) { db.exec("ROLLBACK"); throw error; }
|
||||
} finally { db.close(); }
|
||||
}
|
||||
|
||||
export function documentHistory(sourceSlug: string): Doc[] {
|
||||
const db = database();
|
||||
try {
|
||||
return (db.prepare(`SELECT r.payload FROM document_revisions r JOIN documents d
|
||||
ON r.document_id=d.id WHERE d.source_slug=? ORDER BY r.revision`).all(sourceSlug) as { payload: string }[])
|
||||
.map((row) => JSON.parse(row.payload) as Doc);
|
||||
} finally { db.close(); }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { NextApiRequest, NextApiResponse } from "next";
|
||||
import { readDocument } from "@/lib/docs";
|
||||
|
||||
// The proxy protects Markdown just like the rendered document and search index.
|
||||
export default function handler(request: NextApiRequest, response: NextApiResponse) {
|
||||
response.setHeader("Cache-Control", "private, no-store");
|
||||
if (request.method !== "GET") {
|
||||
response.setHeader("Allow", "GET");
|
||||
return response.status(405).end();
|
||||
}
|
||||
const slug = request.query.slug;
|
||||
if (typeof slug !== "string") return response.status(400).json({ error: "缺少文档地址" });
|
||||
try {
|
||||
const doc = readDocument(slug);
|
||||
if (!doc) return response.status(404).end();
|
||||
response.setHeader("Content-Type", "text/plain; charset=utf-8");
|
||||
response.setHeader("X-Content-Type-Options", "nosniff");
|
||||
return response.status(200).send(doc.content);
|
||||
} catch {
|
||||
return response.status(503).json({ error: "文档数据库暂时不可用" });
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import type { NextApiRequest, NextApiResponse } from "next";
|
||||
import { listDocs, readDoc } from "@/lib/docs";
|
||||
import { allDocuments } from "@/lib/docs";
|
||||
|
||||
// The proxy authenticates this endpoint, including index content.
|
||||
export default async function handler(request: NextApiRequest, response: NextApiResponse) {
|
||||
@@ -8,9 +8,10 @@ export default async function handler(request: NextApiRequest, response: NextApi
|
||||
response.setHeader("Allow", "GET");
|
||||
return response.status(405).end();
|
||||
}
|
||||
const docs = await listDocs();
|
||||
const index = await Promise.all(docs.map(async (doc) => ({
|
||||
...doc, searchText: (await readDoc(doc.slug)).content,
|
||||
})));
|
||||
response.status(200).json(index);
|
||||
try {
|
||||
const index = allDocuments().map(({ content, ...doc }) => ({ ...doc, searchText: content }));
|
||||
return response.status(200).json(index);
|
||||
} catch {
|
||||
return response.status(503).json({ error: "文档数据库暂时不可用" });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import type {
|
||||
GetStaticPaths,
|
||||
GetStaticProps,
|
||||
InferGetStaticPropsType,
|
||||
GetServerSideProps,
|
||||
InferGetServerSidePropsType,
|
||||
} from "next";
|
||||
import Head from "next/head";
|
||||
import dynamic from "next/dynamic";
|
||||
@@ -15,7 +14,7 @@ import {
|
||||
} from "react";
|
||||
import ReactMarkdown from "react-markdown";
|
||||
import remarkGfm from "remark-gfm";
|
||||
import { listDocs, readDoc, type Doc, type DocLink } from "@/lib/docs";
|
||||
import { listDocs, readDocument, type Doc, type DocLink } from "@/lib/docs";
|
||||
import { Shell, Navigation } from "@/components/Shell";
|
||||
import { ArchitectureFigure } from "@/components/ArchitectureFigure";
|
||||
import { FigureGallery } from "@/components/FigureGallery";
|
||||
@@ -26,16 +25,17 @@ import { headingsFor } from "@/lib/navigation";
|
||||
|
||||
type Props = { doc: Doc; docs: DocLink[] };
|
||||
const AICompanyResearch = dynamic(() => import("@/components/AICompanyResearch"));
|
||||
export const getStaticPaths: GetStaticPaths = async () => ({
|
||||
paths: (await listDocs()).map(({ slug }) => ({ params: { slug: slug.split("/") } })),
|
||||
fallback: false,
|
||||
});
|
||||
export const getStaticProps: GetStaticProps<Props> = async ({ params }) => {
|
||||
export const getServerSideProps: GetServerSideProps<Props> = async ({ params, res, resolvedUrl }) => {
|
||||
res.setHeader("Cache-Control", "private, no-store");
|
||||
const slug = Array.isArray(params?.slug) ? params.slug.join("/") : params?.slug;
|
||||
if (typeof slug !== "string") return { notFound: true };
|
||||
const docs = await listDocs();
|
||||
if (!docs.some((d) => d.slug === slug)) return { notFound: true };
|
||||
return { props: { doc: await readDoc(slug), docs } };
|
||||
const doc = readDocument(slug);
|
||||
if (!doc) return { notFound: true };
|
||||
if (slug !== doc.slug) {
|
||||
const query = resolvedUrl.includes("?") ? resolvedUrl.slice(resolvedUrl.indexOf("?")) : "";
|
||||
return { redirect: { destination: `/docs/${doc.slug}${query}`, permanent: true } };
|
||||
}
|
||||
return { props: { doc, docs: await listDocs() } };
|
||||
};
|
||||
function textOf(value: ReactNode): string {
|
||||
return Children.toArray(value)
|
||||
@@ -49,7 +49,7 @@ function textOf(value: ReactNode): string {
|
||||
export default function DocPage({
|
||||
doc,
|
||||
docs,
|
||||
}: InferGetStaticPropsType<typeof getStaticProps>) {
|
||||
}: InferGetServerSidePropsType<typeof getServerSideProps>) {
|
||||
const headings = headingsFor(doc.content);
|
||||
const [active, setActive] = useState("");
|
||||
|
||||
@@ -231,9 +231,9 @@ export default function DocPage({
|
||||
<div className="article-end">
|
||||
|
||||
<a
|
||||
href={`https://gitea.entroia.com/Zayden-Jung/endorphin-tabularium-docs/src/branch/main/content/docs/${doc.sourceSlug}.md`}
|
||||
href={`/api/documents/source?slug=${encodeURIComponent(doc.slug)}`}
|
||||
>
|
||||
查看文档源码 ↗
|
||||
查看 Markdown 原文 ↗
|
||||
</a>
|
||||
</div>
|
||||
<nav className="pagination" aria-label="相邻文档">
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import type { GetStaticProps, InferGetStaticPropsType } from "next";
|
||||
import type { GetServerSideProps, InferGetServerSidePropsType } from "next";
|
||||
import Head from "next/head";
|
||||
import Link from "next/link";
|
||||
import { useState } from "react";
|
||||
@@ -7,11 +7,12 @@ import { Shell, Navigation } from "@/components/Shell";
|
||||
import { Icon } from "@/components/Icon";
|
||||
import { collectionEntries, collectionIcon, collectionLabel, collectionTone } from "@/lib/collections";
|
||||
|
||||
export const getStaticProps: GetStaticProps<{ docs: DocLink[] }> = async () => ({
|
||||
props: { docs: await listDocs() },
|
||||
});
|
||||
export const getServerSideProps: GetServerSideProps<{ docs: DocLink[] }> = async ({ res }) => {
|
||||
res.setHeader("Cache-Control", "private, no-store");
|
||||
return { props: { docs: await listDocs() } };
|
||||
};
|
||||
const PAGE_SIZE = 8;
|
||||
export default function Home({ docs }: InferGetStaticPropsType<typeof getStaticProps>) {
|
||||
export default function Home({ docs }: InferGetServerSidePropsType<typeof getServerSideProps>) {
|
||||
const [collection, setCollection] = useState("");
|
||||
const [query, setQuery] = useState("");
|
||||
const [page, setPage] = useState(0);
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
import { NextResponse, type NextRequest } from "next/server";
|
||||
import { authConfig, SESSION_COOKIE, validSession } from "./lib/auth";
|
||||
import catalog from "../../../content/catalog.json";
|
||||
|
||||
const legacyRoutes = new Map(Object.entries(catalog.documents).map(([source, metadata]) => [
|
||||
`/docs/${source}`, `/docs/${metadata.slug}`,
|
||||
]));
|
||||
|
||||
export function proxy(request: NextRequest) {
|
||||
const path = request.nextUrl.pathname;
|
||||
const headers = {
|
||||
@@ -54,13 +48,6 @@ export function proxy(request: NextRequest) {
|
||||
);
|
||||
}
|
||||
}
|
||||
// Keep legacy URLs behind authentication and out of static generation.
|
||||
const destination = legacyRoutes.get(path);
|
||||
if (destination && ["GET", "HEAD"].includes(request.method)) {
|
||||
const target = request.nextUrl.clone();
|
||||
target.pathname = destination;
|
||||
return NextResponse.redirect(target, { status: 308, headers });
|
||||
}
|
||||
return NextResponse.next({ headers });
|
||||
}
|
||||
if (path.startsWith("/api/") || !["GET", "HEAD"].includes(request.method)) {
|
||||
|
||||
Reference in New Issue
Block a user