docs: annotate Foundry source and deployment lifecycle in Chinese [skip ci]

This commit is contained in:
2026-11-16 17:52:11 +08:00
parent 30829eeb86
commit 1090591503
123 changed files with 1279 additions and 3 deletions
+4
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env bash
# Explicit remote-runner opt-in: invokes real models using the dedicated CLI login.
# 此脚本会编译测试并实际调用模型;只用于明确授权的专用远程 runner,普通检查不自动运行。
set -euo pipefail
umask 077
binary=$(realpath "$1")
@@ -9,6 +10,7 @@ login=${FOUNDRY_REAL_CODEX_HOME:-/opt/endoleap-infra/foundry/codex/home}
test -x "$binary"
test -s "$login/auth.json"
test -s "$login/models_cache.json"
# 随机命名 worker/数据库和临时 socket 目录,退出时一并清理,不复用生产队列。
suffix=$(openssl rand -hex 8)
database=foundry-real-db-$suffix
worker=foundry-real-worker-$suffix
@@ -24,6 +26,7 @@ docker exec "$database" psql -X -U postgres -v ON_ERROR_STOP=1 -c 'CREATE ROLE f
python3 deploy/migrate.py --container "$database" --database postgres --role foundry_fixture
# Only the CLI container sees the login. Core/tests receive the private socket,
# never the login, runtime configuration or Docker socket.
# 只有 CLI 容器只读读取专用登录和模型缓存,测试容器仅挂 socket,不能访问登录或 Docker socket。
docker run -d --name "$worker" --init --network bridge --user 1000:1000 \
--cap-drop ALL --security-opt no-new-privileges --read-only --pids-limit 128 --memory 1g \
--tmpfs /tmp:rw,nosuid,nodev,size=128m --tmpfs /work:ro,nosuid,nodev \
@@ -38,6 +41,7 @@ docker run -d --name "$worker" --init --network bridge --user 1000:1000 \
node:24-bookworm-slim /opt/foundry codex-worker >/dev/null
for attempt in {1..30}; do if [[ -S "$scratch/run/worker.sock" ]]; then break; fi; sleep 1; done
test -S "$scratch/run/worker.sock"
# 先等私有 socket 出现,再在 runner 编译并仅运行真实 worker 专项用例;与普通单元测试开关分离。
CGO_ENABLED=0 go test -mod=readonly -c -o bin/platform-real-tests ./internal/platform
docker run --rm --network "container:$database" --user 1000:1000 \
--read-only --cap-drop ALL --security-opt no-new-privileges \
+7
View File
@@ -1,6 +1,8 @@
#!/usr/bin/env bash
# 远程 runner 的数据库/集成检查,会编译测试二进制并运行容器;不能在未授权本地环境执行。
set -euo pipefail
# 随机隔离数据库容器不映射宿主端口,EXIT 时销毁;trust 认证只用于该临时网络命名空间。
container=foundry-schema-$(openssl rand -hex 8)
cleanup() { docker rm -f "$container" >/dev/null 2>&1 || true; }
trap cleanup EXIT
@@ -12,6 +14,7 @@ for attempt in {1..30}; do
sleep 1
done
psql() { docker exec -i "$container" psql -X -U postgres -v ON_ERROR_STOP=1; }
# 先安装最早认证版本并植入有效/过期旧会话,模拟部署前已有用户数据。
psql < db/001-auth.sql
psql <<'SQL'
INSERT INTO project_accounts(project,id,google_subject,email)
@@ -23,8 +26,10 @@ SQL
psql <<'SQLROLE'
CREATE ROLE foundry_fixture LOGIN;
SQLROLE
# 连续跑两次真实迁移器,验证旧基线接管和校验和重放,而不是仅验证全新数据库建表。
python3 deploy/migrate.py --container "$container" --database postgres --role foundry_fixture
python3 deploy/migrate.py --container "$container" --database postgres --role foundry_fixture
# SQL 断言旧 Google 绑定保留、过期会话不复活、绝对期限添加,再在回滚事务中验证资源与跨项目外键。
psql <<'SQL'
DO $$
BEGIN
@@ -68,6 +73,7 @@ ROLLBACK;
SQL
printf 'Migration preserved existing logins and enforced project isolation.\n'
psql < internal/research/testdata/corpus.sql
# runner 编译 Linux 静态测试程序,随后在同数据库网络空间用最小权限容器运行,源码不挂入测试容器。
for package in platform auth research; do
CGO_ENABLED=0 go test -mod=readonly -c -o "bin/$package-tests" "./internal/$package"
docker run --rm --network "container:$container" --user 1000:1000 \
@@ -81,6 +87,7 @@ done
# Run the actual pinned Endoleap runtime against this build's authenticated HTTP
# routes and the same disposable database. Only this remote container needs Node.
# 可选 pinned Endoleap 客户端需要 Node,额外运行一次真实 HTTP 客户端集成;未提供时跳过该阶段。
if [[ -n "${HEADLESS_CLIENT_SOURCE:-}" ]]; then
test -f "$HEADLESS_CLIENT_SOURCE/tests/headless/foundry.integration.mjs"
docker run --rm --network "container:$container" --user 1000:1000 \
+5
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# 远程私有 worker 启动器;使用与 Core 同一二进制的 codex-worker 模式,CLI 和登录目录由运维预先提供。
set -euo pipefail
base=/opt/endoleap-infra
@@ -7,14 +8,18 @@ binary=$(realpath "$1")
test -x "$binary"
test -s "$dir/home/auth.json"
test -s "$dir/grants/projects.json"
# socket 目录归容器 uid 1000,socket 本身由服务设为 0600;不公布宿主 TCP 端口。
install -d -m 700 "$dir/run"
chown 1000:1000 "$dir/run"
# 可选目录声明已验证图片输出后端;没有此名单时不根据图片输入能力推测图片生成能力。
canvas_env=()
if [[ -s "$dir/canvas.env" ]]; then canvas_env=(--env-file "$dir/canvas.env"); fi
# The host operator supplies an isolated Codex home and starts a single worker.
# The Docker default bridge supplies egress independently of Foundry's network.
# 单 worker 使用独立 bridge 出网,根文件系统只读;临时任务写 /tmp,登录目录需允许 CLI 自身状态更新。
# CLI/二进制只读挂载,Core 只看到 socket,不共享本登录目录。
docker run -d --name foundry-codex-worker --restart unless-stopped --init \
--network bridge --user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \
--read-only --pids-limit 128 --memory 1g --tmpfs /tmp:rw,nosuid,nodev \
+8
View File
@@ -1,5 +1,7 @@
#!/usr/bin/env python3
"""Apply checked migrations atomically from the trusted deployment runner."""
# 部署 runner 入口:组合所有尚未执行的迁移及授权为单个外层事务,失败整体回滚。
# db/*.sql 的校验和覆盖原始注释;历史文件不可仅为加注释而改写,中文对象说明见 docs/database-comments.sql。
import argparse
import hashlib
import os
@@ -7,6 +9,7 @@ from pathlib import Path
import re
import subprocess
# 数据库/角色会插入 SQL 标识符,先严格限制为小写标识;container 通过 argv 传给 docker。
parser = argparse.ArgumentParser()
parser.add_argument("--container", default=os.environ.get("FOUNDRY_POSTGRES_CONTAINER", "foundry-postgres"))
parser.add_argument("--database", default="foundry_auth")
@@ -19,6 +22,7 @@ migrations = sorted((Path(__file__).resolve().parent.parent / "db").glob("[0-9]*
if not migrations:
raise SystemExit("No migrations found")
# psql 遇错立即退出,固定 advisory 事务锁串行化同库部署,lock_timeout 防止无限等锁。
script = [
"\\set ON_ERROR_STOP on",
"BEGIN;",
@@ -33,6 +37,7 @@ for index, path in enumerate(migrations):
name = path.name
if not re.fullmatch(r"[0-9]+-[a-z0-9-]+\.sql", name):
raise SystemExit("Invalid migration name")
# 原文 SHA-256 与已执行记录逐字匹配,禁止同名历史迁移在部署后被静默修改。
checksum = hashlib.sha256(source.encode()).hexdigest()
tables.update(re.findall(r"CREATE TABLE ([a-z_]+)", source))
# The first deployed version predates migration tracking.
@@ -55,6 +60,7 @@ for index, path in enumerate(migrations):
"\\endif",
"\\else",
])
# 每份源文件显式 BEGIN/COMMIT;去除内层边界后放入外层统一事务,而非逐文件部分提交。
source = source.strip()
if not source.startswith("BEGIN;") or not source.endswith("COMMIT;"):
raise SystemExit("Migration must have explicit transaction boundaries")
@@ -63,12 +69,14 @@ for index, path in enumerate(migrations):
f"INSERT INTO foundry_schema_migrations(name,checksum) VALUES('{name}','{checksum}');",
"\\endif",
])
# 从受信任源码的 CREATE TABLE 清单补应用 DML 权限;identity 序列另授予使用/读取权限。
for table in sorted(tables):
script.append(f'GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE "{table}" TO "{args.role}";')
script.extend([
f'GRANT USAGE,SELECT ON SEQUENCE link_clicks_id_seq TO "{args.role}";',
"COMMIT;",
])
# SQL 经标准输入交给容器内 psql,不把数据库密码或整段 SQL 拼成 shell 命令参数。
subprocess.run(
["docker", "exec", "-i", args.container, "sh", "-c",
'exec psql -X -U "${POSTGRES_USER:-postgres}" -d "$1" -v ON_ERROR_STOP=1', "sh", args.database],
+13
View File
@@ -1,4 +1,6 @@
#!/usr/bin/env bash
# 由 main 部署工作流在远程基础设施主机执行,参数一必须是已构建的 Foundry 二进制。
# 本脚本会迁移数据库、替换 worker 并启动候选服务,不能当作本地静态检查执行。
set -euo pipefail
umask 077
@@ -8,6 +10,7 @@ origin=https://api.foundry.entroia.com
candidate=foundry-core-candidate
network=endoleap-dev_default
# 必需凭据仅从环境传入,拒绝换行以保持容器环境参数边界,日志不打印值。
for name in GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET; do
value=${!name-}
if [[ -z "$value" || "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then
@@ -16,6 +19,7 @@ for name in GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET; do
fi
done
# Accept formatted JSON from Actions Variables and never print its contents.
# 将 Actions 中可多行排版的 JSON 压为单行;这里只校验映射形状,URL 协议由服务启动继续验证。
FOUNDRY_PROJECTS=$(python3 - <<'PYCONFIG'
import json
import os
@@ -44,11 +48,14 @@ test -s "$foundry/database.env"
ca_bundle=/etc/ssl/certs/ca-certificates.crt
test -s "$ca_bundle"
docker network inspect "$network" >/dev/null
# 先运行全事务迁移及角色授权,迁移失败立即终止,不让新服务在半升级 schema 上启动。
python3 "$(dirname "$0")/migrate.py"
# 每次发布使用独立二进制目录供只读挂载;本脚本不清除旧 release 目录。
release=$(mktemp -d "$foundry/release.XXXXXXXX")
install -m 755 "$1" "$release/foundry"
export HOST=0.0.0.0 PORT=8080 PUBLIC_ORIGIN="$origin"
# 邮件/研究配置可选且从受保护文件加载;Slack 优先 Actions 环境,缺省才回退服务器文件。
mail_env=()
if [[ -s "$foundry/mail.env" ]]; then
mail_env=(--env-file "$foundry/mail.env")
@@ -73,6 +80,7 @@ elif [[ -s "$foundry/slack.env" ]]; then
fi
codex_mounts=()
codex_dir=$foundry/codex
# 存在 worker 配置才启用 IPC;worker 必须先与当前 Core 二进制协议对齐,启动失败恢复旧 worker。
if [[ -d "$codex_dir" ]]; then
# Core and its private worker must speak the same Canvas artifact protocol.
# Keep the previous container definition for exact startup rollback.
@@ -107,6 +115,7 @@ if [[ -d "$codex_dir" ]]; then
printf 'Canvas worker startup/catalog verification failed; previous worker restored.\n' >&2
exit 1
fi
# 目录探测成功后删除旧 worker 定义;后续候选/路由失败不会再恢复该 worker,回滚范围需据此理解。
if [[ -n "$worker_backup" ]]; then docker rm "$worker_backup" >/dev/null; fi
for attempt in {1..20}; do
[[ ! -S "$codex_dir/run/worker.sock" ]] || break
@@ -122,6 +131,7 @@ if [[ -d "$codex_dir" ]]; then
)
fi
# Core 只挂载 IPC/授权表,不挂载 CLI 登录;以非 root、只读根文件系统和裁剪能力运行。
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" "${research_env[@]}" --network "$network" \
--env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \
-e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \
@@ -132,11 +142,13 @@ docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack
--mount "type=bind,source=$release/foundry,target=/opt/foundry,readonly" \
node:24-bookworm-slim /opt/foundry >/dev/null
# 候选或路由失败时删除本次新 Core;旧 Core 由 route 成功探测后才停止。
cleanup_candidate() {
docker rm -f "${live:-$candidate}" >/dev/null 2>&1 || true
}
trap cleanup_candidate EXIT
# 容器内 smoke 只验证 CA 文件、数据库就绪及匿名请求被拒,不执行真实 OAuth 或付费模型。
docker exec "$candidate" node -e '
const assert = require("node:assert/strict");
const fs = require("node:fs");
@@ -166,6 +178,7 @@ docker exec "$candidate" node -e '
# Keep the previous running container until the new candidate has passed checks.
# Traefik reads the route atomically from its watched config directory.
# 使用容器 ID 前缀命名新 live,route.sh 负责原子路由写入、外部就绪检查和失败恢复路由。
id=$(docker inspect -f '{{.Id}}' "$candidate")
live=foundry-core-${id:0:12}
docker rename "$candidate" "$live"
+4
View File
@@ -1,3 +1,5 @@
-- 运维手动索引步骤:对已发布 research schema 执行,不能由只读服务角色调用,也不能包进事务。
-- 不属于 db/ 版本迁移;CREATE INDEX CONCURRENTLY 需要独立顶层语句,避免长时间阻塞语料读取。
-- Manual operator step against the published Paper Atlas schema, outside a
-- transaction. Core's SELECT-only runtime role must never execute this file.
-- These indexes support the actual author search and exact category filter;
@@ -6,8 +8,10 @@ SET lock_timeout = '2s';
SET statement_timeout = '20min';
SET maintenance_work_mem = '32MB';
SET max_parallel_maintenance_workers = 0;
-- (tag_id,paper_id) 支持纯主题页直接走关联索引,在 LIMIT 前避免读所有宽论文行。
CREATE INDEX CONCURRENTLY IF NOT EXISTS paper_tags_tag_paper
ON research.paper_tags(tag_id,paper_id);
-- simple 配置匹配作者查询的表达式,不对姓名做 english 词干化;索引表达式须与查询一致。
CREATE INDEX CONCURRENTLY IF NOT EXISTS papers_authors_search_gin
ON research.papers USING gin(to_tsvector('simple'::regconfig,authors_json));
-- After an interrupted concurrent build, inspect pg_index.indisvalid before
+7
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env bash
# 远程部署的路由切换阶段,参数必须是本次已通过内部就绪检查的 Core 容器名。
set -euo pipefail
umask 077
cd /opt/endoleap-infra
@@ -8,12 +9,14 @@ live=$1
install -d -m 700 dev/foundry-edge
edge=dev/foundry-edge
route=$edge/foundry.yml
# 从现有动态配置提取旧容器;仅在新路由完成 HTTPS 探测后才停止它。
previous=
if [[ -f "$route" ]]; then
previous=$(sed -n 's@.*url: http://\(foundry-core-[a-f0-9]\{12\}\):8080.*@\1@p' "$route")
fi
backup=$(mktemp)
if [[ -f "$route" ]]; then cp "$route" "$backup"; fi
# EXIT trap 只恢复动态路由文件或删除新路由,不回滚已执行的数据库迁移/worker/Compose 配置。
rollback() {
if [[ -s "$backup" ]]; then
cp "$backup" "$edge/.restore"
@@ -25,6 +28,7 @@ rollback() {
}
trap rollback EXIT
# ACME 文件保存证书状态,限制权限;动态路由先写隐藏临时文件再 mv,避免 watcher 读到半文件。
touch "$edge/acme.json"
chmod 600 "$edge/acme.json"
cat > "$edge/.foundry.yml" <<YAML
@@ -49,6 +53,7 @@ http:
- url: http://$live:8080
YAML
mv "$edge/.foundry.yml" "$route"
# Compose 覆盖文件保持 Traefik 入口、证书与目录 provider 配置,不重建其他服务。
cat > foundry-edge.yml <<'YAML'
services:
traefik:
@@ -68,11 +73,13 @@ services:
YAML
docker compose -p endoleap-dev --env-file dev/env -f compose.yml -f dev.yml -f foundry-edge.yml up -d --no-deps traefik
# --resolve 在本机回环命中 Traefik,同时保留公开域名 Host/TLS 验证,探测真正的 HTTPS 路由。
curl --fail --silent --show-error --output /dev/null --max-time 6 \
--retry 15 --retry-delay 4 --retry-all-errors \
--resolve api.foundry.entroia.com:443:127.0.0.1 \
https://api.foundry.entroia.com/readyz
# HTTPS 就绪后解除回滚并删除备份;旧容器停止失败仅提示手工清理,不能误报新路由失败。
trap - EXIT
rm -f "$backup"
if [[ "$previous" =~ ^foundry-core-[a-f0-9]{12}$ && "$previous" != "$live" ]]; then