docs: annotate Foundry source and deployment lifecycle in Chinese [skip ci]
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# Explicit remote-runner opt-in: invokes real models using the dedicated CLI login.
|
||||
# 此脚本会编译测试并实际调用模型;只用于明确授权的专用远程 runner,普通检查不自动运行。
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
binary=$(realpath "$1")
|
||||
@@ -9,6 +10,7 @@ login=${FOUNDRY_REAL_CODEX_HOME:-/opt/endoleap-infra/foundry/codex/home}
|
||||
test -x "$binary"
|
||||
test -s "$login/auth.json"
|
||||
test -s "$login/models_cache.json"
|
||||
# 随机命名 worker/数据库和临时 socket 目录,退出时一并清理,不复用生产队列。
|
||||
suffix=$(openssl rand -hex 8)
|
||||
database=foundry-real-db-$suffix
|
||||
worker=foundry-real-worker-$suffix
|
||||
@@ -24,6 +26,7 @@ docker exec "$database" psql -X -U postgres -v ON_ERROR_STOP=1 -c 'CREATE ROLE f
|
||||
python3 deploy/migrate.py --container "$database" --database postgres --role foundry_fixture
|
||||
# Only the CLI container sees the login. Core/tests receive the private socket,
|
||||
# never the login, runtime configuration or Docker socket.
|
||||
# 只有 CLI 容器只读读取专用登录和模型缓存,测试容器仅挂 socket,不能访问登录或 Docker socket。
|
||||
docker run -d --name "$worker" --init --network bridge --user 1000:1000 \
|
||||
--cap-drop ALL --security-opt no-new-privileges --read-only --pids-limit 128 --memory 1g \
|
||||
--tmpfs /tmp:rw,nosuid,nodev,size=128m --tmpfs /work:ro,nosuid,nodev \
|
||||
@@ -38,6 +41,7 @@ docker run -d --name "$worker" --init --network bridge --user 1000:1000 \
|
||||
node:24-bookworm-slim /opt/foundry codex-worker >/dev/null
|
||||
for attempt in {1..30}; do if [[ -S "$scratch/run/worker.sock" ]]; then break; fi; sleep 1; done
|
||||
test -S "$scratch/run/worker.sock"
|
||||
# 先等私有 socket 出现,再在 runner 编译并仅运行真实 worker 专项用例;与普通单元测试开关分离。
|
||||
CGO_ENABLED=0 go test -mod=readonly -c -o bin/platform-real-tests ./internal/platform
|
||||
docker run --rm --network "container:$database" --user 1000:1000 \
|
||||
--read-only --cap-drop ALL --security-opt no-new-privileges \
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
#!/usr/bin/env bash
|
||||
# 远程 runner 的数据库/集成检查,会编译测试二进制并运行容器;不能在未授权本地环境执行。
|
||||
set -euo pipefail
|
||||
|
||||
# 随机隔离数据库容器不映射宿主端口,EXIT 时销毁;trust 认证只用于该临时网络命名空间。
|
||||
container=foundry-schema-$(openssl rand -hex 8)
|
||||
cleanup() { docker rm -f "$container" >/dev/null 2>&1 || true; }
|
||||
trap cleanup EXIT
|
||||
@@ -12,6 +14,7 @@ for attempt in {1..30}; do
|
||||
sleep 1
|
||||
done
|
||||
psql() { docker exec -i "$container" psql -X -U postgres -v ON_ERROR_STOP=1; }
|
||||
# 先安装最早认证版本并植入有效/过期旧会话,模拟部署前已有用户数据。
|
||||
psql < db/001-auth.sql
|
||||
psql <<'SQL'
|
||||
INSERT INTO project_accounts(project,id,google_subject,email)
|
||||
@@ -23,8 +26,10 @@ SQL
|
||||
psql <<'SQLROLE'
|
||||
CREATE ROLE foundry_fixture LOGIN;
|
||||
SQLROLE
|
||||
# 连续跑两次真实迁移器,验证旧基线接管和校验和重放,而不是仅验证全新数据库建表。
|
||||
python3 deploy/migrate.py --container "$container" --database postgres --role foundry_fixture
|
||||
python3 deploy/migrate.py --container "$container" --database postgres --role foundry_fixture
|
||||
# SQL 断言旧 Google 绑定保留、过期会话不复活、绝对期限添加,再在回滚事务中验证资源与跨项目外键。
|
||||
psql <<'SQL'
|
||||
DO $$
|
||||
BEGIN
|
||||
@@ -68,6 +73,7 @@ ROLLBACK;
|
||||
SQL
|
||||
printf 'Migration preserved existing logins and enforced project isolation.\n'
|
||||
psql < internal/research/testdata/corpus.sql
|
||||
# runner 编译 Linux 静态测试程序,随后在同数据库网络空间用最小权限容器运行,源码不挂入测试容器。
|
||||
for package in platform auth research; do
|
||||
CGO_ENABLED=0 go test -mod=readonly -c -o "bin/$package-tests" "./internal/$package"
|
||||
docker run --rm --network "container:$container" --user 1000:1000 \
|
||||
@@ -81,6 +87,7 @@ done
|
||||
|
||||
# Run the actual pinned Endoleap runtime against this build's authenticated HTTP
|
||||
# routes and the same disposable database. Only this remote container needs Node.
|
||||
# 可选 pinned Endoleap 客户端需要 Node,额外运行一次真实 HTTP 客户端集成;未提供时跳过该阶段。
|
||||
if [[ -n "${HEADLESS_CLIENT_SOURCE:-}" ]]; then
|
||||
test -f "$HEADLESS_CLIENT_SOURCE/tests/headless/foundry.integration.mjs"
|
||||
docker run --rm --network "container:$container" --user 1000:1000 \
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# 远程私有 worker 启动器;使用与 Core 同一二进制的 codex-worker 模式,CLI 和登录目录由运维预先提供。
|
||||
set -euo pipefail
|
||||
|
||||
base=/opt/endoleap-infra
|
||||
@@ -7,14 +8,18 @@ binary=$(realpath "$1")
|
||||
test -x "$binary"
|
||||
test -s "$dir/home/auth.json"
|
||||
test -s "$dir/grants/projects.json"
|
||||
# socket 目录归容器 uid 1000,socket 本身由服务设为 0600;不公布宿主 TCP 端口。
|
||||
install -d -m 700 "$dir/run"
|
||||
chown 1000:1000 "$dir/run"
|
||||
|
||||
# 可选目录声明已验证图片输出后端;没有此名单时不根据图片输入能力推测图片生成能力。
|
||||
canvas_env=()
|
||||
if [[ -s "$dir/canvas.env" ]]; then canvas_env=(--env-file "$dir/canvas.env"); fi
|
||||
|
||||
# The host operator supplies an isolated Codex home and starts a single worker.
|
||||
# The Docker default bridge supplies egress independently of Foundry's network.
|
||||
# 单 worker 使用独立 bridge 出网,根文件系统只读;临时任务写 /tmp,登录目录需允许 CLI 自身状态更新。
|
||||
# CLI/二进制只读挂载,Core 只看到 socket,不共享本登录目录。
|
||||
docker run -d --name foundry-codex-worker --restart unless-stopped --init \
|
||||
--network bridge --user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \
|
||||
--read-only --pids-limit 128 --memory 1g --tmpfs /tmp:rw,nosuid,nodev \
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Apply checked migrations atomically from the trusted deployment runner."""
|
||||
# 部署 runner 入口:组合所有尚未执行的迁移及授权为单个外层事务,失败整体回滚。
|
||||
# db/*.sql 的校验和覆盖原始注释;历史文件不可仅为加注释而改写,中文对象说明见 docs/database-comments.sql。
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
@@ -7,6 +9,7 @@ from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
# 数据库/角色会插入 SQL 标识符,先严格限制为小写标识;container 通过 argv 传给 docker。
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--container", default=os.environ.get("FOUNDRY_POSTGRES_CONTAINER", "foundry-postgres"))
|
||||
parser.add_argument("--database", default="foundry_auth")
|
||||
@@ -19,6 +22,7 @@ migrations = sorted((Path(__file__).resolve().parent.parent / "db").glob("[0-9]*
|
||||
if not migrations:
|
||||
raise SystemExit("No migrations found")
|
||||
|
||||
# psql 遇错立即退出,固定 advisory 事务锁串行化同库部署,lock_timeout 防止无限等锁。
|
||||
script = [
|
||||
"\\set ON_ERROR_STOP on",
|
||||
"BEGIN;",
|
||||
@@ -33,6 +37,7 @@ for index, path in enumerate(migrations):
|
||||
name = path.name
|
||||
if not re.fullmatch(r"[0-9]+-[a-z0-9-]+\.sql", name):
|
||||
raise SystemExit("Invalid migration name")
|
||||
# 原文 SHA-256 与已执行记录逐字匹配,禁止同名历史迁移在部署后被静默修改。
|
||||
checksum = hashlib.sha256(source.encode()).hexdigest()
|
||||
tables.update(re.findall(r"CREATE TABLE ([a-z_]+)", source))
|
||||
# The first deployed version predates migration tracking.
|
||||
@@ -55,6 +60,7 @@ for index, path in enumerate(migrations):
|
||||
"\\endif",
|
||||
"\\else",
|
||||
])
|
||||
# 每份源文件显式 BEGIN/COMMIT;去除内层边界后放入外层统一事务,而非逐文件部分提交。
|
||||
source = source.strip()
|
||||
if not source.startswith("BEGIN;") or not source.endswith("COMMIT;"):
|
||||
raise SystemExit("Migration must have explicit transaction boundaries")
|
||||
@@ -63,12 +69,14 @@ for index, path in enumerate(migrations):
|
||||
f"INSERT INTO foundry_schema_migrations(name,checksum) VALUES('{name}','{checksum}');",
|
||||
"\\endif",
|
||||
])
|
||||
# 从受信任源码的 CREATE TABLE 清单补应用 DML 权限;identity 序列另授予使用/读取权限。
|
||||
for table in sorted(tables):
|
||||
script.append(f'GRANT SELECT,INSERT,UPDATE,DELETE ON TABLE "{table}" TO "{args.role}";')
|
||||
script.extend([
|
||||
f'GRANT USAGE,SELECT ON SEQUENCE link_clicks_id_seq TO "{args.role}";',
|
||||
"COMMIT;",
|
||||
])
|
||||
# SQL 经标准输入交给容器内 psql,不把数据库密码或整段 SQL 拼成 shell 命令参数。
|
||||
subprocess.run(
|
||||
["docker", "exec", "-i", args.container, "sh", "-c",
|
||||
'exec psql -X -U "${POSTGRES_USER:-postgres}" -d "$1" -v ON_ERROR_STOP=1', "sh", args.database],
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
# 由 main 部署工作流在远程基础设施主机执行,参数一必须是已构建的 Foundry 二进制。
|
||||
# 本脚本会迁移数据库、替换 worker 并启动候选服务,不能当作本地静态检查执行。
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
@@ -8,6 +10,7 @@ origin=https://api.foundry.entroia.com
|
||||
candidate=foundry-core-candidate
|
||||
network=endoleap-dev_default
|
||||
|
||||
# 必需凭据仅从环境传入,拒绝换行以保持容器环境参数边界,日志不打印值。
|
||||
for name in GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET; do
|
||||
value=${!name-}
|
||||
if [[ -z "$value" || "$value" == *$'\n'* || "$value" == *$'\r'* ]]; then
|
||||
@@ -16,6 +19,7 @@ for name in GOOGLE_CLIENT_ID GOOGLE_CLIENT_SECRET; do
|
||||
fi
|
||||
done
|
||||
# Accept formatted JSON from Actions Variables and never print its contents.
|
||||
# 将 Actions 中可多行排版的 JSON 压为单行;这里只校验映射形状,URL 协议由服务启动继续验证。
|
||||
FOUNDRY_PROJECTS=$(python3 - <<'PYCONFIG'
|
||||
import json
|
||||
import os
|
||||
@@ -44,11 +48,14 @@ test -s "$foundry/database.env"
|
||||
ca_bundle=/etc/ssl/certs/ca-certificates.crt
|
||||
test -s "$ca_bundle"
|
||||
docker network inspect "$network" >/dev/null
|
||||
# 先运行全事务迁移及角色授权,迁移失败立即终止,不让新服务在半升级 schema 上启动。
|
||||
python3 "$(dirname "$0")/migrate.py"
|
||||
|
||||
# 每次发布使用独立二进制目录供只读挂载;本脚本不清除旧 release 目录。
|
||||
release=$(mktemp -d "$foundry/release.XXXXXXXX")
|
||||
install -m 755 "$1" "$release/foundry"
|
||||
export HOST=0.0.0.0 PORT=8080 PUBLIC_ORIGIN="$origin"
|
||||
# 邮件/研究配置可选且从受保护文件加载;Slack 优先 Actions 环境,缺省才回退服务器文件。
|
||||
mail_env=()
|
||||
if [[ -s "$foundry/mail.env" ]]; then
|
||||
mail_env=(--env-file "$foundry/mail.env")
|
||||
@@ -73,6 +80,7 @@ elif [[ -s "$foundry/slack.env" ]]; then
|
||||
fi
|
||||
codex_mounts=()
|
||||
codex_dir=$foundry/codex
|
||||
# 存在 worker 配置才启用 IPC;worker 必须先与当前 Core 二进制协议对齐,启动失败恢复旧 worker。
|
||||
if [[ -d "$codex_dir" ]]; then
|
||||
# Core and its private worker must speak the same Canvas artifact protocol.
|
||||
# Keep the previous container definition for exact startup rollback.
|
||||
@@ -107,6 +115,7 @@ if [[ -d "$codex_dir" ]]; then
|
||||
printf 'Canvas worker startup/catalog verification failed; previous worker restored.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
# 目录探测成功后删除旧 worker 定义;后续候选/路由失败不会再恢复该 worker,回滚范围需据此理解。
|
||||
if [[ -n "$worker_backup" ]]; then docker rm "$worker_backup" >/dev/null; fi
|
||||
for attempt in {1..20}; do
|
||||
[[ ! -S "$codex_dir/run/worker.sock" ]] || break
|
||||
@@ -122,6 +131,7 @@ if [[ -d "$codex_dir" ]]; then
|
||||
)
|
||||
fi
|
||||
|
||||
# Core 只挂载 IPC/授权表,不挂载 CLI 登录;以非 root、只读根文件系统和裁剪能力运行。
|
||||
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" "${research_env[@]}" --network "$network" \
|
||||
--env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \
|
||||
-e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \
|
||||
@@ -132,11 +142,13 @@ docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack
|
||||
--mount "type=bind,source=$release/foundry,target=/opt/foundry,readonly" \
|
||||
node:24-bookworm-slim /opt/foundry >/dev/null
|
||||
|
||||
# 候选或路由失败时删除本次新 Core;旧 Core 由 route 成功探测后才停止。
|
||||
cleanup_candidate() {
|
||||
docker rm -f "${live:-$candidate}" >/dev/null 2>&1 || true
|
||||
}
|
||||
trap cleanup_candidate EXIT
|
||||
|
||||
# 容器内 smoke 只验证 CA 文件、数据库就绪及匿名请求被拒,不执行真实 OAuth 或付费模型。
|
||||
docker exec "$candidate" node -e '
|
||||
const assert = require("node:assert/strict");
|
||||
const fs = require("node:fs");
|
||||
@@ -166,6 +178,7 @@ docker exec "$candidate" node -e '
|
||||
|
||||
# Keep the previous running container until the new candidate has passed checks.
|
||||
# Traefik reads the route atomically from its watched config directory.
|
||||
# 使用容器 ID 前缀命名新 live,route.sh 负责原子路由写入、外部就绪检查和失败恢复路由。
|
||||
id=$(docker inspect -f '{{.Id}}' "$candidate")
|
||||
live=foundry-core-${id:0:12}
|
||||
docker rename "$candidate" "$live"
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
-- 运维手动索引步骤:对已发布 research schema 执行,不能由只读服务角色调用,也不能包进事务。
|
||||
-- 不属于 db/ 版本迁移;CREATE INDEX CONCURRENTLY 需要独立顶层语句,避免长时间阻塞语料读取。
|
||||
-- Manual operator step against the published Paper Atlas schema, outside a
|
||||
-- transaction. Core's SELECT-only runtime role must never execute this file.
|
||||
-- These indexes support the actual author search and exact category filter;
|
||||
@@ -6,8 +8,10 @@ SET lock_timeout = '2s';
|
||||
SET statement_timeout = '20min';
|
||||
SET maintenance_work_mem = '32MB';
|
||||
SET max_parallel_maintenance_workers = 0;
|
||||
-- (tag_id,paper_id) 支持纯主题页直接走关联索引,在 LIMIT 前避免读所有宽论文行。
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS paper_tags_tag_paper
|
||||
ON research.paper_tags(tag_id,paper_id);
|
||||
-- simple 配置匹配作者查询的表达式,不对姓名做 english 词干化;索引表达式须与查询一致。
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS papers_authors_search_gin
|
||||
ON research.papers USING gin(to_tsvector('simple'::regconfig,authors_json));
|
||||
-- After an interrupted concurrent build, inspect pg_index.indisvalid before
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
# 远程部署的路由切换阶段,参数必须是本次已通过内部就绪检查的 Core 容器名。
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
cd /opt/endoleap-infra
|
||||
@@ -8,12 +9,14 @@ live=$1
|
||||
install -d -m 700 dev/foundry-edge
|
||||
edge=dev/foundry-edge
|
||||
route=$edge/foundry.yml
|
||||
# 从现有动态配置提取旧容器;仅在新路由完成 HTTPS 探测后才停止它。
|
||||
previous=
|
||||
if [[ -f "$route" ]]; then
|
||||
previous=$(sed -n 's@.*url: http://\(foundry-core-[a-f0-9]\{12\}\):8080.*@\1@p' "$route")
|
||||
fi
|
||||
backup=$(mktemp)
|
||||
if [[ -f "$route" ]]; then cp "$route" "$backup"; fi
|
||||
# EXIT trap 只恢复动态路由文件或删除新路由,不回滚已执行的数据库迁移/worker/Compose 配置。
|
||||
rollback() {
|
||||
if [[ -s "$backup" ]]; then
|
||||
cp "$backup" "$edge/.restore"
|
||||
@@ -25,6 +28,7 @@ rollback() {
|
||||
}
|
||||
trap rollback EXIT
|
||||
|
||||
# ACME 文件保存证书状态,限制权限;动态路由先写隐藏临时文件再 mv,避免 watcher 读到半文件。
|
||||
touch "$edge/acme.json"
|
||||
chmod 600 "$edge/acme.json"
|
||||
cat > "$edge/.foundry.yml" <<YAML
|
||||
@@ -49,6 +53,7 @@ http:
|
||||
- url: http://$live:8080
|
||||
YAML
|
||||
mv "$edge/.foundry.yml" "$route"
|
||||
# Compose 覆盖文件保持 Traefik 入口、证书与目录 provider 配置,不重建其他服务。
|
||||
cat > foundry-edge.yml <<'YAML'
|
||||
services:
|
||||
traefik:
|
||||
@@ -68,11 +73,13 @@ services:
|
||||
YAML
|
||||
docker compose -p endoleap-dev --env-file dev/env -f compose.yml -f dev.yml -f foundry-edge.yml up -d --no-deps traefik
|
||||
|
||||
# --resolve 在本机回环命中 Traefik,同时保留公开域名 Host/TLS 验证,探测真正的 HTTPS 路由。
|
||||
curl --fail --silent --show-error --output /dev/null --max-time 6 \
|
||||
--retry 15 --retry-delay 4 --retry-all-errors \
|
||||
--resolve api.foundry.entroia.com:443:127.0.0.1 \
|
||||
https://api.foundry.entroia.com/readyz
|
||||
|
||||
# HTTPS 就绪后解除回滚并删除备份;旧容器停止失败仅提示手工清理,不能误报新路由失败。
|
||||
trap - EXIT
|
||||
rm -f "$backup"
|
||||
if [[ "$previous" =~ ^foundry-core-[a-f0-9]{12}$ && "$previous" != "$live" ]]; then
|
||||
|
||||
Reference in New Issue
Block a user