diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6e9151c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,12 @@ +.git +.next +node_modules +dist +coverage +.env +.env.* +!.env.example +*.tsbuildinfo +.DS_Store +.agents +.codex diff --git a/.env.example b/.env.example index f2392a5..93691be 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,11 @@ -DATABASE_URL=postgresql://prompt:replace-me@127.0.0.1:18132/prompt -POSTGRES_PASSWORD=replace-me -PG_PORT=18132 -S3_PORT=18133 -S3_ENDPOINT=http://127.0.0.1:18133 -S3_ACCESS_KEY=replace-me +# Replace all three placeholders before starting Docker Compose. +POSTGRES_PASSWORD=replace-with-a-long-random-password +S3_ACCESS_KEY=replace-with-a-unique-access-key S3_SECRET_KEY=replace-with-a-long-random-secret S3_BUCKET=prompt +WEB_BIND_ADDRESS=0.0.0.0 +WEB_PORT=18131 +PG_PORT=18132 +S3_PORT=18133 +PROMPT_IMAGE=prompt:local +RELEASE_SHA=development diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index f2fc319..7349e79 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,4 +1,4 @@ -name: PROMPT CI and deployment +name: PROMPT Docker CI and deployment on: push: @@ -9,42 +9,31 @@ on: jobs: verify-and-deploy: runs-on: ces-control - timeout-minutes: 35 + timeout-minutes: 45 + env: + PROMPT_IMAGE: prompt:${{ gitea.sha }} + PROMPT_TEST_IMAGE: prompt-tests:${{ gitea.sha }} + RELEASE_SHA: ${{ gitea.sha }} steps: - name: Checkout uses: actions/checkout@v4 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - name: Install dependencies - run: npm ci - - name: Type check - run: npm run typecheck - - name: Unit tests - run: npm test - - name: Production build - run: npm run build - - name: Test standalone production server - run: npm run test:smoke - - name: Prepare Docker and FFmpeg for isolated integration tests + - name: Prepare Docker Engine and Compose run: | set -eu if [ "$(id -u)" -eq 0 ]; then SUDO=""; else SUDO="sudo -n"; fi if ! command -v docker >/dev/null || ! docker compose version >/dev/null 2>&1; then $SUDO apt-get update -qq $SUDO apt-get install -y docker.io docker-compose-v2 - $SUDO systemctl enable --now docker fi - if ! command -v ffmpeg >/dev/null; then - $SUDO apt-get update -qq - $SUDO apt-get install -y ffmpeg - fi - - name: Test real PostgreSQL, S3 and video processing in isolation + $SUDO systemctl enable --now docker + - name: Validate the Compose configuration + run: docker compose --env-file .env.example config --quiet + - name: Type check, unit tests, production build and smoke test inside Docker + run: docker build --target integration -t "$PROMPT_TEST_IMAGE" . + - name: Build non-root production image with bundled FFmpeg + run: docker build --target runtime -t "$PROMPT_IMAGE" . + - name: Test the complete isolated Docker stack run: bash scripts/integration.sh - - name: Deploy persistent systemd service + - name: Deploy the persistent Docker Compose stack if: gitea.ref == 'refs/heads/main' && (gitea.event_name == 'push' || gitea.event_name == 'workflow_dispatch') - env: - RELEASE_SHA: ${{ gitea.sha }} - run: bash scripts/deploy.sh "$RELEASE_SHA" "$(command -v node)" + run: bash scripts/deploy.sh "$RELEASE_SHA" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3e9b9ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM node:24-bookworm-slim AS base +ENV NEXT_TELEMETRY_DISABLED=1 +WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends ffmpeg ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +FROM base AS dependencies +COPY package.json package-lock.json ./ +RUN npm ci + +FROM dependencies AS build +COPY . . +RUN npm run typecheck && npm test && npm run build && npm run test:smoke + +FROM build AS integration +USER node +CMD ["npm", "run", "test:integration"] + +FROM base AS runtime +ENV NODE_ENV=production HOSTNAME=0.0.0.0 PORT=3000 \ + FFMPEG_PATH=/usr/bin/ffmpeg FFPROBE_PATH=/usr/bin/ffprobe +COPY --from=build --chown=node:node /app/.next/standalone/ ./ +COPY --from=build --chown=node:node /app/dist/worker.cjs /app/dist/bootstrap.cjs ./ +COPY --from=build --chown=node:node /app/db/ ./db/ +COPY --chown=node:node scripts/container-health.mjs ./ +USER node +EXPOSE 3000 +CMD ["node", "server.js"] diff --git a/README.md b/README.md index ce414a4..4503c2b 100644 --- a/README.md +++ b/README.md @@ -4,22 +4,31 @@ 自己的素材与创作工作台:Next.js Pages Router、React、TypeScript、Tailwind CSS、Node.js 24。中文界面包含瀑布流、日期贡献热力图、素材导入、多媒体详情、分类管理、独立素材架与线条图形工具。没有创建作品的按钮;图文和动态作品由代码实现。 -## 启动 +## 全 Docker 启动 + +本机只需 Docker Engine / Docker Desktop 和 Compose v2。Node、npm、FFmpeg、迁移程序、网站与后台处理都在镜像里。 ```bash -nvm use -npm ci cp .env.example .env -# 修改 .env 中的三个密码 / 密钥,不使用示例值。 -docker compose --env-file .env up -d -set -a; . ./.env; set +a -npm run storage:prepare -npm run dev -# 另一终端加载同一份 .env 后运行后台视频处理: -npm run worker +# 修改 .env 中的三个密码 / 密钥,不能使用示例值。 +docker build --target runtime -t prompt:local . +docker compose --env-file .env up -d --wait postgres objects +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --wait web worker ``` -本机需要 Docker Compose 和 FFmpeg / ffprobe。开发中的 .env、数据库文件和对象不进入 Git;生产凭据由部署脚本随机生成。 +访问 `http://localhost:18131`。图片与其他文件保存在 S3 对象卷,文字、分类、素材包引用和任务保存在 PostgreSQL 卷;停止、重建应用容器不会删除这些数据。 + +| 容器服务 | 职责 | +| --- | --- | +| `web` | Next.js 页面、API、素材上传/读取;宿主机 18131 → 容器 3000 | +| `worker` | FFmpeg 音画分离、封面与音轨处理;没有 HTTP 端口 | +| `postgres` | PostgreSQL 元数据和文字;兼容维护用回环端口 18132 | +| `objects` | RustFS 私有 S3 对象存储;兼容维护用回环端口 18133,控制台关闭 | +| `migrate` | 一次性数据库迁移与对象桶初始化,执行后退出 | +| `tests` | CI 的一次性集成测试执行器,执行后退出 | + +四个常驻服务使用 `restart: unless-stopped`,Web 和 worker 以非 root 用户运行。worker 通过心跳检查存活;Web 健康检查同时验证当前版本、数据库和 S3。 ## 页面与数据 @@ -35,7 +44,7 @@ npm run worker 素材库不插入演示或占位帖子。空分类显示简洁的空状态;空素材包不进入任何瀑布流。分页游标由时间与 UUID 组成,重复时间戳下仍稳定。 -详细模型见 `docs/data-model.md`。PostgreSQL 保存引用、分类、元数据、文字和任务;RustFS 通过 S3 协议保存二进制对象。数据库和 S3 端口只监听宿主机 `127.0.0.1`,浏览器通过应用访问媒体,支持 Range 分段读取。 +详细模型见 `docs/data-model.md`。应用容器通过 Docker 网络的 `postgres:5432` 和 `objects:9000` 访问存储,不使用宿主机回环地址。浏览器通过应用访问媒体,支持 Range 分段读取。 视频原件保持不变,后台生成无声 MP4、封面,以及存在声音时的 M4A 音轨。音轨记录在 `asset_variants`,始终属于原视频;不会生成独立声音资产。后台单任务处理,使用数据库锁领取任务,失败重试三次,支持超时任务恢复。FFmpeg 禁止输入网络协议。 @@ -53,48 +62,48 @@ npm run worker `contribution_events` 保存细分的 `kind`、对象类型、对象 ID、数量、发生时间、元数据和可选幂等键。显示时按上海时区汇总所有类型。当前没有将上传、分类或保存动作自动算作贡献;触发规则留待确定。没有随机或伪造热力图数据。 -## CI/CD 与运维 +## CI/CD 与生产部署 -`.gitea/workflows/ci.yml` 使用 `runs-on: ces-control`。PR 执行验证;main 上的 push 或手动运行,在类型检查、单元测试、生产构建、启动检查及真实 PostgreSQL/S3/视频集成测试通过后才部署。 +`.gitea/workflows/ci.yml` 使用 `runs-on: ces-control`。依赖安装、类型检查、单元测试、生产构建、启动检查都在 Docker 构建阶段运行。集成测试启动隔离的完整容器栈,并由测试容器请求真实的非 root Web / worker 容器,验证 PostgreSQL、RustFS、上传、共享引用和音画分离。 -集成测试启动带独立 project 名称和随机本机端口的 Docker 服务,使用临时随机凭据;结束后只删除该测试项目的容器和卷,不触碰生产卷。 +main 上的 push 或手动运行,在全部检查通过后才发布。测试环境使用独立 Compose 项目、随机端口和随机凭据;测试结束只删除它自己的容器和卷。 -生产约定: +生产使用固定 Compose project `prompt`,复用原有 `prompt_postgres_data` 和 `prompt_objects` 卷,以及 `/etc/prompt.env` 的凭据。由旧部署升级时,发布脚本会停用 `prompt.service`、`prompt-worker.service`,由容器接管运行;宿主机无需项目专用的 Node/FFmpeg 进程。旧发布目录留作首次迁移失败时的回退依据。 | 项目 | 值 | | --- | --- | -| Web | `prompt.service`,HTTP `18131` | -| 视频处理 | `prompt-worker.service` | -| PostgreSQL | Docker Compose `prompt-postgres`,回环端口 `18132` | -| S3 / RustFS | Docker Compose `prompt-objects`,回环端口 `18133`,控制台关闭 | -| 发布 | `/opt/prompt/releases/`,`/opt/prompt/current` 原子软链接 | -| 部署清单 | `/opt/prompt/infra/compose.yml` | -| 密钥 | `/etc/prompt.env`,root:prompt / 0640,不进代码或日志 | -| 运行数据 | `/var/lib/prompt`;数据库与 S3 使用 Docker named volumes | -| 发布前数据库备份 | `/var/lib/prompt/backups`,保留最近 10 份 | +| 生产运行 | Docker Compose:web、worker、postgres、objects | +| 网站端口 | `18131`,映射到 Web 容器 3000 | +| 应用镜像 | `prompt:`;回退使用上一版镜像 | +| 发布清单 | `/opt/prompt/releases//compose.yml` 和 `release.env` | +| 当前发布 | `/opt/prompt/current` 原子软链接 | +| 密钥 | `/etc/prompt.env`,保留既有凭据,新建时随机生成;不进代码和日志 | +| 持久数据 | Docker named volumes,不随应用发布删除 | +| 数据库备份 | `/var/lib/prompt/backups`,每次发布前备份,保留最近 10 份 | -服务由 systemd 管理,支持开机启动、故障重启和失败回退。迁移采用追加方式;发布前创建 PostgreSQL dump,回退应用版本不会破坏已写入数据。S3 对象卷不随发布清理;对象备份需要另外快照该卷,数据库 dump 不能代替对象备份。 +发布使用宿主机文件锁串行执行,自动跳过已落后的提交。先备份、迁移,再切换 Web / worker;新版本失败时恢复上一版容器,首次迁移失败时可恢复原先的宿主机服务。生产发布从不执行 `down -v`。对象卷需要独立快照,数据库 dump 不能代替对象备份。 + +在服务器上维护当前版本: ```bash -curl -fsS http://127.0.0.1:18131/api/health # 当前应用版本 -curl -fsS http://127.0.0.1:18131/api/ready # 数据库和 S3 连接 -sudo systemctl status prompt.service prompt-worker.service -sudo journalctl -u prompt-worker.service -n 100 --no-pager -sudo docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml ps +cd /opt/prompt/current +sudo docker compose -p prompt --env-file /etc/prompt.env --env-file release.env ps +sudo docker compose -p prompt --env-file /etc/prompt.env --env-file release.env logs --tail 100 web worker +sudo docker compose -p prompt --env-file /etc/prompt.env --env-file release.env exec web node container-health.mjs ``` -首次部署可能安装 Docker Compose 和 FFmpeg;runner 需能访问宿主机 systemd、Docker,并具备 root 或免密 sudo。服务不配置公网域名或防火墙,按现有内网访问方式使用。 +`GET /api/health` 返回应用版本;`GET /api/ready` 检查数据库与对象存储。 -## 验证命令 +## 内网与公网访问 + +`172.31.212.211` 是服务器内网 IP,不能作为公网入口。Web 端口默认映射为 `0.0.0.0:18131:3000`;公网访问仍需要实例绑定公网 IP / EIP 或反向代理,并配置相应安全组和防火墙。Docker 化不会使内网 IP 自动变成公网 IP。本项目不会擅自修改云安全组或开放数据库/S3 的公网端口。 + +## 独立运行 CI 验证 ```bash -npm run typecheck -npm test -npm run build -npm run test:smoke -# 在已配置的隔离 PostgreSQL / S3 环境: -npm run storage:prepare -PROMPT_INTEGRATION_TEST=1 npm run test:integration -# 或在 Docker 主机上一键创建并清理隔离环境: -bash scripts/integration.sh +docker build --target integration -t prompt:tests . +docker build --target runtime -t prompt:local . +PROMPT_IMAGE=prompt:local PROMPT_TEST_IMAGE=prompt:tests bash scripts/integration.sh ``` + +整个验证过程无需在宿主机安装 Node.js、PostgreSQL、S3 或 FFmpeg。runner 仅需 Docker Engine、Compose、Git、Bash、flock 和管理 Docker/旧服务所需的权限。 diff --git a/compose.yml b/compose.yml index 7558039..d9c0a6d 100644 --- a/compose.yml +++ b/compose.yml @@ -1,4 +1,68 @@ +x-app-environment: &app-environment + HOSTNAME: 0.0.0.0 + PORT: "3000" + DATABASE_URL: postgresql://prompt:${POSTGRES_PASSWORD:?Set a unique PostgreSQL password}@postgres:5432/prompt + S3_ENDPOINT: http://objects:9000 + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set a unique S3 access key} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set a unique S3 secret key} + S3_BUCKET: ${S3_BUCKET:-prompt} + RELEASE_SHA: ${RELEASE_SHA:-development} + FFMPEG_PATH: /usr/bin/ffmpeg + FFPROBE_PATH: /usr/bin/ffprobe + +x-app: &app + image: ${PROMPT_IMAGE:-prompt:local} + restart: unless-stopped + environment: *app-environment + init: true + security_opt: + - no-new-privileges:true + cap_drop: [ALL] + depends_on: + postgres: + condition: service_healthy + objects: + condition: service_started + services: + web: + <<: *app + ports: + - "${WEB_BIND_ADDRESS:-0.0.0.0}:${WEB_PORT:-18131}:3000" + healthcheck: + test: ["CMD", "node", "container-health.mjs"] + interval: 5s + timeout: 8s + retries: 24 + start_period: 10s + worker: + <<: *app + command: ["node", "worker.cjs"] + stop_grace_period: 11m + healthcheck: + test: ["CMD", "node", "-e", "const f=require('fs');try{if(Date.now()-f.statSync('/tmp/prompt-worker-health').mtimeMs>60000)process.exit(1)}catch{process.exit(1)}"] + interval: 15s + timeout: 5s + retries: 4 + start_period: 10s + migrate: + <<: *app + profiles: [tools] + restart: "no" + command: ["node", "bootstrap.cjs"] + tests: + image: ${PROMPT_TEST_IMAGE:-prompt:tests} + profiles: [tools] + init: true + environment: + <<: *app-environment + PROMPT_INTEGRATION_TEST: "1" + PROMPT_TEST_BASE_URL: http://web:3000 + depends_on: + web: + condition: service_healthy + worker: + condition: service_healthy postgres: image: postgres:17-alpine restart: unless-stopped diff --git a/package.json b/package.json index 6b80e0a..adc1ff0 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ }, "scripts": { "dev": "next dev", - "build": "next build && esbuild src/worker.ts --bundle --platform=node --format=cjs --external:pg-native --outfile=dist/worker.cjs && node scripts/prepare-standalone.mjs", + "build": "next build && esbuild src/worker.ts --bundle --platform=node --format=cjs --external:pg-native --outfile=dist/worker.cjs && esbuild scripts/bootstrap.ts --bundle --platform=node --format=cjs --external:pg-native --outfile=dist/bootstrap.cjs && node scripts/prepare-standalone.mjs", "start": "next start", "typecheck": "next typegen && tsc --noEmit", "test": "tsx --test tests/health.test.ts tests/model.test.ts", diff --git a/scripts/container-health.mjs b/scripts/container-health.mjs new file mode 100644 index 0000000..4518ee1 --- /dev/null +++ b/scripts/container-health.mjs @@ -0,0 +1,9 @@ +try { + const port = process.env.PORT || '3000'; + const [health, ready] = await Promise.all([ + fetch(`http://127.0.0.1:${port}/api/health`, { signal: AbortSignal.timeout(5000) }), + fetch(`http://127.0.0.1:${port}/api/ready`, { signal: AbortSignal.timeout(5000) }), + ]); + const data = await health.json(); + if (!health.ok || !ready.ok || data.service !== 'prompt' || data.release !== (process.env.RELEASE_SHA || 'development')) process.exit(1); +} catch { process.exit(1); } diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 4071eb9..2a4d81f 100644 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -1,171 +1,97 @@ #!/usr/bin/env bash set -Eeuo pipefail - -# Run on the actual ces-control host, with root or passwordless sudo. -if [ "$(id -u)" -ne 0 ]; then - exec sudo -n bash "$0" "$@" -fi - +if [ "$(id -u)" -ne 0 ]; then exec sudo -n bash "$0" "$@"; fi RELEASE_SHA="${1:?Release SHA is required}" -NODE_SOURCE="${2:?Node executable path is required}" [[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid release SHA'; exit 1; } -test -x "$NODE_SOURCE" -test -d /run/systemd/system || { echo 'Deployment requires the host systemd environment'; exit 1; } -systemctl show-environment >/dev/null -test -f .next/standalone/server.js - +IMAGE="prompt:$RELEASE_SHA" APP_ROOT=/opt/prompt -DATA_ROOT=/var/lib/prompt -SERVICE=prompt.service -SERVICE_USER=prompt -PORT=18131 RELEASE_DIR="$APP_ROOT/releases/$RELEASE_SHA" - +PORT=18131 +command -v docker >/dev/null +docker compose version >/dev/null +docker image inspect "$IMAGE" >/dev/null mkdir -p "$APP_ROOT/releases" exec 9>"$APP_ROOT/.deploy.lock" flock -w 300 9 || { echo 'Another PROMPT deployment is running'; exit 1; } - -# Do not replace the release of a newer main commit after a slower old run. -if [ "$(git -c safe.directory="$PWD" rev-parse HEAD)" != "$RELEASE_SHA" ]; then - echo 'Checkout does not match release SHA'; exit 1 -fi +test "$(git -c safe.directory="$PWD" rev-parse HEAD)" = "$RELEASE_SHA" LATEST_MAIN=$(git -c safe.directory="$PWD" ls-remote origin refs/heads/main | cut -f1) -if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then - echo 'Skipping deployment: main has a newer commit' - exit 0 -fi - +if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then echo 'Skipping deployment: main has a newer commit'; exit 0; fi PREVIOUS="" -if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then - PREVIOUS=$(readlink -f "$APP_ROOT/current") +if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then PREVIOUS=$(readlink -f "$APP_ROOT/current"); fi +bash scripts/infra.sh "$IMAGE" +mkdir -p "$RELEASE_DIR" +install -m 0644 compose.yml "$RELEASE_DIR/compose.yml" +printf 'PROMPT_IMAGE=%s\nRELEASE_SHA=%s\nWEB_PORT=%s\n' "$IMAGE" "$RELEASE_SHA" "$PORT" > "$RELEASE_DIR/release.env" +compose_at(){ local release="$1"; shift; env -u PROMPT_IMAGE -u PROMPT_TEST_IMAGE -u RELEASE_SHA -u WEB_PORT -u WEB_BIND_ADDRESS docker compose -p prompt --env-file /etc/prompt.env --env-file "$release/release.env" -f "$release/compose.yml" "$@"; } +compose(){ compose_at "$RELEASE_DIR" "$@"; } +LEGACY_WEB=0 +LEGACY_WORKER=0 +LEGACY_WEB_ENABLED=0 +LEGACY_WORKER_ENABLED=0 +if command -v systemctl >/dev/null; then + if systemctl is-active --quiet prompt.service; then LEGACY_WEB=1; fi + if systemctl is-active --quiet prompt-worker.service; then LEGACY_WORKER=1; fi + if systemctl is-enabled --quiet prompt.service 2>/dev/null; then LEGACY_WEB_ENABLED=1; fi + if systemctl is-enabled --quiet prompt-worker.service 2>/dev/null; then LEGACY_WORKER_ENABLED=1; fi fi -if [ -z "$PREVIOUS" ]; then - "$NODE_SOURCE" -e "const s=require('net').createServer();s.once('error',()=>process.exit(1));s.listen($PORT,'0.0.0.0',()=>s.close());" || { - echo "Port $PORT is already occupied"; exit 1; - } -fi - -if ! id "$SERVICE_USER" >/dev/null 2>&1; then - useradd --system --home "$DATA_ROOT" --shell /usr/sbin/nologin "$SERVICE_USER" -fi -install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$DATA_ROOT" -bash scripts/infra.sh "$NODE_SOURCE" "$RELEASE_SHA" -set -a -. /etc/prompt.env -set +a -"$NODE_SOURCE" node_modules/tsx/dist/cli.mjs scripts/bootstrap.ts -# Keep runtime data outside the checkout and releases; future modules can use it. -if [ ! -d "$RELEASE_DIR" ]; then - STAGING=$(mktemp -d "$APP_ROOT/releases/.staging.XXXXXX") - cp -a .next/standalone/. "$STAGING/" - mkdir -p "$STAGING/.next" "$STAGING/bin" - cp -a .next/static "$STAGING/.next/static" - if [ -d public ]; then cp -a public "$STAGING/public"; fi - install -m 0755 "$NODE_SOURCE" "$STAGING/bin/node" - install -m 0644 dist/worker.cjs "$STAGING/worker.cjs" - printf 'RELEASE_SHA=%s\n' "$RELEASE_SHA" > "$STAGING/release.env" - chmod -R a+rX "$STAGING" - chmod 0755 "$STAGING" - mv "$STAGING" "$RELEASE_DIR" -fi - -cat > /etc/systemd/system/prompt.service < /etc/systemd/system/prompt-worker.service < "/var/lib/prompt/backups/$(date -u +%Y%m%dT%H%M%S)-$RELEASE_SHA.dump" +find /var/lib/prompt/backups -name '*.dump' -printf '%T@ %p\n' | sort -rn | tail -n +11 | cut -d' ' -f2- | xargs -r rm -- +compose run --rm --no-deps migrate +CUTOVER=1 +# One-time migration: finish a current host worker job before switching the web port. +if [ "$LEGACY_WORKER" -eq 1 ] || [ "$LEGACY_WORKER_ENABLED" -eq 1 ]; then systemctl disable --now prompt-worker.service; fi +if [ "$LEGACY_WEB" -eq 1 ] || [ "$LEGACY_WEB_ENABLED" -eq 1 ]; then systemctl disable --now prompt.service; fi +compose up -d --no-build --wait --wait-timeout 180 web worker +compose exec -T web node container-health.mjs +compose exec -T web node -e "fetch('http://127.0.0.1:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))" +compose exec -T worker sh -c 'test "$(id -u)" -ne 0 && ffmpeg -version >/dev/null' ln -sfn "$RELEASE_DIR" "$APP_ROOT/.current.next" mv -Tf "$APP_ROOT/.current.next" "$APP_ROOT/current" -SWITCHED=1 -systemctl daemon-reload -systemctl enable "$SERVICE" -systemctl restart "$SERVICE" -systemctl enable --now prompt-worker.service -systemctl restart prompt-worker.service - -HEALTHY=0 -for attempt in $(seq 1 30); do - if "$NODE_SOURCE" -e "Promise.all([fetch('http://127.0.0.1:$PORT/api/health'),fetch('http://127.0.0.1:$PORT/api/ready')]).then(async ([r,s])=>{const d=await r.json();if(!r.ok||!s.ok||d.service!=='prompt'||d.release!=='$RELEASE_SHA')process.exit(1)}).catch(()=>process.exit(1))"; then - HEALTHY=1 - break - fi - sleep 2 +compose ps +for service in web worker postgres objects; do + id=$(compose ps -q "$service") + docker inspect --format '{{.Name}} running={{.State.Running}} restart={{.HostConfig.RestartPolicy.Name}}' "$id" done -if [ "$HEALTHY" -ne 1 ]; then - echo 'Health check failed; rolling back' - false +if command -v systemctl >/dev/null; then + if systemctl is-active --quiet prompt.service || systemctl is-active --quiet prompt-worker.service; then echo 'Legacy host service is unexpectedly running'; false; fi fi -systemctl is-enabled "$SERVICE" -systemctl is-active "$SERVICE" -systemctl is-active prompt-worker.service -echo "Persistent deployment verified: service=$SERVICE port=$PORT release=$RELEASE_SHA" +echo "Full Docker deployment verified: web+worker+postgres+objects port=$PORT release=$RELEASE_SHA" +# Read-only network diagnostics: a private address alone is not a public entrypoint. echo "Host: $(hostname)" hostname -I || true +if command -v curl >/dev/null; then + for field in eipv4 public-ipv4; do + address=$(curl --noproxy '*' -fsS --connect-timeout 2 --max-time 4 "http://100.100.100.200/latest/meta-data/$field" 2>/dev/null || true) + if [[ "$address" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "Instance $field: $address"; fi + done +fi diff --git a/scripts/infra.sh b/scripts/infra.sh index f58f142..8e0b65b 100644 --- a/scripts/infra.sh +++ b/scripts/infra.sh @@ -1,34 +1,12 @@ #!/usr/bin/env bash set -Eeuo pipefail -NODE_SOURCE="${1:?Node executable is required}" -if ! command -v docker >/dev/null || ! docker compose version >/dev/null 2>&1; then - apt-get update -qq - DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io docker-compose-v2 - systemctl enable --now docker -fi -if ! command -v ffmpeg >/dev/null; then - apt-get update -qq - DEBIAN_FRONTEND=noninteractive apt-get install -y ffmpeg -fi -umask 027 +IMAGE="${1:?Application image is required}" +# Preserve existing credentials and Docker project/volume names during migration. if [ ! -f /etc/prompt.env ]; then - "$NODE_SOURCE" <<'NODE' > /etc/prompt.env -const {randomBytes}=require('crypto'); -const password=randomBytes(32).toString('hex'); -console.log(`POSTGRES_PASSWORD=${password}\nPG_PORT=18132\nS3_PORT=18133\nDATABASE_URL=postgresql://prompt:${password}@127.0.0.1:18132/prompt\nS3_ENDPOINT=http://127.0.0.1:18133\nS3_ACCESS_KEY=${randomBytes(16).toString('hex')}\nS3_SECRET_KEY=${randomBytes(32).toString('hex')}\nS3_BUCKET=prompt\nFFMPEG_PATH=/usr/bin/ffmpeg\nFFPROBE_PATH=/usr/bin/ffprobe`); -NODE + umask 077 + TEMP_ENV=$(mktemp /etc/prompt.env.XXXXXX) + trap 'rm -f "$TEMP_ENV"' EXIT + docker run --rm --entrypoint node "$IMAGE" -e "const {randomBytes}=require('crypto');const hex=()=>randomBytes(32).toString('hex');console.log('POSTGRES_PASSWORD='+hex()+'\nS3_ACCESS_KEY='+hex()+'\nS3_SECRET_KEY='+hex()+'\nS3_BUCKET=prompt')" > "$TEMP_ENV" + mv "$TEMP_ENV" /etc/prompt.env + trap - EXIT fi -chown root:prompt /etc/prompt.env -chmod 0640 /etc/prompt.env -install -d -m 0755 /opt/prompt/infra -install -m 0644 compose.yml /opt/prompt/infra/compose.yml -docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml up -d -for attempt in $(seq 1 60); do - if docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml exec -T postgres pg_isready -U prompt -d prompt >/dev/null 2>&1; then break; fi - if [ "$attempt" -eq 60 ]; then echo 'PostgreSQL startup timed out'; exit 1; fi - sleep 2 -done -# Additive migrations follow this backup. Keep the most recent 10 database dumps. -install -d -o root -g root -m 0700 /var/lib/prompt/backups -docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml exec -T postgres pg_dump -U prompt -d prompt -Fc > "/var/lib/prompt/backups/$(date -u +%Y%m%dT%H%M%S)-${2:-deploy}.dump" -find /var/lib/prompt/backups -name '*.dump' -printf '%T@ %p\n' | sort -rn | tail -n +11 | cut -d' ' -f2- | xargs -r rm -- diff --git a/scripts/integration.sh b/scripts/integration.sh index 3253706..dc9271f 100644 --- a/scripts/integration.sh +++ b/scripts/integration.sh @@ -1,18 +1,20 @@ #!/usr/bin/env bash set -Eeuo pipefail +: "${PROMPT_IMAGE:?Build the runtime image first}" +: "${PROMPT_TEST_IMAGE:?Build the integration image first}" export COMPOSE_PROJECT_NAME="prompt-ci-${GITEA_RUN_ID:-$$}" -export POSTGRES_PASSWORD S3_ACCESS_KEY S3_SECRET_KEY -POSTGRES_PASSWORD=$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))") -S3_ACCESS_KEY=$(node -e "console.log(require('crypto').randomBytes(12).toString('hex'))") -S3_SECRET_KEY=$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))") -export PG_PORT=0 S3_PORT=0 -cleanup(){ docker compose -f compose.yml down -v --remove-orphans; } +export PG_PORT=0 S3_PORT=0 WEB_PORT=0 WEB_BIND_ADDRESS=127.0.0.1 +TEST_ENV=$(mktemp) +chmod 0600 "$TEST_ENV" +compose(){ docker compose -p "$COMPOSE_PROJECT_NAME" --env-file "$TEST_ENV" -f compose.yml "$@"; } +cleanup(){ compose down -v --remove-orphans; rm -f "$TEST_ENV"; } trap cleanup EXIT -docker compose -f compose.yml up -d -PG_PORT=$(docker compose -f compose.yml port postgres 5432 | tail -1 | sed 's/.*://') -S3_PORT=$(docker compose -f compose.yml port objects 9000 | tail -1 | sed 's/.*://') -export DATABASE_URL="postgresql://prompt:$POSTGRES_PASSWORD@127.0.0.1:$PG_PORT/prompt" -export S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_BUCKET=prompt -export PROMPT_INTEGRATION_TEST=1 -npm run storage:prepare -npm run test:integration +# Random credentials exist only for this disposable stack; no host Node required. +docker run --rm --entrypoint node "$PROMPT_IMAGE" -e "const c=require('crypto');const hex=()=>c.randomBytes(24).toString('hex');console.log('POSTGRES_PASSWORD='+hex()+'\nS3_ACCESS_KEY='+hex()+'\nS3_SECRET_KEY='+hex())" > "$TEST_ENV" +compose up -d --no-build --wait --wait-timeout 120 postgres objects +compose run --rm --no-deps migrate +compose up -d --no-build --wait --wait-timeout 150 web worker +# The test runner calls the actual non-root production web/worker containers. +compose run --rm --no-deps tests +compose exec -T web node container-health.mjs +compose exec -T worker sh -c 'test "$(id -u)" -ne 0 && ffmpeg -version >/dev/null' diff --git a/src/worker.ts b/src/worker.ts index 7136e9f..48b4459 100644 --- a/src/worker.ts +++ b/src/worker.ts @@ -1,4 +1,4 @@ -import { mkdtemp,rm } from "node:fs/promises"; +import { mkdtemp,rm,writeFile } from "node:fs/promises"; import { createWriteStream } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -40,5 +40,10 @@ async function processOne(){ }finally{await rm(dir,{recursive:true,force:true});} return true; } -async function main(){while(!stopping){try{if(!await processOne())await setTimeout(2000);}catch{console.error('Worker connection error; retrying');await setTimeout(5000);}}await db().end();} +async function main(){ + const heartbeat=()=>writeFile('/tmp/prompt-worker-health',String(Date.now())).catch(()=>console.error('Worker heartbeat failed')); + await heartbeat();const timer=setInterval(()=>{void heartbeat();},15000); + try{while(!stopping){try{if(!await processOne())await setTimeout(2000);}catch{console.error('Worker connection error; retrying');await setTimeout(5000);}}} + finally{clearInterval(timer);await db().end();} +} main().catch(()=>process.exit(1)); diff --git a/tests/integration.test.ts b/tests/integration.test.ts index 70627cd..4ea8d14 100644 --- a/tests/integration.test.ts +++ b/tests/integration.test.ts @@ -15,8 +15,10 @@ import {probe,run} from '../src/lib/probe'; test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video separation', {timeout:180000}, async()=>{ assert.equal(process.env.PROMPT_INTEGRATION_TEST,'1','Run only against an explicitly isolated test database'); const socket=createServer();socket.listen(0,'127.0.0.1');await once(socket,'listening');const address=socket.address();assert.ok(address&&typeof address!=='string');const port=address.port;await new Promise(r=>socket.close(()=>r())); - const base=`http://127.0.0.1:${port}`;const server=spawn(process.execPath,['.next/standalone/server.js'],{env:{...process.env,PORT:String(port),HOSTNAME:'127.0.0.1'},stdio:'inherit'}); - const worker=spawn(process.execPath,['dist/worker.cjs'],{env:process.env,stdio:'inherit'});const dir=await mkdtemp(join(tmpdir(),'prompt-test-')); + const external=process.env.PROMPT_TEST_BASE_URL; + const base=external||`http://127.0.0.1:${port}`; + const server=external?null:spawn(process.execPath,['.next/standalone/server.js'],{env:{...process.env,PORT:String(port),HOSTNAME:'127.0.0.1'},stdio:'inherit'}); + const worker=external?null:spawn(process.execPath,['dist/worker.cjs'],{env:process.env,stdio:'inherit'});const dir=await mkdtemp(join(tmpdir(),'prompt-test-')); const request=async(path:string,body?:unknown)=>{const res=await fetch(base+path,body?{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(body)}:undefined);assert.ok(res.ok,`${path}: ${res.status} ${res.ok?'':await res.text()}`);return res.json();}; try{ let ready=false;for(let i=0;i<60;i++){try{if((await fetch(base+'/api/health')).ok){ready=true;break;}}catch{}await setTimeout(250);}assert.ok(ready); @@ -45,7 +47,7 @@ test('real PostgreSQL + S3 library, sharing, taxonomy, uploads, range and video assert.equal((await db().query("SELECT count(*)::integer AS n FROM assets WHERE kind='audio'")).rows[0].n,0,'Bound soundtracks are variants, not independent audio assets'); const audio=join(dir,'audio.wav');await run('ffmpeg',['-y','-v','error','-f','lavfi','-i','sine=frequency=220','-t','0.2',audio]);await upload('audio',[{data:await readFile(audio),name:'audio.wav',mime:'audio/wav'}]);assert.equal((await request('/api/feed?section=audio')).posts.length,1); }finally{ - for(const child of [worker,server])if(child.exitCode===null&&child.signalCode===null){const stopped=once(child,'exit');child.kill('SIGTERM');await stopped;} + for(const child of [worker,server])if(child&&child.exitCode===null&&child.signalCode===null){const stopped=once(child,'exit');child.kill('SIGTERM');await stopped;} await rm(dir,{recursive:true,force:true});await db().end(); } });