Run the complete PROMPT stack and CI verification in Docker
PROMPT Docker CI and deployment / verify-and-deploy (push) Successful in 14m10s
PROMPT Docker CI and deployment / verify-and-deploy (push) Successful in 14m10s
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
try {
|
||||
const port = process.env.PORT || '3000';
|
||||
const [health, ready] = await Promise.all([
|
||||
fetch(`http://127.0.0.1:${port}/api/health`, { signal: AbortSignal.timeout(5000) }),
|
||||
fetch(`http://127.0.0.1:${port}/api/ready`, { signal: AbortSignal.timeout(5000) }),
|
||||
]);
|
||||
const data = await health.json();
|
||||
if (!health.ok || !ready.ok || data.service !== 'prompt' || data.release !== (process.env.RELEASE_SHA || 'development')) process.exit(1);
|
||||
} catch { process.exit(1); }
|
||||
+68
-142
@@ -1,171 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
# Run on the actual ces-control host, with root or passwordless sudo.
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
exec sudo -n bash "$0" "$@"
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then exec sudo -n bash "$0" "$@"; fi
|
||||
RELEASE_SHA="${1:?Release SHA is required}"
|
||||
NODE_SOURCE="${2:?Node executable path is required}"
|
||||
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid release SHA'; exit 1; }
|
||||
test -x "$NODE_SOURCE"
|
||||
test -d /run/systemd/system || { echo 'Deployment requires the host systemd environment'; exit 1; }
|
||||
systemctl show-environment >/dev/null
|
||||
test -f .next/standalone/server.js
|
||||
|
||||
IMAGE="prompt:$RELEASE_SHA"
|
||||
APP_ROOT=/opt/prompt
|
||||
DATA_ROOT=/var/lib/prompt
|
||||
SERVICE=prompt.service
|
||||
SERVICE_USER=prompt
|
||||
PORT=18131
|
||||
RELEASE_DIR="$APP_ROOT/releases/$RELEASE_SHA"
|
||||
|
||||
PORT=18131
|
||||
command -v docker >/dev/null
|
||||
docker compose version >/dev/null
|
||||
docker image inspect "$IMAGE" >/dev/null
|
||||
mkdir -p "$APP_ROOT/releases"
|
||||
exec 9>"$APP_ROOT/.deploy.lock"
|
||||
flock -w 300 9 || { echo 'Another PROMPT deployment is running'; exit 1; }
|
||||
|
||||
# Do not replace the release of a newer main commit after a slower old run.
|
||||
if [ "$(git -c safe.directory="$PWD" rev-parse HEAD)" != "$RELEASE_SHA" ]; then
|
||||
echo 'Checkout does not match release SHA'; exit 1
|
||||
fi
|
||||
test "$(git -c safe.directory="$PWD" rev-parse HEAD)" = "$RELEASE_SHA"
|
||||
LATEST_MAIN=$(git -c safe.directory="$PWD" ls-remote origin refs/heads/main | cut -f1)
|
||||
if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then
|
||||
echo 'Skipping deployment: main has a newer commit'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then echo 'Skipping deployment: main has a newer commit'; exit 0; fi
|
||||
PREVIOUS=""
|
||||
if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then
|
||||
PREVIOUS=$(readlink -f "$APP_ROOT/current")
|
||||
if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then PREVIOUS=$(readlink -f "$APP_ROOT/current"); fi
|
||||
bash scripts/infra.sh "$IMAGE"
|
||||
mkdir -p "$RELEASE_DIR"
|
||||
install -m 0644 compose.yml "$RELEASE_DIR/compose.yml"
|
||||
printf 'PROMPT_IMAGE=%s\nRELEASE_SHA=%s\nWEB_PORT=%s\n' "$IMAGE" "$RELEASE_SHA" "$PORT" > "$RELEASE_DIR/release.env"
|
||||
compose_at(){ local release="$1"; shift; env -u PROMPT_IMAGE -u PROMPT_TEST_IMAGE -u RELEASE_SHA -u WEB_PORT -u WEB_BIND_ADDRESS docker compose -p prompt --env-file /etc/prompt.env --env-file "$release/release.env" -f "$release/compose.yml" "$@"; }
|
||||
compose(){ compose_at "$RELEASE_DIR" "$@"; }
|
||||
LEGACY_WEB=0
|
||||
LEGACY_WORKER=0
|
||||
LEGACY_WEB_ENABLED=0
|
||||
LEGACY_WORKER_ENABLED=0
|
||||
if command -v systemctl >/dev/null; then
|
||||
if systemctl is-active --quiet prompt.service; then LEGACY_WEB=1; fi
|
||||
if systemctl is-active --quiet prompt-worker.service; then LEGACY_WORKER=1; fi
|
||||
if systemctl is-enabled --quiet prompt.service 2>/dev/null; then LEGACY_WEB_ENABLED=1; fi
|
||||
if systemctl is-enabled --quiet prompt-worker.service 2>/dev/null; then LEGACY_WORKER_ENABLED=1; fi
|
||||
fi
|
||||
if [ -z "$PREVIOUS" ]; then
|
||||
"$NODE_SOURCE" -e "const s=require('net').createServer();s.once('error',()=>process.exit(1));s.listen($PORT,'0.0.0.0',()=>s.close());" || {
|
||||
echo "Port $PORT is already occupied"; exit 1;
|
||||
}
|
||||
fi
|
||||
|
||||
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
|
||||
useradd --system --home "$DATA_ROOT" --shell /usr/sbin/nologin "$SERVICE_USER"
|
||||
fi
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$DATA_ROOT"
|
||||
bash scripts/infra.sh "$NODE_SOURCE" "$RELEASE_SHA"
|
||||
set -a
|
||||
. /etc/prompt.env
|
||||
set +a
|
||||
"$NODE_SOURCE" node_modules/tsx/dist/cli.mjs scripts/bootstrap.ts
|
||||
# Keep runtime data outside the checkout and releases; future modules can use it.
|
||||
if [ ! -d "$RELEASE_DIR" ]; then
|
||||
STAGING=$(mktemp -d "$APP_ROOT/releases/.staging.XXXXXX")
|
||||
cp -a .next/standalone/. "$STAGING/"
|
||||
mkdir -p "$STAGING/.next" "$STAGING/bin"
|
||||
cp -a .next/static "$STAGING/.next/static"
|
||||
if [ -d public ]; then cp -a public "$STAGING/public"; fi
|
||||
install -m 0755 "$NODE_SOURCE" "$STAGING/bin/node"
|
||||
install -m 0644 dist/worker.cjs "$STAGING/worker.cjs"
|
||||
printf 'RELEASE_SHA=%s\n' "$RELEASE_SHA" > "$STAGING/release.env"
|
||||
chmod -R a+rX "$STAGING"
|
||||
chmod 0755 "$STAGING"
|
||||
mv "$STAGING" "$RELEASE_DIR"
|
||||
fi
|
||||
|
||||
cat > /etc/systemd/system/prompt.service <<EOF
|
||||
[Unit]
|
||||
Description=PROMPT
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$SERVICE_USER
|
||||
Group=$SERVICE_USER
|
||||
WorkingDirectory=$APP_ROOT/current
|
||||
Environment=NODE_ENV=production
|
||||
Environment=HOSTNAME=0.0.0.0
|
||||
Environment=PORT=$PORT
|
||||
Environment=PROMPT_DATA_DIR=$DATA_ROOT
|
||||
EnvironmentFile=$APP_ROOT/current/release.env
|
||||
EnvironmentFile=/etc/prompt.env
|
||||
ExecStart=$APP_ROOT/current/bin/node server.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
UMask=0027
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/prompt-worker.service <<EOF
|
||||
[Unit]
|
||||
Description=PROMPT media processing
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$SERVICE_USER
|
||||
Group=$SERVICE_USER
|
||||
WorkingDirectory=$APP_ROOT/current
|
||||
Environment=NODE_ENV=production
|
||||
EnvironmentFile=/etc/prompt.env
|
||||
ExecStart=$APP_ROOT/current/bin/node worker.cjs
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
TimeoutStopSec=650
|
||||
NoNewPrivileges=true
|
||||
UMask=0027
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
SWITCHED=0
|
||||
rollback() {
|
||||
CUTOVER=0
|
||||
rollback(){
|
||||
local code=$?
|
||||
trap - ERR
|
||||
set +e
|
||||
if [ "$SWITCHED" -eq 1 ]; then
|
||||
if [ "$CUTOVER" -eq 1 ]; then
|
||||
if [ -n "$PREVIOUS" ] && [ -f "$PREVIOUS/compose.yml" ]; then
|
||||
compose_at "$PREVIOUS" up -d --no-build web worker
|
||||
echo 'Restored the previous Docker application image'
|
||||
else
|
||||
compose stop web worker
|
||||
compose rm -f web worker
|
||||
if [ "$LEGACY_WORKER_ENABLED" -eq 1 ]; then systemctl enable prompt-worker.service; fi
|
||||
if [ "$LEGACY_WEB_ENABLED" -eq 1 ]; then systemctl enable prompt.service; fi
|
||||
if [ "$LEGACY_WEB" -eq 1 ]; then systemctl start prompt.service; fi
|
||||
if [ "$LEGACY_WORKER" -eq 1 ]; then systemctl start prompt-worker.service; fi
|
||||
echo 'Restored legacy application services if previously running'
|
||||
fi
|
||||
if [ -n "$PREVIOUS" ]; then
|
||||
ln -sfn "$PREVIOUS" "$APP_ROOT/.current.rollback"
|
||||
mv -Tf "$APP_ROOT/.current.rollback" "$APP_ROOT/current"
|
||||
systemctl restart "$SERVICE"
|
||||
if [ -f "$PREVIOUS/worker.cjs" ]; then systemctl restart prompt-worker.service; else systemctl disable --now prompt-worker.service; fi
|
||||
echo 'Restored previous release'
|
||||
else
|
||||
systemctl disable --now "$SERVICE"
|
||||
systemctl disable --now prompt-worker.service
|
||||
rm -f "$APP_ROOT/current"
|
||||
echo 'Stopped failed first deployment'
|
||||
fi
|
||||
fi
|
||||
echo "Deployment failed (exit $code)"
|
||||
echo "Docker deployment failed (exit $code); persistent data volumes were retained"
|
||||
exit "$code"
|
||||
}
|
||||
trap rollback ERR
|
||||
# Keep the same prompt_postgres_data and prompt_objects volumes. No production down -v.
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres objects
|
||||
install -d -o root -g root -m 0700 /var/lib/prompt/backups
|
||||
compose exec -T postgres pg_dump -U prompt -d prompt -Fc > "/var/lib/prompt/backups/$(date -u +%Y%m%dT%H%M%S)-$RELEASE_SHA.dump"
|
||||
find /var/lib/prompt/backups -name '*.dump' -printf '%T@ %p\n' | sort -rn | tail -n +11 | cut -d' ' -f2- | xargs -r rm --
|
||||
compose run --rm --no-deps migrate
|
||||
CUTOVER=1
|
||||
# One-time migration: finish a current host worker job before switching the web port.
|
||||
if [ "$LEGACY_WORKER" -eq 1 ] || [ "$LEGACY_WORKER_ENABLED" -eq 1 ]; then systemctl disable --now prompt-worker.service; fi
|
||||
if [ "$LEGACY_WEB" -eq 1 ] || [ "$LEGACY_WEB_ENABLED" -eq 1 ]; then systemctl disable --now prompt.service; fi
|
||||
compose up -d --no-build --wait --wait-timeout 180 web worker
|
||||
compose exec -T web node container-health.mjs
|
||||
compose exec -T web node -e "fetch('http://127.0.0.1:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"
|
||||
compose exec -T worker sh -c 'test "$(id -u)" -ne 0 && ffmpeg -version >/dev/null'
|
||||
ln -sfn "$RELEASE_DIR" "$APP_ROOT/.current.next"
|
||||
mv -Tf "$APP_ROOT/.current.next" "$APP_ROOT/current"
|
||||
SWITCHED=1
|
||||
systemctl daemon-reload
|
||||
systemctl enable "$SERVICE"
|
||||
systemctl restart "$SERVICE"
|
||||
systemctl enable --now prompt-worker.service
|
||||
systemctl restart prompt-worker.service
|
||||
|
||||
HEALTHY=0
|
||||
for attempt in $(seq 1 30); do
|
||||
if "$NODE_SOURCE" -e "Promise.all([fetch('http://127.0.0.1:$PORT/api/health'),fetch('http://127.0.0.1:$PORT/api/ready')]).then(async ([r,s])=>{const d=await r.json();if(!r.ok||!s.ok||d.service!=='prompt'||d.release!=='$RELEASE_SHA')process.exit(1)}).catch(()=>process.exit(1))"; then
|
||||
HEALTHY=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
compose ps
|
||||
for service in web worker postgres objects; do
|
||||
id=$(compose ps -q "$service")
|
||||
docker inspect --format '{{.Name}} running={{.State.Running}} restart={{.HostConfig.RestartPolicy.Name}}' "$id"
|
||||
done
|
||||
if [ "$HEALTHY" -ne 1 ]; then
|
||||
echo 'Health check failed; rolling back'
|
||||
false
|
||||
if command -v systemctl >/dev/null; then
|
||||
if systemctl is-active --quiet prompt.service || systemctl is-active --quiet prompt-worker.service; then echo 'Legacy host service is unexpectedly running'; false; fi
|
||||
fi
|
||||
systemctl is-enabled "$SERVICE"
|
||||
systemctl is-active "$SERVICE"
|
||||
systemctl is-active prompt-worker.service
|
||||
echo "Persistent deployment verified: service=$SERVICE port=$PORT release=$RELEASE_SHA"
|
||||
echo "Full Docker deployment verified: web+worker+postgres+objects port=$PORT release=$RELEASE_SHA"
|
||||
# Read-only network diagnostics: a private address alone is not a public entrypoint.
|
||||
echo "Host: $(hostname)"
|
||||
hostname -I || true
|
||||
if command -v curl >/dev/null; then
|
||||
for field in eipv4 public-ipv4; do
|
||||
address=$(curl --noproxy '*' -fsS --connect-timeout 2 --max-time 4 "http://100.100.100.200/latest/meta-data/$field" 2>/dev/null || true)
|
||||
if [[ "$address" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "Instance $field: $address"; fi
|
||||
done
|
||||
fi
|
||||
|
||||
+8
-30
@@ -1,34 +1,12 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
NODE_SOURCE="${1:?Node executable is required}"
|
||||
if ! command -v docker >/dev/null || ! docker compose version >/dev/null 2>&1; then
|
||||
apt-get update -qq
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y docker.io docker-compose-v2
|
||||
systemctl enable --now docker
|
||||
fi
|
||||
if ! command -v ffmpeg >/dev/null; then
|
||||
apt-get update -qq
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y ffmpeg
|
||||
fi
|
||||
umask 027
|
||||
IMAGE="${1:?Application image is required}"
|
||||
# Preserve existing credentials and Docker project/volume names during migration.
|
||||
if [ ! -f /etc/prompt.env ]; then
|
||||
"$NODE_SOURCE" <<'NODE' > /etc/prompt.env
|
||||
const {randomBytes}=require('crypto');
|
||||
const password=randomBytes(32).toString('hex');
|
||||
console.log(`POSTGRES_PASSWORD=${password}\nPG_PORT=18132\nS3_PORT=18133\nDATABASE_URL=postgresql://prompt:${password}@127.0.0.1:18132/prompt\nS3_ENDPOINT=http://127.0.0.1:18133\nS3_ACCESS_KEY=${randomBytes(16).toString('hex')}\nS3_SECRET_KEY=${randomBytes(32).toString('hex')}\nS3_BUCKET=prompt\nFFMPEG_PATH=/usr/bin/ffmpeg\nFFPROBE_PATH=/usr/bin/ffprobe`);
|
||||
NODE
|
||||
umask 077
|
||||
TEMP_ENV=$(mktemp /etc/prompt.env.XXXXXX)
|
||||
trap 'rm -f "$TEMP_ENV"' EXIT
|
||||
docker run --rm --entrypoint node "$IMAGE" -e "const {randomBytes}=require('crypto');const hex=()=>randomBytes(32).toString('hex');console.log('POSTGRES_PASSWORD='+hex()+'\nS3_ACCESS_KEY='+hex()+'\nS3_SECRET_KEY='+hex()+'\nS3_BUCKET=prompt')" > "$TEMP_ENV"
|
||||
mv "$TEMP_ENV" /etc/prompt.env
|
||||
trap - EXIT
|
||||
fi
|
||||
chown root:prompt /etc/prompt.env
|
||||
chmod 0640 /etc/prompt.env
|
||||
install -d -m 0755 /opt/prompt/infra
|
||||
install -m 0644 compose.yml /opt/prompt/infra/compose.yml
|
||||
docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml up -d
|
||||
for attempt in $(seq 1 60); do
|
||||
if docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml exec -T postgres pg_isready -U prompt -d prompt >/dev/null 2>&1; then break; fi
|
||||
if [ "$attempt" -eq 60 ]; then echo 'PostgreSQL startup timed out'; exit 1; fi
|
||||
sleep 2
|
||||
done
|
||||
# Additive migrations follow this backup. Keep the most recent 10 database dumps.
|
||||
install -d -o root -g root -m 0700 /var/lib/prompt/backups
|
||||
docker compose -p prompt --env-file /etc/prompt.env -f /opt/prompt/infra/compose.yml exec -T postgres pg_dump -U prompt -d prompt -Fc > "/var/lib/prompt/backups/$(date -u +%Y%m%dT%H%M%S)-${2:-deploy}.dump"
|
||||
find /var/lib/prompt/backups -name '*.dump' -printf '%T@ %p\n' | sort -rn | tail -n +11 | cut -d' ' -f2- | xargs -r rm --
|
||||
|
||||
+16
-14
@@ -1,18 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
: "${PROMPT_IMAGE:?Build the runtime image first}"
|
||||
: "${PROMPT_TEST_IMAGE:?Build the integration image first}"
|
||||
export COMPOSE_PROJECT_NAME="prompt-ci-${GITEA_RUN_ID:-$$}"
|
||||
export POSTGRES_PASSWORD S3_ACCESS_KEY S3_SECRET_KEY
|
||||
POSTGRES_PASSWORD=$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))")
|
||||
S3_ACCESS_KEY=$(node -e "console.log(require('crypto').randomBytes(12).toString('hex'))")
|
||||
S3_SECRET_KEY=$(node -e "console.log(require('crypto').randomBytes(24).toString('hex'))")
|
||||
export PG_PORT=0 S3_PORT=0
|
||||
cleanup(){ docker compose -f compose.yml down -v --remove-orphans; }
|
||||
export PG_PORT=0 S3_PORT=0 WEB_PORT=0 WEB_BIND_ADDRESS=127.0.0.1
|
||||
TEST_ENV=$(mktemp)
|
||||
chmod 0600 "$TEST_ENV"
|
||||
compose(){ docker compose -p "$COMPOSE_PROJECT_NAME" --env-file "$TEST_ENV" -f compose.yml "$@"; }
|
||||
cleanup(){ compose down -v --remove-orphans; rm -f "$TEST_ENV"; }
|
||||
trap cleanup EXIT
|
||||
docker compose -f compose.yml up -d
|
||||
PG_PORT=$(docker compose -f compose.yml port postgres 5432 | tail -1 | sed 's/.*://')
|
||||
S3_PORT=$(docker compose -f compose.yml port objects 9000 | tail -1 | sed 's/.*://')
|
||||
export DATABASE_URL="postgresql://prompt:$POSTGRES_PASSWORD@127.0.0.1:$PG_PORT/prompt"
|
||||
export S3_ENDPOINT="http://127.0.0.1:$S3_PORT" S3_BUCKET=prompt
|
||||
export PROMPT_INTEGRATION_TEST=1
|
||||
npm run storage:prepare
|
||||
npm run test:integration
|
||||
# Random credentials exist only for this disposable stack; no host Node required.
|
||||
docker run --rm --entrypoint node "$PROMPT_IMAGE" -e "const c=require('crypto');const hex=()=>c.randomBytes(24).toString('hex');console.log('POSTGRES_PASSWORD='+hex()+'\nS3_ACCESS_KEY='+hex()+'\nS3_SECRET_KEY='+hex())" > "$TEST_ENV"
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres objects
|
||||
compose run --rm --no-deps migrate
|
||||
compose up -d --no-build --wait --wait-timeout 150 web worker
|
||||
# The test runner calls the actual non-root production web/worker containers.
|
||||
compose run --rm --no-deps tests
|
||||
compose exec -T web node container-health.mjs
|
||||
compose exec -T worker sh -c 'test "$(id -u)" -ne 0 && ffmpeg -version >/dev/null'
|
||||
|
||||
Reference in New Issue
Block a user