Run the complete PROMPT stack and CI verification in Docker
PROMPT Docker CI and deployment / verify-and-deploy (push) Successful in 14m10s
PROMPT Docker CI and deployment / verify-and-deploy (push) Successful in 14m10s
This commit is contained in:
+68
-142
@@ -1,171 +1,97 @@
|
||||
#!/usr/bin/env bash
|
||||
set -Eeuo pipefail
|
||||
|
||||
# Run on the actual ces-control host, with root or passwordless sudo.
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
exec sudo -n bash "$0" "$@"
|
||||
fi
|
||||
|
||||
if [ "$(id -u)" -ne 0 ]; then exec sudo -n bash "$0" "$@"; fi
|
||||
RELEASE_SHA="${1:?Release SHA is required}"
|
||||
NODE_SOURCE="${2:?Node executable path is required}"
|
||||
[[ "$RELEASE_SHA" =~ ^[0-9a-f]{40}$ ]] || { echo 'Invalid release SHA'; exit 1; }
|
||||
test -x "$NODE_SOURCE"
|
||||
test -d /run/systemd/system || { echo 'Deployment requires the host systemd environment'; exit 1; }
|
||||
systemctl show-environment >/dev/null
|
||||
test -f .next/standalone/server.js
|
||||
|
||||
IMAGE="prompt:$RELEASE_SHA"
|
||||
APP_ROOT=/opt/prompt
|
||||
DATA_ROOT=/var/lib/prompt
|
||||
SERVICE=prompt.service
|
||||
SERVICE_USER=prompt
|
||||
PORT=18131
|
||||
RELEASE_DIR="$APP_ROOT/releases/$RELEASE_SHA"
|
||||
|
||||
PORT=18131
|
||||
command -v docker >/dev/null
|
||||
docker compose version >/dev/null
|
||||
docker image inspect "$IMAGE" >/dev/null
|
||||
mkdir -p "$APP_ROOT/releases"
|
||||
exec 9>"$APP_ROOT/.deploy.lock"
|
||||
flock -w 300 9 || { echo 'Another PROMPT deployment is running'; exit 1; }
|
||||
|
||||
# Do not replace the release of a newer main commit after a slower old run.
|
||||
if [ "$(git -c safe.directory="$PWD" rev-parse HEAD)" != "$RELEASE_SHA" ]; then
|
||||
echo 'Checkout does not match release SHA'; exit 1
|
||||
fi
|
||||
test "$(git -c safe.directory="$PWD" rev-parse HEAD)" = "$RELEASE_SHA"
|
||||
LATEST_MAIN=$(git -c safe.directory="$PWD" ls-remote origin refs/heads/main | cut -f1)
|
||||
if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then
|
||||
echo 'Skipping deployment: main has a newer commit'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then echo 'Skipping deployment: main has a newer commit'; exit 0; fi
|
||||
PREVIOUS=""
|
||||
if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then
|
||||
PREVIOUS=$(readlink -f "$APP_ROOT/current")
|
||||
if [ -L "$APP_ROOT/current" ] && [ -d "$APP_ROOT/current" ]; then PREVIOUS=$(readlink -f "$APP_ROOT/current"); fi
|
||||
bash scripts/infra.sh "$IMAGE"
|
||||
mkdir -p "$RELEASE_DIR"
|
||||
install -m 0644 compose.yml "$RELEASE_DIR/compose.yml"
|
||||
printf 'PROMPT_IMAGE=%s\nRELEASE_SHA=%s\nWEB_PORT=%s\n' "$IMAGE" "$RELEASE_SHA" "$PORT" > "$RELEASE_DIR/release.env"
|
||||
compose_at(){ local release="$1"; shift; env -u PROMPT_IMAGE -u PROMPT_TEST_IMAGE -u RELEASE_SHA -u WEB_PORT -u WEB_BIND_ADDRESS docker compose -p prompt --env-file /etc/prompt.env --env-file "$release/release.env" -f "$release/compose.yml" "$@"; }
|
||||
compose(){ compose_at "$RELEASE_DIR" "$@"; }
|
||||
LEGACY_WEB=0
|
||||
LEGACY_WORKER=0
|
||||
LEGACY_WEB_ENABLED=0
|
||||
LEGACY_WORKER_ENABLED=0
|
||||
if command -v systemctl >/dev/null; then
|
||||
if systemctl is-active --quiet prompt.service; then LEGACY_WEB=1; fi
|
||||
if systemctl is-active --quiet prompt-worker.service; then LEGACY_WORKER=1; fi
|
||||
if systemctl is-enabled --quiet prompt.service 2>/dev/null; then LEGACY_WEB_ENABLED=1; fi
|
||||
if systemctl is-enabled --quiet prompt-worker.service 2>/dev/null; then LEGACY_WORKER_ENABLED=1; fi
|
||||
fi
|
||||
if [ -z "$PREVIOUS" ]; then
|
||||
"$NODE_SOURCE" -e "const s=require('net').createServer();s.once('error',()=>process.exit(1));s.listen($PORT,'0.0.0.0',()=>s.close());" || {
|
||||
echo "Port $PORT is already occupied"; exit 1;
|
||||
}
|
||||
fi
|
||||
|
||||
if ! id "$SERVICE_USER" >/dev/null 2>&1; then
|
||||
useradd --system --home "$DATA_ROOT" --shell /usr/sbin/nologin "$SERVICE_USER"
|
||||
fi
|
||||
install -d -o "$SERVICE_USER" -g "$SERVICE_USER" -m 0750 "$DATA_ROOT"
|
||||
bash scripts/infra.sh "$NODE_SOURCE" "$RELEASE_SHA"
|
||||
set -a
|
||||
. /etc/prompt.env
|
||||
set +a
|
||||
"$NODE_SOURCE" node_modules/tsx/dist/cli.mjs scripts/bootstrap.ts
|
||||
# Keep runtime data outside the checkout and releases; future modules can use it.
|
||||
if [ ! -d "$RELEASE_DIR" ]; then
|
||||
STAGING=$(mktemp -d "$APP_ROOT/releases/.staging.XXXXXX")
|
||||
cp -a .next/standalone/. "$STAGING/"
|
||||
mkdir -p "$STAGING/.next" "$STAGING/bin"
|
||||
cp -a .next/static "$STAGING/.next/static"
|
||||
if [ -d public ]; then cp -a public "$STAGING/public"; fi
|
||||
install -m 0755 "$NODE_SOURCE" "$STAGING/bin/node"
|
||||
install -m 0644 dist/worker.cjs "$STAGING/worker.cjs"
|
||||
printf 'RELEASE_SHA=%s\n' "$RELEASE_SHA" > "$STAGING/release.env"
|
||||
chmod -R a+rX "$STAGING"
|
||||
chmod 0755 "$STAGING"
|
||||
mv "$STAGING" "$RELEASE_DIR"
|
||||
fi
|
||||
|
||||
cat > /etc/systemd/system/prompt.service <<EOF
|
||||
[Unit]
|
||||
Description=PROMPT
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$SERVICE_USER
|
||||
Group=$SERVICE_USER
|
||||
WorkingDirectory=$APP_ROOT/current
|
||||
Environment=NODE_ENV=production
|
||||
Environment=HOSTNAME=0.0.0.0
|
||||
Environment=PORT=$PORT
|
||||
Environment=PROMPT_DATA_DIR=$DATA_ROOT
|
||||
EnvironmentFile=$APP_ROOT/current/release.env
|
||||
EnvironmentFile=/etc/prompt.env
|
||||
ExecStart=$APP_ROOT/current/bin/node server.js
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
NoNewPrivileges=true
|
||||
UMask=0027
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
cat > /etc/systemd/system/prompt-worker.service <<EOF
|
||||
[Unit]
|
||||
Description=PROMPT media processing
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=$SERVICE_USER
|
||||
Group=$SERVICE_USER
|
||||
WorkingDirectory=$APP_ROOT/current
|
||||
Environment=NODE_ENV=production
|
||||
EnvironmentFile=/etc/prompt.env
|
||||
ExecStart=$APP_ROOT/current/bin/node worker.cjs
|
||||
Restart=always
|
||||
RestartSec=5
|
||||
TimeoutStopSec=650
|
||||
NoNewPrivileges=true
|
||||
UMask=0027
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
SWITCHED=0
|
||||
rollback() {
|
||||
CUTOVER=0
|
||||
rollback(){
|
||||
local code=$?
|
||||
trap - ERR
|
||||
set +e
|
||||
if [ "$SWITCHED" -eq 1 ]; then
|
||||
if [ "$CUTOVER" -eq 1 ]; then
|
||||
if [ -n "$PREVIOUS" ] && [ -f "$PREVIOUS/compose.yml" ]; then
|
||||
compose_at "$PREVIOUS" up -d --no-build web worker
|
||||
echo 'Restored the previous Docker application image'
|
||||
else
|
||||
compose stop web worker
|
||||
compose rm -f web worker
|
||||
if [ "$LEGACY_WORKER_ENABLED" -eq 1 ]; then systemctl enable prompt-worker.service; fi
|
||||
if [ "$LEGACY_WEB_ENABLED" -eq 1 ]; then systemctl enable prompt.service; fi
|
||||
if [ "$LEGACY_WEB" -eq 1 ]; then systemctl start prompt.service; fi
|
||||
if [ "$LEGACY_WORKER" -eq 1 ]; then systemctl start prompt-worker.service; fi
|
||||
echo 'Restored legacy application services if previously running'
|
||||
fi
|
||||
if [ -n "$PREVIOUS" ]; then
|
||||
ln -sfn "$PREVIOUS" "$APP_ROOT/.current.rollback"
|
||||
mv -Tf "$APP_ROOT/.current.rollback" "$APP_ROOT/current"
|
||||
systemctl restart "$SERVICE"
|
||||
if [ -f "$PREVIOUS/worker.cjs" ]; then systemctl restart prompt-worker.service; else systemctl disable --now prompt-worker.service; fi
|
||||
echo 'Restored previous release'
|
||||
else
|
||||
systemctl disable --now "$SERVICE"
|
||||
systemctl disable --now prompt-worker.service
|
||||
rm -f "$APP_ROOT/current"
|
||||
echo 'Stopped failed first deployment'
|
||||
fi
|
||||
fi
|
||||
echo "Deployment failed (exit $code)"
|
||||
echo "Docker deployment failed (exit $code); persistent data volumes were retained"
|
||||
exit "$code"
|
||||
}
|
||||
trap rollback ERR
|
||||
# Keep the same prompt_postgres_data and prompt_objects volumes. No production down -v.
|
||||
compose up -d --no-build --wait --wait-timeout 120 postgres objects
|
||||
install -d -o root -g root -m 0700 /var/lib/prompt/backups
|
||||
compose exec -T postgres pg_dump -U prompt -d prompt -Fc > "/var/lib/prompt/backups/$(date -u +%Y%m%dT%H%M%S)-$RELEASE_SHA.dump"
|
||||
find /var/lib/prompt/backups -name '*.dump' -printf '%T@ %p\n' | sort -rn | tail -n +11 | cut -d' ' -f2- | xargs -r rm --
|
||||
compose run --rm --no-deps migrate
|
||||
CUTOVER=1
|
||||
# One-time migration: finish a current host worker job before switching the web port.
|
||||
if [ "$LEGACY_WORKER" -eq 1 ] || [ "$LEGACY_WORKER_ENABLED" -eq 1 ]; then systemctl disable --now prompt-worker.service; fi
|
||||
if [ "$LEGACY_WEB" -eq 1 ] || [ "$LEGACY_WEB_ENABLED" -eq 1 ]; then systemctl disable --now prompt.service; fi
|
||||
compose up -d --no-build --wait --wait-timeout 180 web worker
|
||||
compose exec -T web node container-health.mjs
|
||||
compose exec -T web node -e "fetch('http://127.0.0.1:3000/').then(r=>{if(!r.ok)process.exit(1)}).catch(()=>process.exit(1))"
|
||||
compose exec -T worker sh -c 'test "$(id -u)" -ne 0 && ffmpeg -version >/dev/null'
|
||||
ln -sfn "$RELEASE_DIR" "$APP_ROOT/.current.next"
|
||||
mv -Tf "$APP_ROOT/.current.next" "$APP_ROOT/current"
|
||||
SWITCHED=1
|
||||
systemctl daemon-reload
|
||||
systemctl enable "$SERVICE"
|
||||
systemctl restart "$SERVICE"
|
||||
systemctl enable --now prompt-worker.service
|
||||
systemctl restart prompt-worker.service
|
||||
|
||||
HEALTHY=0
|
||||
for attempt in $(seq 1 30); do
|
||||
if "$NODE_SOURCE" -e "Promise.all([fetch('http://127.0.0.1:$PORT/api/health'),fetch('http://127.0.0.1:$PORT/api/ready')]).then(async ([r,s])=>{const d=await r.json();if(!r.ok||!s.ok||d.service!=='prompt'||d.release!=='$RELEASE_SHA')process.exit(1)}).catch(()=>process.exit(1))"; then
|
||||
HEALTHY=1
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
compose ps
|
||||
for service in web worker postgres objects; do
|
||||
id=$(compose ps -q "$service")
|
||||
docker inspect --format '{{.Name}} running={{.State.Running}} restart={{.HostConfig.RestartPolicy.Name}}' "$id"
|
||||
done
|
||||
if [ "$HEALTHY" -ne 1 ]; then
|
||||
echo 'Health check failed; rolling back'
|
||||
false
|
||||
if command -v systemctl >/dev/null; then
|
||||
if systemctl is-active --quiet prompt.service || systemctl is-active --quiet prompt-worker.service; then echo 'Legacy host service is unexpectedly running'; false; fi
|
||||
fi
|
||||
systemctl is-enabled "$SERVICE"
|
||||
systemctl is-active "$SERVICE"
|
||||
systemctl is-active prompt-worker.service
|
||||
echo "Persistent deployment verified: service=$SERVICE port=$PORT release=$RELEASE_SHA"
|
||||
echo "Full Docker deployment verified: web+worker+postgres+objects port=$PORT release=$RELEASE_SHA"
|
||||
# Read-only network diagnostics: a private address alone is not a public entrypoint.
|
||||
echo "Host: $(hostname)"
|
||||
hostname -I || true
|
||||
if command -v curl >/dev/null; then
|
||||
for field in eipv4 public-ipv4; do
|
||||
address=$(curl --noproxy '*' -fsS --connect-timeout 2 --max-time 4 "http://100.100.100.200/latest/meta-data/$field" 2>/dev/null || true)
|
||||
if [[ "$address" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "Instance $field: $address"; fi
|
||||
done
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user