Files
ai-taxonomy-literature-anal…/scripts/deploy.sh
T
claude-bot 402a721790
Deploy / deploy (push) Successful in 3m45s
CI: keep the node_modules cache outside the releases
The first version looked for node_modules in the current release, which does not work when the same
commit is deployed again (its directory is replaced). Cache hard-linked installs under
/opt/llm-survey/cache/node_modules/<lock hash> instead.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FbMbJPadPzVwvt9rU9GBNo
2026-10-24 00:05:07 +08:00

142 lines
7.4 KiB
Bash

#!/bin/sh
# Permanent deploy on the ces-control host, fully driven by CI:
# 1. install + provision PostgreSQL (apt, systemd; role/db created once, password generated once)
# 2. fetch raw sources (cached), build a staging SQLite with the Python scripts, export it to JSON lines
# 3. build the site, load the data into Postgres (schema "next" -> swapped in as "live")
# 4. switch /opt/llm-survey/current, (re)start systemd unit `llm-survey`, health-check
# Port is never 3000 or 18131 (taken).
# Caches under $ROOT/cache (a deploy that changes neither the data scripts nor the lockfile takes about 2 minutes, not 8):
# export/<key> the JSON-lines export, keyed by a hash of the data scripts + data/labels (rebuilt when they change,
# or when REFRESH_DATA=true: nightly run / manual run, which also re-download the raw sources)
# loaded id of the export now in Postgres schema "live" (the load is skipped when it matches)
# next Next.js compiler cache, linked into .next/cache
# node_modules/<key> hard-linked copy of the installed dependencies, keyed by package-lock.json + node version.
set -eu
SHA="${1:?usage: deploy.sh <git-sha>}"
APP=llm-survey
ROOT=/opt/$APP
ENVF=/etc/$APP/db.env
PORT="${APP_PORT:-18231}"
DB=llm_survey
case "$PORT" in 3000|18131) echo "refusing reserved port $PORT" >&2; exit 1;; esac
for c in node python3 git curl bash; do command -v $c >/dev/null || { echo "$c is not installed on this host" >&2; exit 1; }; done
node -e 'process.exit(+process.versions.node.split(".")[0] >= 18 ? 0 : 1)' || { echo "need node >= 18" >&2; exit 1; }
# ---- 1. PostgreSQL --------------------------------------------------------------------
if ! command -v psql >/dev/null || ! [ -d /usr/lib/postgresql ]; then
echo "installing postgresql"
export DEBIAN_FRONTEND=noninteractive
apt-get update -qq && apt-get install -y -qq postgresql postgresql-contrib
fi
systemctl enable --now postgresql >/dev/null 2>&1 || true
i=0; until su postgres -c "pg_isready -q" 2>/dev/null || [ $i -ge 30 ]; do i=$((i+1)); sleep 2; done
su postgres -c "pg_isready -q" || { echo "postgresql did not become ready" >&2; exit 1; }
PGPORT=$(su postgres -c "psql -tAc 'show port'")
psql_admin() { su postgres -c "psql -v ON_ERROR_STOP=1 -qtA $*"; }
mkdir -p "$(dirname "$ENVF")"; chmod 755 "$(dirname "$ENVF")"
if [ ! -s "$ENVF" ]; then
PW=$(head -c 64 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | head -c 28)
umask 077
printf 'DATABASE_URL=postgres://%s:%s@127.0.0.1:%s/%s\n' "$DB" "$PW" "$PGPORT" "$DB" > "$ENVF"
fi
PW=$(sed -E 's#^DATABASE_URL=postgres://[^:]+:([^@]+)@.*#\1#' "$ENVF")
# keep the role password and the env file in sync; create role/db only when missing
if [ "$(psql_admin "-c \"select 1 from pg_roles where rolname='$DB'\"")" = 1 ]; then
psql_admin "-c \"alter role $DB with login password '$PW'\"" >/dev/null
else
psql_admin "-c \"create role $DB login password '$PW'\"" >/dev/null
fi
[ "$(psql_admin "-c \"select 1 from pg_database where datname='$DB'\"")" = 1 ] || psql_admin "-c \"create database $DB owner $DB\"" >/dev/null
chmod 600 "$ENVF"
set -a; . "$ENVF"; set +a
PGPASSWORD=$PW psql -h 127.0.0.1 -p "$PGPORT" -U "$DB" -d "$DB" -qtAc 'select 1' >/dev/null || { echo "cannot log in to postgres as $DB" >&2; exit 1; }
# port must be free, or held by our own service
if ss -ltnH "sport = :$PORT" | grep -q . && ! systemctl is-active --quiet "$APP"; then
echo "port $PORT is in use by another process" >&2; exit 1
fi
# ---- 2. release dir + data build ------------------------------------------------------
REL="$ROOT/releases/$SHA"
mkdir -p "$ROOT/releases" "$ROOT/cache/raw" "$ROOT/cache/export" "$ROOT/cache/next"
rm -rf "$REL"; mkdir -p "$REL"
tar --exclude=.git --exclude=node_modules --exclude=.next -cf - . | tar -xf - -C "$REL"
cd "$REL"
mkdir -p build data
ln -sfn "$ROOT/cache/raw" data/raw # raw downloads survive between releases
# the export only depends on the data scripts, the hand labels and the raw sources
KEY=$( { cat scripts/fetch_data.sh scripts/ingest.py scripts/taxonomy.py scripts/build_worktypes.py scripts/export_jsonl.py scripts/worktypes.json
find data/labels -type f | sort | xargs cat; } | sha256sum | cut -c1-16)
EXPORT="$ROOT/cache/export/$KEY"
if [ "${REFRESH_DATA:-false}" != true ] && [ -f "$EXPORT/.complete" ]; then
echo "data: reusing cached export $KEY (no data script changed; REFRESH_DATA is not set)"
else
echo "data: building export $KEY (refresh=${REFRESH_DATA:-false})"
[ -d data/raw/acl-anthology ] && git -C data/raw/acl-anthology pull --ff-only -q || true
bash scripts/fetch_data.sh
python3 scripts/ingest.py --raw data/raw --out build/papers.db
python3 scripts/build_worktypes.py --db build/papers.db
python3 scripts/export_jsonl.py --db build/papers.db --out build/export
rm -rf "$EXPORT"; cp -a build/export "$EXPORT"; touch "$EXPORT/.complete"
ls -1dt "$ROOT"/cache/export/* | tail -n +3 | xargs -r rm -rf # keep the two newest
fi
EXPORT_ID=$(find "$EXPORT" -type f ! -name .complete | sort | xargs cat | sha256sum | cut -c1-16)
# ---- 3. site build + load into Postgres ----------------------------------------------
LOCK=$( { cat package-lock.json; node -v; } | sha256sum | cut -c1-16)
NM="$ROOT/cache/node_modules/$LOCK"
if [ -f "$NM/.complete" ]; then
echo "node_modules: reusing cached install $LOCK (lockfile and node version unchanged)"
cp -al "$NM" node_modules && rm -f node_modules/.complete # hard links: seconds, no extra disk
else
npm ci --no-audit --no-fund
mkdir -p "$ROOT/cache/node_modules"; rm -rf "$NM"
cp -al node_modules "$NM" && touch "$NM/.complete"
ls -1dt "$ROOT"/cache/node_modules/* | tail -n +3 | xargs -r rm -rf # keep the two newest
fi
mkdir -p .next; ln -sfn "$ROOT/cache/next" .next/cache # compiler cache outlives the release
export NEXT_TELEMETRY_DISABLED=1
npm run build
LIVE=$(PGPASSWORD=$PW psql -h 127.0.0.1 -p "$PGPORT" -U "$DB" -d "$DB" -qtAc "select 1 from information_schema.schemata where schema_name='live'" 2>/dev/null || true)
if [ "$LIVE" = 1 ] && [ "$(cat "$ROOT/cache/loaded" 2>/dev/null)" = "$EXPORT_ID" ]; then
echo "postgres: schema live already holds export $EXPORT_ID, load skipped"
else
node scripts/load-pg.js --dir "$EXPORT"
echo "$EXPORT_ID" > "$ROOT/cache/loaded"
fi
rm -rf build
# ---- 4. switch + restart --------------------------------------------------------------
ln -sfn "$REL" "$ROOT/current.new" && mv -T "$ROOT/current.new" "$ROOT/current"
cat > /etc/systemd/system/$APP.service <<UNIT
[Unit]
Description=LLM research survey (Next.js + PostgreSQL)
After=network.target postgresql.service
Wants=postgresql.service
[Service]
WorkingDirectory=$ROOT/current
EnvironmentFile=$ENVF
Environment=NODE_ENV=production PORT=$PORT
ExecStart=$(command -v node) node_modules/next/dist/bin/next start -p $PORT -H 0.0.0.0
Restart=always
RestartSec=3
[Install]
WantedBy=multi-user.target
UNIT
systemctl daemon-reload
systemctl enable "$APP" >/dev/null 2>&1
systemctl restart "$APP"
ok=0
for i in 1 2 3 4 5 6 7 8 9 10; do
if curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" && curl -fsS -o /dev/null "http://127.0.0.1:$PORT/papers?q=agent"; then ok=1; break; fi
sleep 2
done
[ "$ok" = 1 ] || { journalctl -u "$APP" -n 60 --no-pager; exit 1; }
echo "healthy on :$PORT ($SHA); postgres on :$PGPORT db=$DB"
echo "public ip: $(curl -fsS -m 5 https://api.ipify.org 2>/dev/null || echo unknown)"
ls -1dt "$ROOT"/releases/* | tail -n +4 | xargs -r rm -rf