#!/bin/sh # Permanent deploy on the ces-control host, fully driven by CI: # 1. install + provision PostgreSQL (apt, systemd; role/db created once, password generated once) # 2. fetch raw sources (cached), build a staging SQLite with the Python scripts, export it to JSON lines # 3. build the site, load the data into Postgres (schema "next" -> swapped in as "live") # 4. switch /opt/llm-survey/current, (re)start systemd unit `llm-survey`, health-check # Port is never 3000 or 18131 (taken). # Caches under $ROOT/cache (a deploy that changes neither the data scripts nor the lockfile takes about 2 minutes, not 8): # export/ the JSON-lines export, keyed by a hash of the data scripts + data/labels (rebuilt when they change, # or when REFRESH_DATA=true: nightly run / manual run, which also re-download the raw sources) # loaded id of the export now in Postgres schema "live" (the load is skipped when it matches) # next Next.js compiler cache, linked into .next/cache # node_modules/ hard-linked copy of the installed dependencies, keyed by package-lock.json + node version. set -eu SHA="${1:?usage: deploy.sh }" APP=llm-survey ROOT=/opt/$APP ENVF=/etc/$APP/db.env PORT="${APP_PORT:-18231}" DB=llm_survey case "$PORT" in 3000|18131) echo "refusing reserved port $PORT" >&2; exit 1;; esac for c in node python3 git curl bash; do command -v $c >/dev/null || { echo "$c is not installed on this host" >&2; exit 1; }; done node -e 'process.exit(+process.versions.node.split(".")[0] >= 18 ? 0 : 1)' || { echo "need node >= 18" >&2; exit 1; } # ---- 1. PostgreSQL -------------------------------------------------------------------- if ! command -v psql >/dev/null || ! [ -d /usr/lib/postgresql ]; then echo "installing postgresql" export DEBIAN_FRONTEND=noninteractive apt-get update -qq && apt-get install -y -qq postgresql postgresql-contrib fi systemctl enable --now postgresql >/dev/null 2>&1 || true i=0; until su postgres -c "pg_isready -q" 2>/dev/null || [ $i -ge 30 ]; do i=$((i+1)); sleep 2; done su postgres -c "pg_isready -q" || { echo "postgresql did not become ready" >&2; exit 1; } PGPORT=$(su postgres -c "psql -tAc 'show port'") psql_admin() { su postgres -c "psql -v ON_ERROR_STOP=1 -qtA $*"; } mkdir -p "$(dirname "$ENVF")"; chmod 755 "$(dirname "$ENVF")" if [ ! -s "$ENVF" ]; then PW=$(head -c 64 /dev/urandom | base64 | tr -dc 'A-Za-z0-9' | head -c 28) umask 077 printf 'DATABASE_URL=postgres://%s:%s@127.0.0.1:%s/%s\n' "$DB" "$PW" "$PGPORT" "$DB" > "$ENVF" fi PW=$(sed -nE 's#^DATABASE_URL=postgres://[^:]+:([^@]+)@.*#\1#p' "$ENVF") # keep the role password and the env file in sync; create role/db only when missing if [ "$(psql_admin "-c \"select 1 from pg_roles where rolname='$DB'\"")" = 1 ]; then psql_admin "-c \"alter role $DB with login password '$PW'\"" >/dev/null else psql_admin "-c \"create role $DB login password '$PW'\"" >/dev/null fi [ "$(psql_admin "-c \"select 1 from pg_database where datname='$DB'\"")" = 1 ] || psql_admin "-c \"create database $DB owner $DB\"" >/dev/null chmod 600 "$ENVF" python3 scripts/configure-editor.py "$ENVF" set -a; . "$ENVF"; set +a PGPASSWORD=$PW psql -h 127.0.0.1 -p "$PGPORT" -U "$DB" -d "$DB" -qtAc 'select 1' >/dev/null || { echo "cannot log in to postgres as $DB" >&2; exit 1; } # port must be free, or held by our own service if ss -ltnH "sport = :$PORT" | grep -q . && ! systemctl is-active --quiet "$APP"; then echo "port $PORT is in use by another process" >&2; exit 1 fi # ---- 2. release dir + data build ------------------------------------------------------ REL="$ROOT/releases/$SHA" mkdir -p "$ROOT/releases" "$ROOT/cache/raw" "$ROOT/cache/export" "$ROOT/cache/next" rm -rf "$REL"; mkdir -p "$REL" tar --exclude=.git --exclude=node_modules --exclude=.next -cf - . | tar -xf - -C "$REL" cd "$REL" mkdir -p build data ln -sfn "$ROOT/cache/raw" data/raw # raw downloads survive between releases # the export only depends on the data scripts, the hand labels and the raw sources KEY=$( { cat scripts/fetch_data.sh scripts/ingest.py scripts/taxonomy.py scripts/build_worktypes.py scripts/export_jsonl.py scripts/worktypes.json find data/labels -type f | sort | xargs cat; } | sha256sum | cut -c1-16) EXPORT="$ROOT/cache/export/$KEY" if [ "${REFRESH_DATA:-false}" != true ] && [ -f "$EXPORT/.complete" ]; then echo "data: reusing cached export $KEY (no data script changed; REFRESH_DATA is not set)" else echo "data: building export $KEY (refresh=${REFRESH_DATA:-false})" [ -d data/raw/acl-anthology ] && git -C data/raw/acl-anthology pull --ff-only -q || true bash scripts/fetch_data.sh python3 scripts/ingest.py --raw data/raw --out build/papers.db python3 scripts/build_worktypes.py --db build/papers.db python3 scripts/export_jsonl.py --db build/papers.db --out build/export rm -rf "$EXPORT"; cp -a build/export "$EXPORT"; touch "$EXPORT/.complete" ls -1dt "$ROOT"/cache/export/* | tail -n +3 | xargs -r rm -rf # keep the two newest fi EXPORT_ID=$(find "$EXPORT" -type f ! -name .complete | sort | xargs cat | sha256sum | cut -c1-16) # ---- 3. site build + load into Postgres ---------------------------------------------- LOCK=$( { cat package-lock.json; node -v; } | sha256sum | cut -c1-16) NM="$ROOT/cache/node_modules/$LOCK" if [ -f "$NM/.complete" ]; then echo "node_modules: reusing cached install $LOCK (lockfile and node version unchanged)" cp -al "$NM" node_modules && rm -f node_modules/.complete # hard links: seconds, no extra disk else npm ci --no-audit --no-fund mkdir -p "$ROOT/cache/node_modules"; rm -rf "$NM" cp -al node_modules "$NM" && touch "$NM/.complete" ls -1dt "$ROOT"/cache/node_modules/* | tail -n +3 | xargs -r rm -rf # keep the two newest fi mkdir -p .next; ln -sfn "$ROOT/cache/next" .next/cache # compiler cache outlives the release export NEXT_TELEMETRY_DISABLED=1 npm run build LIVE=$(PGPASSWORD=$PW psql -h 127.0.0.1 -p "$PGPORT" -U "$DB" -d "$DB" -qtAc "select 1 from information_schema.schemata where schema_name='live'" 2>/dev/null || true) if [ "$LIVE" = 1 ] && [ "$(cat "$ROOT/cache/loaded" 2>/dev/null)" = "$EXPORT_ID" ]; then echo "postgres: schema live already holds export $EXPORT_ID, load skipped" else node scripts/load-pg.js --dir "$EXPORT" echo "$EXPORT_ID" > "$ROOT/cache/loaded" fi rm -rf build # ---- 4. switch + restart -------------------------------------------------------------- ln -sfn "$REL" "$ROOT/current.new" && mv -T "$ROOT/current.new" "$ROOT/current" cat > /etc/systemd/system/$APP.service </dev/null 2>&1 systemctl restart "$APP" ok=0 for i in 1 2 3 4 5 6 7 8 9 10; do if curl -fsS -o /dev/null "http://127.0.0.1:$PORT/" && curl -fsS -o /dev/null "http://127.0.0.1:$PORT/contents"; then ok=1; break; fi sleep 2 done [ "$ok" = 1 ] || { journalctl -u "$APP" -n 60 --no-pager; exit 1; } echo "healthy on :$PORT ($SHA); postgres on :$PGPORT db=$DB" echo "public ip: $(curl -fsS -m 5 https://api.ipify.org 2>/dev/null || echo unknown)" ls -1dt "$ROOT"/releases/* | tail -n +4 | xargs -r rm -rf