Retain verified wiki releases and expire unused CI images

scripts/deploy.sh

@@ -17,7 +17,7 @@ test "$(git -c safe.directory="$PWD" rev-parse HEAD)" = "$RELEASE_SHA"
 LATEST_MAIN=$(git -c safe.directory="$PWD" ls-remote origin refs/heads/main | cut -f1)
 if [ "$LATEST_MAIN" != "$RELEASE_SHA" ]; then
   git -c safe.directory="$PWD" fetch origin main
-  if ! git -c safe.directory="$PWD" diff --quiet "$RELEASE_SHA" origin/main -- apps/waterfall-wiki Dockerfile compose.yml scripts/deploy.sh scripts/infra.sh scripts/integration.sh scripts/auth-admin.sh .gitea/workflows/waterfall-wiki.yml pnpm-workspace.yaml; then
+  if ! git -c safe.directory="$PWD" diff --quiet "$RELEASE_SHA" origin/main -- apps/waterfall-wiki Dockerfile compose.yml scripts/deploy.sh scripts/wiki-image-lifecycle.py scripts/infra.sh scripts/integration.sh scripts/auth-admin.sh .gitea/workflows/waterfall-wiki.yml pnpm-workspace.yaml; then
     echo 'Skipping deployment: Wiki has a newer commit'; exit 0
   fi
 fi
@@ -135,6 +135,10 @@ if command -v systemctl >/dev/null; then
   if systemctl is-active --quiet prompt.service || systemctl is-active --quiet prompt-worker.service; then echo 'Legacy host service is unexpectedly running'; false; fi
 fi
 echo "Full Docker deployment verified: web+worker+postgres+objects port=$PORT release=$RELEASE_SHA"
+# Register only after every deployment check succeeded; ledger failure retains old images.
+if ! python3 scripts/wiki-image-lifecycle.py record-success --sha "$RELEASE_SHA"; then
+  echo "Warning: successful-release ledger update failed; runtime cleanup stays conservative" >&2
+fi
 # Read-only network diagnostics: a private address alone is not a public entrypoint.
 echo "Host: $(hostname)"
 hostname -I || true

scripts/wiki-image-lifecycle.py

@@ -0,0 +1,210 @@
+#!/usr/bin/env python3
+"""Wiki-only image retention. Preview is the default; never prune the daemon."""
+import argparse
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import fcntl
+import json
+import os
+from pathlib import Path
+import re
+import subprocess
+import sys
+
+SHA = re.compile(r"^[0-9a-f]{40}$")
+PROJECT = "io.entroia.project"
+ROOT = Path("/opt/prompt")
+TTL = 3 * 86400
+
+
+def run(args, **kwargs):
+    return subprocess.run(args, check=True, text=True, **kwargs)
+
+
+def output(args):
+    return run(args, stdout=subprocess.PIPE).stdout
+
+
+def timestamp(value):
+    return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp()
+
+
+@contextmanager
+def lock(path):
+    # Fail rather than wait behind a build and consume the workflow timeout.
+    with open(path, "a") as handle:
+        try:
+            fcntl.flock(handle, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except BlockingIOError:
+            raise RuntimeError(f"Busy: {path}; no cleanup performed")
+        yield
+
+
+def history(root):
+    path = root / "successful-releases"
+    values = path.read_text().splitlines() if path.exists() else []
+    if any(not SHA.fullmatch(value) for value in values):
+        raise RuntimeError("Invalid successful-releases ledger; refusing cleanup")
+    return list(dict.fromkeys(values))
+
+
+def current_sha(root):
+    path = root / "current" / "release.env"
+    if not path.is_file():
+        raise RuntimeError("Missing current/release.env; refusing cleanup")
+    values = [line.split("=", 1)[1] for line in path.read_text().splitlines()
+              if line.startswith("RELEASE_SHA=")]
+    if len(values) != 1 or not SHA.fullmatch(values[0]):
+        raise RuntimeError("Invalid current release SHA; refusing cleanup")
+    return values[0]
+
+
+def record(root, sha):
+    if current_sha(root) != sha:
+        raise RuntimeError("Only the verified current release can be recorded")
+    values = [sha] + [item for item in history(root) if item != sha]
+    temp = root / ".successful-releases.next"
+    temp.write_text("\n".join(values) + "\n")
+    os.replace(temp, root / "successful-releases")
+
+
+def decision(image, referenced, retained, ready, now, completed_test=None):
+    """Return keep/delete with a reviewable reason; all tags must be in scope."""
+    tags = image.get("RepoTags") or []
+    labels = image.get("Config", {}).get("Labels") or {}
+    if image["Id"] in referenced:
+        return "keep", "container reference (including stopped containers)"
+    parsed = []
+    for tag in tags:
+        repo, _, sha = tag.rpartition(":")
+        if repo not in ("waterfall-wiki", "waterfall-wiki-tests") or not SHA.fullmatch(sha):
+            return "keep", "foreign or non-release tag"
+        parsed.append((repo, sha))
+    if any(digest.partition("@")[0] not in ("waterfall-wiki", "waterfall-wiki-tests")
+           for digest in image.get("RepoDigests") or []):
+        return "keep", "foreign registry digest retained conservatively"
+    if any(sha in retained for _, sha in parsed):
+        return "keep", "retained release or its operational test image"
+    if any(repo == "waterfall-wiki" for repo, _ in parsed) and not ready:
+        return "keep", "bootstrap: fewer than three successful releases recorded"
+    if parsed and all(repo == "waterfall-wiki-tests" and sha == completed_test
+                      for repo, sha in parsed):
+        return "delete", "this job's tests completed successfully"
+    if now - timestamp(image["Created"]) < TTL:
+        return "keep", "three-day build/debug window"
+    if parsed:
+        return "delete", "expired project release/test image"
+    if labels.get(PROJECT) == "waterfall-wiki":
+        return "delete", "expired project-labelled intermediate image"
+    return "keep", "unlabelled image: ownership unknown"
+
+
+def container_expired(container, now):
+    labels = container.get("Config", {}).get("Labels") or {}
+    return (labels.get(PROJECT) == "waterfall-wiki"
+            and labels.get("io.entroia.wiki.build") == "true"
+            and container["State"]["Status"] == "exited"
+            and not container.get("Mounts")
+            and now - timestamp(container["Created"]) >= TTL)
+
+
+def containers():
+    ids = output(["docker", "ps", "-aq"]).split()
+    return json.loads(output(["docker", "inspect", *ids])) if ids else []
+
+
+def image_ids():
+    # Include labelled build intermediates, but never enumerate foreign image metadata.
+    ids = set()
+    for name in ("waterfall-wiki", "waterfall-wiki-tests"):
+        ids.update(output(["docker", "image", "ls", "-aq", "--no-trunc", name]).split())
+    ids.update(output(["docker", "image", "ls", "-aq", "--no-trunc", "--filter",
+                       f"label={PROJECT}=waterfall-wiki"]).split())
+    return sorted(ids)
+
+
+def cleanup(root, apply=False, completed_test=None):
+    # CI holds the image lock throughout build, integration, deployment and cleanup.
+    # Taking the deployment lock also protects direct/manual deployment invocations.
+    with lock(root / ".deploy.lock"):
+        successes = history(root)
+        retained = set(successes[:3]) | {current_sha(root)}
+        ready = len(successes) >= 3
+        now = datetime.now(timezone.utc).timestamp()
+        print("APPLY" if apply else "PREVIEW", "retained releases:", sorted(retained), flush=True)
+        for container in containers():
+            if container_expired(container, now):
+                print("delete-container", container["Id"], "expired, project-labelled, no mounts", flush=True)
+                if apply:
+                    fresh = json.loads(output(["docker", "inspect", container["Id"]]))[0]
+                    if container_expired(fresh, now):
+                        run(["docker", "container", "rm", container["Id"]])
+        referenced = {item["Image"] for item in containers()}
+        ids = image_ids()
+        images = json.loads(output(["docker", "image", "inspect", *ids])) if ids else []
+        for image in images:
+            image_id = image["Id"]
+            action, reason = decision(image, referenced, retained, ready, now, completed_test)
+            print(action, image_id, json.dumps(image.get("RepoTags") or []), reason, flush=True)
+            if apply and action == "delete":
+                # Recheck references and tags immediately before removal. No -f or volume operations.
+                fresh = json.loads(output(["docker", "image", "inspect", image_id]))[0]
+                refs = {item["Image"] for item in containers()}
+                if decision(fresh, refs, retained, ready, now, completed_test)[0] != "delete":
+                    continue
+                for target in fresh.get("RepoTags") or [image_id]:
+                    result = subprocess.run(["docker", "image", "rm", target], text=True)
+                    if result.returncode:
+                        print("skip", target, "Docker refused removal; retained", flush=True)
+                        break
+
+
+def ci(root, sha):
+    """One lock spans the whole job; failed build images get a three-day grace."""
+    env = dict(os.environ, WATERFALL_WIKI_IMAGE=f"waterfall-wiki:{sha}",
+               WATERFALL_WIKI_TEST_IMAGE=f"waterfall-wiki-tests:{sha}")
+    code = 1
+    try:
+        run(["docker", "compose", "--env-file", "apps/waterfall-wiki/.env.example", "config", "--quiet"])
+        run(["docker", "build", "--target", "integration", "-t", env["WATERFALL_WIKI_TEST_IMAGE"], "."])
+        run(["docker", "run", "--rm", "--entrypoint", "node", env["WATERFALL_WIKI_TEST_IMAGE"],
+             "--test", "tests/character-pack.test.mjs"])
+        run(["docker", "build", "--target", "runtime", "-t", env["WATERFALL_WIKI_IMAGE"], "."])
+        run(["bash", "scripts/integration.sh"], env=env)
+        run(["bash", "scripts/deploy.sh", sha], env=env)
+        code = 0
+    except subprocess.CalledProcessError as error:
+        code = error.returncode
+    finally:
+        try:
+            cleanup(root, apply=True, completed_test=sha if code == 0 else None)
+        except Exception as error:
+            # Housekeeping must not mask the build/deployment result or roll back healthy services.
+            print(f"Cleanup failed; images retained for later retry: {error}", file=sys.stderr)
+    return code
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("command", nargs="?", choices=("preview", "apply", "record-success", "ci"), default="preview")
+    parser.add_argument("--sha")
+    args = parser.parse_args()
+    if args.command in ("ci", "record-success") and not SHA.fullmatch(args.sha or ""):
+        parser.error("--sha must be a full lowercase Git SHA")
+    if args.command == "record-success":
+        # Called at the end of deploy.sh while its deployment lock is still held.
+        record(ROOT, args.sha)
+        return 0
+    with lock(ROOT / ".image-lifecycle.lock"):
+        if args.command == "ci":
+            return ci(ROOT, args.sha)
+        cleanup(ROOT, apply=args.command == "apply")
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        sys.exit(main())
+    except Exception as error:
+        print(str(error), file=sys.stderr)
+        sys.exit(1)