Rebuild lean-bench: pages in, verified indexed Lean out

Replaces the previous tree entirely. The pipeline is four separable pieces, each testable on its own: 1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only, with retries, JSON coercion and a scripted client so the rest of the code is testable without a network. 2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is mathematics asserted as fact (definition/theorem/lemma/..., deliberately not sub-classified); case 2 is a problem left open to the reader. Hits come back as spans -- start page and index, end page and index -- so several per page is normal. A hit cut off by the page end makes the detector re-read that page together with the next, repeatedly, so an item spread over three pages is found whole. Offsets come from verbatim quotes resolved by leanbench/anchors.py (exact -> whitespace-insensitive -> fuzzy), because models quote well and count badly. 3. leanbench/formalize.py + leanbench/verify.py each span becomes one Lean 4 declaration, judged by the real compiler: full elaboration for case 1, statement legality only for case 2, which has no proof to check. Lean's own diagnostics feed back for a repair attempt. "verified" (Lean accepted it) and "proved" (accepted with no sorry) are kept strictly apart. 4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by the types of its inputs, or by its own type when it is not a proof, and records the source book (sha256) and the page it starts on -- so `drop --source H --pages A-B` followed by a re-ingest redoes exactly those pages and nothing else. Also: lean/ Lake project pinned to Mathlib, docker/ image carrying Lean 4 and a prebuilt Mathlib (the Mathlib revision and the toolchain are resolved against each other at build time rather than both hardcoded), and .gitea/workflows for tests, image build and push to the Gitea registry, and a manual "submit a book" run. 58 tests; they need neither network nor Lean.
2026-09-19 07:05:52 +00:00
commit d3555d9a52
37 changed files with 4251 additions and 0 deletions
@@ -0,0 +1,72 @@
+# lean-bench: Lean 4 + Mathlib + the ingestion pipeline, in one image.
+#
+# Build from the repository root:
+#   docker build -f docker/Dockerfile -t lean-bench:latest .
+#
+# The Mathlib revision and the Lean toolchain are resolved against each other at
+# build time (see docker/resolve-mathlib.sh); override with --build-arg MATHLIB_REV=v4.33.0.
+FROM ubuntu:24.04
+
+LABEL org.opencontainers.image.title="lean-bench" \
+      org.opencontainers.image.description="Detect formalizable mathematics in books and turn it into verified, indexed Lean 4" \
+      org.opencontainers.image.source="https://gitea.entroia.com/Zayden-Jung/lean-bench"
+
+ENV DEBIAN_FRONTEND=noninteractive \
+    ELAN_HOME=/opt/elan \
+    PATH=/opt/elan/bin:$PATH \
+    PYTHONUNBUFFERED=1
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        build-essential ca-certificates curl git jq sqlite3 \
+        libgmp-dev python3 python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+# -- elan (the Lean toolchain manager) --------------------------------------
+RUN curl -fsSL https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh -o /tmp/elan-init.sh \
+    && sh /tmp/elan-init.sh -y --no-modify-path --default-toolchain none \
+    && rm /tmp/elan-init.sh \
+    && elan --version
+
+WORKDIR /opt/lean-bench
+
+# -- Lean project: toolchain, Mathlib, and its prebuilt cache ---------------
+# Its own layer, because it is the expensive one and it changes rarely.
+ARG MATHLIB_REV=auto
+COPY lean/ lean/
+COPY docker/resolve-mathlib.sh docker/resolve-mathlib.sh
+RUN chmod +x docker/resolve-mathlib.sh && docker/resolve-mathlib.sh lean "$MATHLIB_REV"
+RUN elan toolchain install "$(cat lean/lean-toolchain)" \
+    && elan default "$(cat lean/lean-toolchain)"
+RUN cd lean \
+    && lake update \
+    && (lake exe cache get || echo "no Mathlib cache available; building from source") \
+    && lake build \
+    && lake env lean --version
+
+# -- the pipeline -----------------------------------------------------------
+COPY pyproject.toml README.md ./
+COPY leanbench/ leanbench/
+RUN pip install --break-system-packages --no-cache-dir . \
+    && lean-bench --help > /dev/null
+
+COPY docker/entrypoint.sh /usr/local/bin/leanbench-entrypoint
+RUN chmod +x /usr/local/bin/leanbench-entrypoint
+
+ENV LEANBENCH_LEAN_PROJECT=/opt/lean-bench/lean \
+    LEANBENCH_LEAN_COMMAND='["lake","env","lean"]' \
+    LEANBENCH_DATABASE=/data/leanbench.sqlite \
+    LEANBENCH_LLM_MODEL=qwen-max \
+    LEANBENCH_LLM_BASE_URL=https://dashscope.aliyuncs.com/compatible-mode/v1 \
+    LEANBENCH_LLM_API_KEY_ENV=DASHSCOPE_API_KEY
+
+# /books is where you put the source you are submitting; /data holds the database.
+VOLUME ["/data", "/books"]
+WORKDIR /work
+
+# Prove the environment works while the image is still being built.
+RUN mkdir -p /work && cd /opt/lean-bench/lean \
+    && printf 'import Mathlib\ntheorem image_smoke (n : Nat) : n + 0 = n := by simp\n' > /tmp/Smoke.lean \
+    && lake env lean /tmp/Smoke.lean && rm /tmp/Smoke.lean
+
+ENTRYPOINT ["leanbench-entrypoint"]
+CMD ["doctor"]
@@ -0,0 +1,25 @@
+# Run the pipeline against a book on your machine:
+#
+#   export DASHSCOPE_API_KEY=sk-...
+#   mkdir -p books data && cp ~/my-book.pdf books/
+#   docker compose -f docker/docker-compose.yml run --rm leanbench \
+#       ingest /books/my-book.pdf --pages 30-60 --export /data/items.jsonl
+#
+# The database persists in ./data, so re-running a page range replaces just those items.
+services:
+  leanbench:
+    image: ${LEANBENCH_IMAGE:-gitea.entroia.com/zayden-jung/lean-bench:latest}
+    build:
+      context: ..
+      dockerfile: docker/Dockerfile
+      args:
+        MATHLIB_REV: ${MATHLIB_REV:-auto}
+    environment:
+      DASHSCOPE_API_KEY: ${DASHSCOPE_API_KEY:?set DASHSCOPE_API_KEY (or your provider key)}
+      LEANBENCH_LLM_MODEL: ${LEANBENCH_LLM_MODEL:-qwen-max}
+      LEANBENCH_LLM_BASE_URL: ${LEANBENCH_LLM_BASE_URL:-https://dashscope.aliyuncs.com/compatible-mode/v1}
+      LEANBENCH_DATABASE: /data/leanbench.sqlite
+    volumes:
+      - ../books:/books
+      - ../data:/data
+    command: ["doctor"]
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Thin front door for the image. `docker run <image> <lean-bench subcommand> ...`
+# works, `submit` is an alias for `ingest`, and anything else runs verbatim.
+set -euo pipefail
+
+if [ -f /run/secrets/llm_api_key ] && [ -z "${DASHSCOPE_API_KEY:-}" ]; then
+  DASHSCOPE_API_KEY="$(cat /run/secrets/llm_api_key)"
+  export DASHSCOPE_API_KEY
+fi
+
+mkdir -p "$(dirname "${LEANBENCH_DATABASE:-/data/leanbench.sqlite}")" 2>/dev/null || true
+
+case "${1:-doctor}" in
+  bash | sh)
+    shift
+    exec bash "$@"
+    ;;
+  submit)
+    shift
+    exec lean-bench ingest "$@"
+    ;;
+  doctor | ingest | detect | search | export | sources | stats | drop)
+    exec lean-bench "$@"
+    ;;
+  lean-bench)
+    shift
+    exec lean-bench "$@"
+    ;;
+  *)
+    exec "$@"
+    ;;
+esac
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Pin Mathlib and the Lean toolchain to each other.
+#
+# Mathlib tags do not exist for every Lean point release, so hardcoding both is how
+# image builds break six weeks later. Instead: choose a Mathlib revision, then take
+# *its* lean-toolchain as the truth and write both into the Lean project.
+#
+#   resolve-mathlib.sh <lean-project-dir> [revision]
+#
+# revision defaults to "auto": the newest Mathlib tag whose toolchain matches the
+# toolchain already in the project, else the newest tag, else master.
+set -euo pipefail
+
+PROJECT="${1:?usage: resolve-mathlib.sh <lean-project-dir> [revision]}"
+REQUESTED="${2:-auto}"
+REPO="https://github.com/leanprover-community/mathlib4.git"
+RAW="https://raw.githubusercontent.com/leanprover-community/mathlib4"
+TOOLCHAIN_FILE="$PROJECT/lean-toolchain"
+LAKEFILE="$PROJECT/lakefile.toml"
+
+toolchain_of() {  # toolchain_of <rev> -> prints e.g. leanprover/lean4:v4.33.0
+  curl -fsSL --retry 3 --max-time 30 "$RAW/$1/lean-toolchain" 2>/dev/null | tr -d '[:space:]'
+}
+
+candidates() {
+  git ls-remote --tags --refs "$REPO" 'v*' 2>/dev/null \
+    | awk '{print $2}' | sed 's#refs/tags/##' \
+    | grep -E '^v[0-9]+\.[0-9]+\.[0-9]+$' \
+    | sort -V -r
+}
+
+wanted="$(tr -d '[:space:]' < "$TOOLCHAIN_FILE" 2>/dev/null || true)"
+rev=""
+
+if [ "$REQUESTED" != "auto" ]; then
+  rev="$REQUESTED"
+  echo "resolve-mathlib: using requested revision $rev"
+else
+  echo "resolve-mathlib: looking for a Mathlib tag built on ${wanted:-<unset>}"
+  all="$(candidates || true)"
+  if [ -n "$wanted" ]; then
+    # only the tags sharing a major.minor with what we asked for are plausible
+    series="$(printf '%s' "$wanted" | sed -E 's#.*:v([0-9]+\.[0-9]+).*#v\1#')"
+    for tag in $(printf '%s\n' "$all" | grep -E "^${series}\." || true); do
+      if [ "$(toolchain_of "$tag")" = "$wanted" ]; then
+        rev="$tag"
+        echo "resolve-mathlib: $tag matches $wanted exactly"
+        break
+      fi
+    done
+    if [ -z "$rev" ]; then
+      rev="$(printf '%s\n' "$all" | grep -E "^${series}\." | head -1 || true)"
+      [ -n "$rev" ] && echo "resolve-mathlib: no exact match; newest in $series series is $rev"
+    fi
+  fi
+  if [ -z "$rev" ]; then
+    rev="$(printf '%s\n' "$all" | head -1 || true)"
+    echo "resolve-mathlib: falling back to newest tag ${rev:-master}"
+  fi
+  rev="${rev:-master}"
+fi
+
+actual="$(toolchain_of "$rev" || true)"
+if [ -z "$actual" ]; then
+  echo "resolve-mathlib: could not read the toolchain for $rev; keeping ${wanted:-project default}" >&2
+  actual="$wanted"
+fi
+
+printf '%s\n' "$actual" > "$TOOLCHAIN_FILE"
+# rewrite only the rev line inside the [[require]] block for mathlib
+python3 - "$LAKEFILE" "$rev" <<'PY'
+import re
+import sys
+
+path, rev = sys.argv[1], sys.argv[2]
+with open(path, encoding="utf-8") as stream:
+    text = stream.read()
+text = re.sub(r'(?m)^rev\s*=\s*".*"$', f'rev = "{rev}"', text, count=1)
+with open(path, "w", encoding="utf-8") as stream:
+    stream.write(text)
+PY
+
+echo "resolve-mathlib: mathlib $rev on $actual"