Rebuild lean-bench: pages in, verified indexed Lean out
Replaces the previous tree entirely. The pipeline is four separable pieces, each
testable on its own:
1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only,
with retries, JSON coercion and a scripted client so the
rest of the code is testable without a network.
2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is
mathematics asserted as fact (definition/theorem/lemma/...,
deliberately not sub-classified); case 2 is a problem left
open to the reader. Hits come back as spans -- start page
and index, end page and index -- so several per page is
normal. A hit cut off by the page end makes the detector
re-read that page together with the next, repeatedly, so an
item spread over three pages is found whole.
Offsets come from verbatim quotes resolved by
leanbench/anchors.py (exact -> whitespace-insensitive ->
fuzzy), because models quote well and count badly.
3. leanbench/formalize.py + leanbench/verify.py
each span becomes one Lean 4 declaration, judged by the real
compiler: full elaboration for case 1, statement legality
only for case 2, which has no proof to check. Lean's own
diagnostics feed back for a repair attempt. "verified" (Lean
accepted it) and "proved" (accepted with no sorry) are kept
strictly apart.
4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by
the types of its inputs, or by its own type when it is not a
proof, and records the source book (sha256) and the page it
starts on -- so `drop --source H --pages A-B` followed by a
re-ingest redoes exactly those pages and nothing else.
Also: lean/ Lake project pinned to Mathlib, docker/ image carrying Lean 4 and a
prebuilt Mathlib (the Mathlib revision and the toolchain are resolved against each
other at build time rather than both hardcoded), and .gitea/workflows for tests,
image build and push to the Gitea registry, and a manual "submit a book" run.
58 tests; they need neither network nor Lean.
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
pull_request:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Ensure Python
|
||||
run: |
|
||||
set -eu
|
||||
if ! command -v python3 >/dev/null; then
|
||||
apt-get update
|
||||
apt-get install -y --no-install-recommends python3 python3-pip python3-venv
|
||||
fi
|
||||
python3 --version
|
||||
|
||||
- name: Install
|
||||
run: |
|
||||
set -eu
|
||||
PIP="python3 -m pip"
|
||||
# Debian/Ubuntu mark the system Python as externally managed; this is a
|
||||
# throwaway CI container, so installing into it is fine.
|
||||
$PIP install --break-system-packages --upgrade pip \
|
||||
|| $PIP install --upgrade pip
|
||||
$PIP install --break-system-packages -e ".[dev]" \
|
||||
|| $PIP install -e ".[dev]"
|
||||
|
||||
- name: Lint
|
||||
run: python3 -m ruff check .
|
||||
|
||||
- name: Test
|
||||
run: python3 -m pytest -q
|
||||
|
||||
- name: Pipeline smoke test (no network, no Lean)
|
||||
run: python3 -m pytest -q tests/test_pipeline.py
|
||||
|
||||
- name: CLI is wired up
|
||||
run: |
|
||||
python3 -m leanbench.cli --help
|
||||
python3 -m leanbench.cli doctor --skip-llm --env /dev/null
|
||||
|
||||
lean-project:
|
||||
# Cheap guard: the Lean project's pins must at least resolve.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Resolve the Mathlib / toolchain pair
|
||||
run: |
|
||||
set -eu
|
||||
command -v curl >/dev/null || (apt-get update && apt-get install -y curl)
|
||||
command -v git >/dev/null || (apt-get update && apt-get install -y git)
|
||||
chmod +x docker/resolve-mathlib.sh
|
||||
docker/resolve-mathlib.sh lean "${MATHLIB_REV:-auto}"
|
||||
echo "toolchain: $(cat lean/lean-toolchain)"
|
||||
grep -n 'rev' lean/lakefile.toml
|
||||
@@ -0,0 +1,114 @@
|
||||
name: docker
|
||||
|
||||
# Builds the Lean 4 + Mathlib image and pushes it to this Gitea's container
|
||||
# registry, so `docker pull gitea.entroia.com/<owner>/lean-bench` is all anyone
|
||||
# needs in order to submit a book.
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mathlib_rev:
|
||||
description: "Mathlib revision (tag, branch, commit, or 'auto')"
|
||||
required: false
|
||||
default: "auto"
|
||||
push_image:
|
||||
description: "Push to the registry (false = build only)"
|
||||
required: false
|
||||
default: "true"
|
||||
|
||||
env:
|
||||
REGISTRY: gitea.entroia.com
|
||||
IMAGE_NAME: lean-bench
|
||||
|
||||
jobs:
|
||||
image:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Check the runner can build images
|
||||
run: |
|
||||
set -eu
|
||||
if ! command -v docker >/dev/null; then
|
||||
echo "::error::No docker CLI on this runner. Give the act_runner the host"
|
||||
echo "::error::docker socket (-v /var/run/docker.sock:/var/run/docker.sock)"
|
||||
echo "::error::or run it with the docker-in-docker image."
|
||||
exit 1
|
||||
fi
|
||||
docker version
|
||||
|
||||
- name: Work out the tags
|
||||
id: meta
|
||||
run: |
|
||||
set -eu
|
||||
owner="$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]')"
|
||||
image="${REGISTRY}/${owner}/${IMAGE_NAME}"
|
||||
ref="${GITHUB_REF_NAME:-main}"
|
||||
sha="$(echo "${GITHUB_SHA:-unknown}" | cut -c1-12)"
|
||||
case "${GITHUB_REF:-}" in
|
||||
refs/tags/*) version="${ref}" ;;
|
||||
*) version="${ref//\//-}" ;;
|
||||
esac
|
||||
{
|
||||
echo "image=${image}"
|
||||
echo "tags=${image}:${version} ${image}:sha-${sha}$([ "${ref}" = "main" ] && echo " ${image}:latest" || true)"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
echo "building ${image}:${version}"
|
||||
|
||||
- name: Log in to the registry
|
||||
if: ${{ github.event_name != 'workflow_dispatch' || inputs.push_image == 'true' }}
|
||||
run: |
|
||||
set -eu
|
||||
token='${{ secrets.GITEA_TOKEN }}'
|
||||
[ -n "$token" ] || token='${{ secrets.GITHUB_TOKEN }}'
|
||||
if [ -z "$token" ]; then
|
||||
echo "::error::No token available to log in to ${REGISTRY}."
|
||||
echo "::error::Add a GITEA_TOKEN secret with package:write scope."
|
||||
exit 1
|
||||
fi
|
||||
echo "$token" | docker login "${REGISTRY}" -u '${{ github.actor }}' --password-stdin
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
set -eu
|
||||
args=""
|
||||
for tag in ${{ steps.meta.outputs.tags }}; do args="$args -t $tag"; done
|
||||
docker build \
|
||||
-f docker/Dockerfile \
|
||||
--build-arg "MATHLIB_REV=${{ inputs.mathlib_rev || 'auto' }}" \
|
||||
$args .
|
||||
|
||||
- name: Check the image actually works
|
||||
run: |
|
||||
set -eu
|
||||
image="$(echo '${{ steps.meta.outputs.tags }}' | awk '{print $1}')"
|
||||
docker run --rm "$image" doctor --skip-llm
|
||||
docker run --rm --entrypoint bash "$image" -lc \
|
||||
'cd $LEANBENCH_LEAN_PROJECT && printf "import Mathlib\ntheorem ci_smoke (n : Nat) : n + 0 = n := by simp\n" > /tmp/CI.lean && lake env lean /tmp/CI.lean && echo "lean+mathlib ok"'
|
||||
|
||||
- name: Push
|
||||
if: ${{ github.event_name != 'workflow_dispatch' || inputs.push_image == 'true' }}
|
||||
run: |
|
||||
set -eu
|
||||
for tag in ${{ steps.meta.outputs.tags }}; do
|
||||
echo "pushing $tag"
|
||||
docker push "$tag"
|
||||
done
|
||||
|
||||
- name: Summary
|
||||
if: always()
|
||||
run: |
|
||||
{
|
||||
echo "### lean-bench image"
|
||||
echo ""
|
||||
for tag in ${{ steps.meta.outputs.tags }}; do echo "- \`$tag\`"; done
|
||||
echo ""
|
||||
echo "Pull and submit a book:"
|
||||
echo '```'
|
||||
echo "docker run --rm -e DASHSCOPE_API_KEY -v \$PWD/books:/books -v \$PWD/data:/data \\"
|
||||
echo " ${{ steps.meta.outputs.image }}:latest ingest /books/your-book.pdf --pages 1-40"
|
||||
echo '```'
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
@@ -0,0 +1,100 @@
|
||||
name: ingest
|
||||
|
||||
# Submit a book and run the whole pipeline on the runner, inside the image the
|
||||
# docker workflow built. The database and the JSONL export come back as artifacts.
|
||||
#
|
||||
# The source can be a path inside the repository or a URL the runner can reach.
|
||||
# Set the DASHSCOPE_API_KEY secret (or whichever key your provider uses) first.
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
source:
|
||||
description: "Path in the repo, or a URL, of the book to ingest"
|
||||
required: true
|
||||
pages:
|
||||
description: "Page range, e.g. 30-60 (blank = the whole book)"
|
||||
required: false
|
||||
default: ""
|
||||
title:
|
||||
description: "Human title for this source"
|
||||
required: false
|
||||
default: ""
|
||||
image:
|
||||
description: "Image to run"
|
||||
required: false
|
||||
default: "gitea.entroia.com/zayden-jung/lean-bench:latest"
|
||||
verify:
|
||||
description: "Run Lean verification (slow but the point)"
|
||||
required: false
|
||||
default: "true"
|
||||
|
||||
jobs:
|
||||
ingest:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Fetch the source
|
||||
id: source
|
||||
run: |
|
||||
set -eu
|
||||
mkdir -p books data out
|
||||
src='${{ inputs.source }}'
|
||||
case "$src" in
|
||||
http://*|https://*)
|
||||
name="$(basename "${src%%\?*}")"
|
||||
curl -fsSL --retry 3 -o "books/$name" "$src"
|
||||
;;
|
||||
*)
|
||||
[ -f "$src" ] || { echo "::error::$src is not in the repository"; exit 1; }
|
||||
name="$(basename "$src")"
|
||||
cp "$src" "books/$name"
|
||||
;;
|
||||
esac
|
||||
ls -l "books/$name"
|
||||
echo "name=$name" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Log in to the registry
|
||||
run: |
|
||||
set -eu
|
||||
token='${{ secrets.GITEA_TOKEN }}'
|
||||
[ -n "$token" ] || token='${{ secrets.GITHUB_TOKEN }}'
|
||||
echo "$token" | docker login gitea.entroia.com -u '${{ github.actor }}' --password-stdin
|
||||
|
||||
- name: Ingest
|
||||
env:
|
||||
DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
|
||||
LEANBENCH_LLM_MODEL: ${{ vars.LEANBENCH_LLM_MODEL }}
|
||||
LEANBENCH_LLM_BASE_URL: ${{ vars.LEANBENCH_LLM_BASE_URL }}
|
||||
run: |
|
||||
set -eu
|
||||
[ -n "${DASHSCOPE_API_KEY:-}" ] || { echo "::error::Set the DASHSCOPE_API_KEY secret"; exit 1; }
|
||||
docker pull '${{ inputs.image }}'
|
||||
args="ingest /books/${{ steps.source.outputs.name }} --export /data/items.jsonl --detections /data/detections.jsonl"
|
||||
[ -n "${{ inputs.pages }}" ] && args="$args --pages ${{ inputs.pages }}"
|
||||
[ -n "${{ inputs.title }}" ] && args="$args --title '${{ inputs.title }}'"
|
||||
[ '${{ inputs.verify }}' = 'true' ] || args="$args --no-verify"
|
||||
docker run --rm \
|
||||
-e DASHSCOPE_API_KEY \
|
||||
-e LEANBENCH_LLM_MODEL \
|
||||
-e LEANBENCH_LLM_BASE_URL \
|
||||
-v "$PWD/books:/books:ro" \
|
||||
-v "$PWD/data:/data" \
|
||||
'${{ inputs.image }}' \
|
||||
sh -c "lean-bench $args && lean-bench stats"
|
||||
|
||||
- name: Collect
|
||||
if: always()
|
||||
run: |
|
||||
set -eu
|
||||
cp -f data/*.jsonl out/ 2>/dev/null || true
|
||||
cp -f data/leanbench.sqlite out/ 2>/dev/null || true
|
||||
ls -l out || true
|
||||
|
||||
- uses: actions/upload-artifact@v3
|
||||
if: always()
|
||||
with:
|
||||
name: leanbench-${{ steps.source.outputs.name }}
|
||||
path: out/
|
||||
retention-days: 30
|
||||
Reference in New Issue
Block a user