claude-bot
b9e37952e6
Replaces the previous tree entirely. The pipeline is four separable pieces, each
testable on its own:
1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only,
with retries, JSON coercion and a scripted client so the
rest of the code is testable without a network.
2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is
mathematics asserted as fact (definition/theorem/lemma/...,
deliberately not sub-classified); case 2 is a problem left
open to the reader. Hits come back as spans -- start page
and index, end page and index -- so several per page is
normal. A hit cut off by the page end makes the detector
re-read that page together with the next, repeatedly, so an
item spread over three pages is found whole.
Offsets come from verbatim quotes resolved by
leanbench/anchors.py (exact -> whitespace-insensitive ->
fuzzy), because models quote well and count badly.
3. leanbench/formalize.py + leanbench/verify.py
each span becomes one Lean 4 declaration, judged by the real
compiler: full elaboration for case 1, statement legality
only for case 2, which has no proof to check. Lean's own
diagnostics feed back for a repair attempt. "verified" (Lean
accepted it) and "proved" (accepted with no sorry) are kept
strictly apart.
4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by
the types of its inputs, or by its own type when it is not a
proof, and records the source book (sha256) and the page it
starts on -- so `drop --source H --pages A-B` followed by a
re-ingest redoes exactly those pages and nothing else.
Infrastructure:
- lean/ Lake project pinned to Mathlib.
- docker/ image carrying Lean 4 and a prebuilt Mathlib. The Mathlib
revision and the toolchain are resolved against each other at
build time rather than both hardcoded, so the build does not rot.
docker/smoke.py runs the whole pipeline inside the image against
the real compiler, with the LLM scripted.
- .gitea/workflows ci (lint + tests in a venv, since the runner image's Python is
Debian-managed and refuses installs into itself), docker (build,
verify, optionally push), ingest (submit a book by hand).
The docker workflow is not a throwaway build: it leaves the image, the layer cache,
an idle `lean-bench` container and the `lean-bench-data` / `lean-bench-books`
volumes in place on the runner, and never prunes, so the environment is there to be
used afterwards. Pushing to the registry is optional and cannot fail the build --
Gitea's container registry does not accept the built-in Actions token, so that needs
a REGISTRY_TOKEN secret with write:package.
58 tests; they need neither network nor Lean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ci5T3gWDhhmqDec3e1XtYZ
91 lines
3.5 KiB
Python
91 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
from leanbench.detect import KIND_PROBLEM, KIND_STATEMENT, Detection
|
|
from leanbench.formalize import formalize, formalize_all, verification_mode
|
|
from leanbench.llm import ScriptedClient
|
|
from leanbench.verify import MODE_FULL, MODE_STATEMENT
|
|
|
|
from .conftest import formalize_reply
|
|
|
|
|
|
def make_detection(kind: str = KIND_STATEMENT) -> Detection:
|
|
return Detection(
|
|
kind=kind,
|
|
label="Theorem 1.6.2",
|
|
summary="divisibility is additive",
|
|
start_page=12,
|
|
start_index=100,
|
|
end_page=12,
|
|
end_index=260,
|
|
text="Theorem 1.6.2. If a divides b and a divides c then a divides b + c.",
|
|
)
|
|
|
|
|
|
def test_open_problems_are_only_checked_as_statements() -> None:
|
|
assert verification_mode(make_detection(KIND_PROBLEM)) == MODE_STATEMENT
|
|
assert verification_mode(make_detection(KIND_STATEMENT)) == MODE_FULL
|
|
|
|
|
|
def test_formalize_produces_verified_code_and_types(fake_checker) -> None:
|
|
client = ScriptedClient([formalize_reply()])
|
|
result = formalize(client, make_detection(), checker=fake_checker)
|
|
assert result.lean_code.startswith("theorem test_decl")
|
|
assert result.verified
|
|
assert not result.proved, "a sorry in the code means it is not proved"
|
|
assert ("Int", "input") in result.index_types
|
|
|
|
|
|
def test_a_real_proof_counts_as_proved(fake_checker) -> None:
|
|
client = ScriptedClient(
|
|
[formalize_reply(lean_code="theorem t (n : Nat) : n = n := rfl", has_proof=True)]
|
|
)
|
|
result = formalize(client, make_detection(), checker=fake_checker)
|
|
assert result.has_proof and result.proved
|
|
|
|
|
|
def test_lean_diagnostics_drive_a_repair_attempt(fake_checker) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
formalize_reply(lean_code="theorem t : BAD := by sorry"),
|
|
formalize_reply(lean_code="theorem t (n : Nat) : n = n := rfl", has_proof=True),
|
|
]
|
|
)
|
|
result = formalize(client, make_detection(), checker=fake_checker, repair_attempts=1)
|
|
assert result.attempts == 2
|
|
assert result.verified
|
|
assert len(client.calls) == 2
|
|
repair_prompt = client.calls[1][-1]["content"]
|
|
assert "error: bad" in repair_prompt, "the compiler's own words go back to the model"
|
|
|
|
|
|
def test_repair_gives_up_after_the_budget(fake_checker) -> None:
|
|
client = ScriptedClient([formalize_reply(lean_code="theorem t : BAD := by sorry")] * 3)
|
|
result = formalize(client, make_detection(), checker=fake_checker, repair_attempts=1)
|
|
assert result.attempts == 2
|
|
assert not result.verified
|
|
|
|
|
|
def test_model_may_decline_an_unformalizable_passage(fake_checker) -> None:
|
|
client = ScriptedClient(['{"skip": true, "reason": "prose only"}'])
|
|
result = formalize(client, make_detection(), checker=fake_checker)
|
|
assert result.skipped and result.skip_reason == "prose only"
|
|
assert fake_checker.calls == []
|
|
|
|
|
|
def test_imports_inside_the_body_are_stripped(fake_checker) -> None:
|
|
client = ScriptedClient(
|
|
[formalize_reply(lean_code="import Mathlib\ntheorem t (n : Nat) : n = n := rfl")]
|
|
)
|
|
result = formalize(client, make_detection(), checker=fake_checker)
|
|
assert "import" not in result.lean_code
|
|
|
|
|
|
def test_one_bad_item_does_not_stop_the_batch(fake_checker) -> None:
|
|
client = ScriptedClient(["garbage, not json", formalize_reply()])
|
|
results = formalize_all(
|
|
client, [make_detection(), make_detection(KIND_PROBLEM)], checker=fake_checker
|
|
)
|
|
assert len(results) == 2
|
|
assert results[0].skipped and "formalizer failed" in results[0].skip_reason
|
|
assert results[1].verified
|