Rebuild lean-bench: pages in, verified indexed Lean out
ci / test (push) Failing after 12s
ci / lean-project (push) Successful in 5s
docker / image (push) Failing after 5s

Replaces the previous tree entirely. The pipeline is four separable pieces, each
testable on its own:

1. leanbench/llm.py      provider-neutral OpenAI-compatible connector, stdlib only,
                        with retries, JSON coercion and a scripted client so the
                        rest of the code is testable without a network.
2. leanbench/detect.py   per-page detection of formalizable mathematics. Case 1 is
                        mathematics asserted as fact (definition/theorem/lemma/...,
                        deliberately not sub-classified); case 2 is a problem left
                        open to the reader. Hits come back as spans -- start page
                        and index, end page and index -- so several per page is
                        normal. A hit cut off by the page end makes the detector
                        re-read that page together with the next, repeatedly, so an
                        item spread over three pages is found whole.
                        Offsets come from verbatim quotes resolved by
                        leanbench/anchors.py (exact -> whitespace-insensitive ->
                        fuzzy), because models quote well and count badly.
3. leanbench/formalize.py + leanbench/verify.py
                        each span becomes one Lean 4 declaration, judged by the real
                        compiler: full elaboration for case 1, statement legality
                        only for case 2, which has no proof to check. Lean's own
                        diagnostics feed back for a repair attempt. "verified" (Lean
                        accepted it) and "proved" (accepted with no sorry) are kept
                        strictly apart.
4. leanbench/db.py       indexed SQLite plus JSONL export. Every item is indexed by
                        the types of its inputs, or by its own type when it is not a
                        proof, and records the source book (sha256) and the page it
                        starts on -- so `drop --source H --pages A-B` followed by a
                        re-ingest redoes exactly those pages and nothing else.

Also: lean/ Lake project pinned to Mathlib, docker/ image carrying Lean 4 and a
prebuilt Mathlib (the Mathlib revision and the toolchain are resolved against each
other at build time rather than both hardcoded), and .gitea/workflows for tests,
image build and push to the Gitea registry, and a manual "submit a book" run.

58 tests; they need neither network nor Lean.
This commit is contained in:
Zayden-Jung
2026-09-19 07:05:52 +00:00
commit d3555d9a52
37 changed files with 4251 additions and 0 deletions
+81
View File
@@ -0,0 +1,81 @@
from __future__ import annotations
from leanbench.typeindex import ROLE_GOAL, ROLE_INPUT, ROLE_OWN, index_types, parse_signature
def test_theorem_is_indexed_by_the_types_of_its_inputs() -> None:
code = "theorem add_dvd (a b c : Int) (h : a ∣ b) (k : a ∣ c) : a ∣ b + c := by\n sorry"
signature = parse_signature(code)
assert signature is not None
assert signature.is_proof
assert signature.name == "add_dvd"
assert "Int" in signature.inputs
pairs = index_types(code)
assert ("Int", ROLE_INPUT) in pairs
def test_definition_is_indexed_by_its_own_type() -> None:
code = "def divides (a b : Int) : Prop := ∃ k : Int, b = a * k"
signature = parse_signature(code)
assert signature is not None
assert not signature.is_proof
pairs = index_types(code)
assert ("Int", ROLE_INPUT) in pairs
assert ("Prop", ROLE_OWN) in pairs, "a predicate's own type is the point of it"
assert all(role != ROLE_GOAL for _, role in pairs)
def test_a_proof_is_not_indexed_by_Prop() -> None:
pairs = index_types("theorem t (n : Nat) : n = n := rfl")
assert all(name != "Prop" for name, _ in pairs)
def test_definition_of_a_concrete_type_records_it() -> None:
pairs = index_types("def double (n : Nat) : Nat := 2 * n")
assert ("Nat", ROLE_OWN) in pairs
def test_instance_binders_count_as_inputs() -> None:
code = "theorem foo {G : Type*} [Group G] (x : G) : x * 1 = x := by simp"
signature = parse_signature(code)
assert signature is not None
assert "Group" in signature.inputs
def test_unicode_number_types_are_normalised() -> None:
code = "theorem sq_nonneg' (n : ℤ) : 0 ≤ n ^ 2 := by positivity"
pairs = dict(index_types(code))
assert pairs.get("Int") == ROLE_INPUT
def test_structure_records_its_own_type() -> None:
code = "structure Monoid' (M : Type) : Type where\n one : M"
signature = parse_signature(code)
assert signature is not None
assert signature.keyword == "structure"
assert signature.own or signature.inputs
def test_model_supplied_types_are_merged_without_duplicates() -> None:
code = "theorem t (n : Nat) : n = n := rfl"
pairs = index_types(code, ["Nat", "Polynomial", " "])
names = [name for name, _ in pairs]
assert names.count("Nat") == 1
assert "Polynomial" in names
def test_comments_do_not_leak_into_the_index() -> None:
code = "-- about Matrix stuff\n/- Group -/\ntheorem t (n : Nat) : n = n := rfl"
names = [name for name, _ in index_types(code)]
assert "Matrix" not in names and "Group" not in names
def test_declaration_keyword_detected_after_attributes() -> None:
code = "@[simp]\nprivate theorem t (n : Nat) : n = n := rfl"
signature = parse_signature(code)
assert signature is not None and signature.keyword == "theorem"
def test_unparsable_code_yields_nothing_rather_than_nonsense() -> None:
assert parse_signature("this is not lean") is None
assert index_types("this is not lean") == []