Rebuild lean-bench: pages in, verified indexed Lean out

Replaces the previous tree entirely. The pipeline is four separable pieces, each testable on its own: 1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only, with retries, JSON coercion and a scripted client so the rest of the code is testable without a network. 2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is mathematics asserted as fact (definition/theorem/lemma/..., deliberately not sub-classified); case 2 is a problem left open to the reader. Hits come back as spans -- start page and index, end page and index -- so several per page is normal. A hit cut off by the page end makes the detector re-read that page together with the next, repeatedly, so an item spread over three pages is found whole. Offsets come from verbatim quotes resolved by leanbench/anchors.py (exact -> whitespace-insensitive -> fuzzy), because models quote well and count badly. 3. leanbench/formalize.py + leanbench/verify.py each span becomes one Lean 4 declaration, judged by the real compiler: full elaboration for case 1, statement legality only for case 2, which has no proof to check. Lean's own diagnostics feed back for a repair attempt. "verified" (Lean accepted it) and "proved" (accepted with no sorry) are kept strictly apart. 4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by the types of its inputs, or by its own type when it is not a proof, and records the source book (sha256) and the page it starts on -- so `drop --source H --pages A-B` followed by a re-ingest redoes exactly those pages and nothing else. Infrastructure: - lean/ Lake project pinned to Mathlib. - docker/ image carrying Lean 4 and a prebuilt Mathlib. The Mathlib revision and the toolchain are resolved against each other at build time rather than both hardcoded, so the build does not rot. docker/smoke.py runs the whole pipeline inside the image against the real compiler, with the LLM scripted. - .gitea/workflows ci (lint + tests in a venv, since the runner image's Python is Debian-managed and refuses installs into itself), docker (build, verify, optionally push), ingest (submit a book by hand). The docker workflow is not a throwaway build: it leaves the image, the layer cache, an idle `lean-bench` container and the `lean-bench-data` / `lean-bench-books` volumes in place on the runner, and never prunes, so the environment is there to be used afterwards. Pushing to the registry is optional and cannot fail the build -- Gitea's container registry does not accept the built-in Actions token, so that needs a REGISTRY_TOKEN secret with write:package. 58 tests; they need neither network nor Lean. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ci5T3gWDhhmqDec3e1XtYZ
2026-09-19 07:16:37 +00:00
commit b9e37952e6
38 changed files with 4548 additions and 0 deletions
@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from leanbench.db import Database, ItemRow, item_id
+
+
+def row(**overrides) -> ItemRow:
+    data = dict(
+        id="",
+        source_hash="book-hash-a",
+        kind="statement",
+        label="Theorem 1.6.2",
+        summary="divisibility is additive",
+        informal="a | b and a | c implies a | b + c",
+        start_page=12,
+        start_index=100,
+        end_page=12,
+        end_index=260,
+        page_checksum="abc",
+        source_text="Theorem 1.6.2 ...",
+        lean_name="dvd_add'",
+        lean_code="theorem dvd_add' (a b c : Int) : True := trivial",
+        imports=["Mathlib"],
+        has_proof=True,
+        verified=True,
+        proved=True,
+        verify_mode="full",
+        diagnostics="",
+        confidence=0.9,
+        match_method="exact",
+        match_score=1.0,
+        truncated=False,
+        types=[("Int", "input")],
+    )
+    data.update(overrides)
+    if not data["id"]:
+        data["id"] = item_id(data["source_hash"], data["start_page"], data["start_index"],
+                             data["end_page"], data["end_index"])
+    return ItemRow(**data)
+
+
+@pytest.fixture()
+def database(tmp_path: Path) -> Database:
+    db = Database(tmp_path / "test.sqlite")
+    db.upsert_source("book-hash-a", title="Algebra", path="a.pdf", pages=400)
+    yield db
+    db.close()
+
+
+def test_item_id_is_stable_for_the_same_span() -> None:
+    first = item_id("h", 1, 2, 3, 4)
+    assert first == item_id("h", 1, 2, 3, 4)
+    assert first != item_id("h", 1, 2, 3, 5)
+
+
+def test_store_and_read_back(database: Database) -> None:
+    identifier = database.upsert_item(row())
+    stored = database.get(identifier)
+    assert stored is not None
+    assert stored["imports"] == ["Mathlib"]
+    assert stored["types"] == [{"name": "Int", "role": "input"}]
+
+
+def test_reingesting_the_same_span_updates_rather_than_duplicates(database: Database) -> None:
+    database.upsert_item(row())
+    database.upsert_item(row(lean_name="renamed", types=[("Nat", "input")]))
+    assert database.stats()["items"] == 1
+    stored = database.get(item_id("book-hash-a", 12, 100, 12, 260))
+    assert stored is not None and stored["lean_name"] == "renamed"
+    assert stored["types"] == [{"name": "Nat", "role": "input"}], "stale types are dropped"
+
+
+def test_search_by_type_is_the_point(database: Database) -> None:
+    database.upsert_item(row(types=[("Int", "input")]))
+    database.upsert_item(row(start_index=500, end_index=700, kind="problem",
+                             types=[("Polynomial", "input"), ("Int", "input")]))
+    assert len(database.search(types=["Int"])) == 2
+    assert len(database.search(types=["Polynomial"])) == 1
+    assert len(database.search(types=["Int", "Polynomial"], match_all_types=True)) == 1
+    assert len(database.search(types=["Int"], kind="problem")) == 1
+
+
+def test_search_by_role_separates_definitions_from_theorems(database: Database) -> None:
+    database.upsert_item(row(types=[("Ideal", "own")]))
+    database.upsert_item(row(start_index=900, end_index=950, types=[("Ideal", "input")]))
+    assert len(database.search(types=["Ideal"], roles=["own"])) == 1
+    assert len(database.search(types=["Ideal"], roles=["input"])) == 1
+
+
+def test_search_by_page_finds_items_spanning_it(database: Database) -> None:
+    database.upsert_item(row(start_page=12, end_page=14))
+    assert len(database.search(page=13)) == 1
+    assert len(database.search(page=20)) == 0
+
+
+def test_dropping_a_page_range_lets_you_redo_part_of_a_book(database: Database) -> None:
+    database.upsert_item(row(start_page=12))
+    database.upsert_item(row(start_page=40, start_index=1, end_page=40, end_index=50))
+    removed = database.delete_pages("book-hash-a", 10, 20)
+    assert removed == 1
+    assert database.stats()["items"] == 1
+    assert database.search(page=40)
+
+
+def test_dropping_a_source_takes_its_items_with_it(database: Database) -> None:
+    database.upsert_item(row())
+    database.delete_source("book-hash-a")
+    assert database.stats()["items"] == 0
+    assert database.sources() == []
+
+
+def test_export_jsonl_leaves_out_bulk_by_default(database: Database, tmp_path: Path) -> None:
+    database.upsert_item(row())
+    out = tmp_path / "items.jsonl"
+    assert database.export_jsonl(out) == 1
+    record = json.loads(out.read_text(encoding="utf-8").splitlines()[0])
+    assert "source_text" not in record and "diagnostics" not in record
+    assert record["types"] == [{"name": "Int", "role": "input"}]
+    assert database.export_jsonl(out, include_source_text=True) == 1
+    assert "source_text" in json.loads(out.read_text(encoding="utf-8").splitlines()[0])
+
+
+def test_stats_counts_the_things_that_matter(database: Database) -> None:
+    database.upsert_item(row())
+    database.upsert_item(row(start_index=2, kind="problem", proved=False, verified=True))
+    stats = database.stats()
+    assert stats["items"] == 2 and stats["statements"] == 1 and stats["problems"] == 1
+    assert stats["verified"] == 2 and stats["proved"] == 1
+    assert database.type_counts()[0]["name"] == "Int"