claude-bot
b9e37952e6
Replaces the previous tree entirely. The pipeline is four separable pieces, each
testable on its own:
1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only,
with retries, JSON coercion and a scripted client so the
rest of the code is testable without a network.
2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is
mathematics asserted as fact (definition/theorem/lemma/...,
deliberately not sub-classified); case 2 is a problem left
open to the reader. Hits come back as spans -- start page
and index, end page and index -- so several per page is
normal. A hit cut off by the page end makes the detector
re-read that page together with the next, repeatedly, so an
item spread over three pages is found whole.
Offsets come from verbatim quotes resolved by
leanbench/anchors.py (exact -> whitespace-insensitive ->
fuzzy), because models quote well and count badly.
3. leanbench/formalize.py + leanbench/verify.py
each span becomes one Lean 4 declaration, judged by the real
compiler: full elaboration for case 1, statement legality
only for case 2, which has no proof to check. Lean's own
diagnostics feed back for a repair attempt. "verified" (Lean
accepted it) and "proved" (accepted with no sorry) are kept
strictly apart.
4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by
the types of its inputs, or by its own type when it is not a
proof, and records the source book (sha256) and the page it
starts on -- so `drop --source H --pages A-B` followed by a
re-ingest redoes exactly those pages and nothing else.
Infrastructure:
- lean/ Lake project pinned to Mathlib.
- docker/ image carrying Lean 4 and a prebuilt Mathlib. The Mathlib
revision and the toolchain are resolved against each other at
build time rather than both hardcoded, so the build does not rot.
docker/smoke.py runs the whole pipeline inside the image against
the real compiler, with the LLM scripted.
- .gitea/workflows ci (lint + tests in a venv, since the runner image's Python is
Debian-managed and refuses installs into itself), docker (build,
verify, optionally push), ingest (submit a book by hand).
The docker workflow is not a throwaway build: it leaves the image, the layer cache,
an idle `lean-bench` container and the `lean-bench-data` / `lean-bench-books`
volumes in place on the runner, and never prunes, so the environment is there to be
used afterwards. Pushing to the registry is optional and cannot fail the build --
Gitea's container registry does not accept the built-in Actions token, so that needs
a REGISTRY_TOKEN secret with write:package.
58 tests; they need neither network nor Lean.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ci5T3gWDhhmqDec3e1XtYZ
134 lines
4.8 KiB
Python
134 lines
4.8 KiB
Python
from __future__ import annotations
|
|
|
|
from leanbench.detect import KIND_PROBLEM, KIND_STATEMENT, detect_document
|
|
from leanbench.document import Document
|
|
from leanbench.llm import ScriptedClient
|
|
|
|
from .conftest import detection_reply
|
|
|
|
|
|
def test_single_page_detection_reports_page_and_index(two_page_book: Document) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
detection_reply(
|
|
[
|
|
{
|
|
"kind": "statement",
|
|
"label": "Definition 1.6.1",
|
|
"summary": "divisibility",
|
|
"start_quote": "Definition 1.6.1. Let a and b be integers.",
|
|
"end_quote": "b = a k.",
|
|
"continues_past_end": False,
|
|
}
|
|
]
|
|
),
|
|
detection_reply([]),
|
|
]
|
|
)
|
|
report = detect_document(client, two_page_book, max_window_pages=3)
|
|
assert len(report.detections) == 1
|
|
found = report.detections[0]
|
|
assert found.kind == KIND_STATEMENT
|
|
assert found.start_page == 1 and found.end_page == 1
|
|
page = two_page_book.page(1).text
|
|
assert page[found.start_index : found.end_index].startswith("Definition 1.6.1")
|
|
assert page[found.start_index : found.end_index].endswith("b = a k.")
|
|
|
|
|
|
def test_item_running_off_the_page_triggers_a_second_page(two_page_book: Document) -> None:
|
|
truncated = detection_reply(
|
|
[
|
|
{
|
|
"kind": "statement",
|
|
"label": "Theorem 1.6.3",
|
|
"summary": "gcd exists",
|
|
"start_quote": "Theorem 1.6.3. Let a and b be nonzero integers.",
|
|
"end_quote": "the set of common",
|
|
"continues_past_end": True,
|
|
}
|
|
]
|
|
)
|
|
whole = detection_reply(
|
|
[
|
|
{
|
|
"kind": "statement",
|
|
"label": "Theorem 1.6.3",
|
|
"summary": "gcd exists",
|
|
"start_quote": "Theorem 1.6.3. Let a and b be nonzero integers.",
|
|
"end_quote": "the greatest common divisor.",
|
|
"continues_past_end": False,
|
|
},
|
|
{
|
|
"kind": "problem",
|
|
"label": "Exercise 1.6.5",
|
|
"summary": "odd + odd is even",
|
|
"start_quote": "Exercise 1.6.5. Prove that the sum of two odd",
|
|
"end_quote": "odd integers is even.",
|
|
"continues_past_end": False,
|
|
},
|
|
]
|
|
)
|
|
client = ScriptedClient([truncated, whole])
|
|
report = detect_document(client, two_page_book, max_window_pages=3)
|
|
|
|
assert len(client.calls) == 2, "the page should be re-read with its successor"
|
|
theorem = report.detections[0]
|
|
assert theorem.start_page == 1
|
|
assert theorem.end_page == 2, "the span must cross the page boundary"
|
|
assert theorem.spans_pages == 2
|
|
assert not theorem.truncated
|
|
assert two_page_book.page(2).text[: theorem.end_index].endswith("greatest common divisor.")
|
|
|
|
exercise = report.detections[1]
|
|
assert exercise.kind == KIND_PROBLEM
|
|
assert exercise.start_page == 2
|
|
|
|
|
|
def test_window_stops_growing_at_the_limit_and_says_so(two_page_book: Document) -> None:
|
|
always_truncated = detection_reply(
|
|
[
|
|
{
|
|
"kind": "statement",
|
|
"label": "",
|
|
"summary": "",
|
|
"start_quote": "Definition 1.6.1. Let a and b be integers.",
|
|
"end_quote": "b = a k.",
|
|
"continues_past_end": True,
|
|
}
|
|
]
|
|
)
|
|
client = ScriptedClient([always_truncated] * 4)
|
|
report = detect_document(client, two_page_book, max_window_pages=2)
|
|
assert len(client.calls) == 2
|
|
assert report.detections[0].truncated is True
|
|
|
|
|
|
def test_unresolvable_quotes_are_recorded_not_invented(two_page_book: Document) -> None:
|
|
client = ScriptedClient(
|
|
[
|
|
detection_reply(
|
|
[
|
|
{
|
|
"kind": "statement",
|
|
"label": "Hallucination 9",
|
|
"summary": "not in the book",
|
|
"start_quote": "Theorem 9.9.9 about topological quantum widgets",
|
|
"end_quote": "which is absurd.",
|
|
"continues_past_end": False,
|
|
}
|
|
]
|
|
),
|
|
detection_reply([]),
|
|
]
|
|
)
|
|
report = detect_document(client, two_page_book)
|
|
assert report.detections == []
|
|
assert len(report.unresolved) == 1
|
|
|
|
|
|
def test_a_failing_page_does_not_kill_the_run(two_page_book: Document) -> None:
|
|
client = ScriptedClient(["not json at all", detection_reply([])])
|
|
report = detect_document(client, two_page_book)
|
|
assert report.pages_scanned == 2
|
|
assert any("detector failed" in u.get("reason", "") for u in report.unresolved)
|