"""End-to-end smoke test for the image, with the real Lean compiler attached. The LLM is scripted (the image has no API key and CI should not need one), but everything else is real: pages are extracted, spans are resolved to page/index, Lean elaborates the generated code, and the items land in an indexed database. Run it against a built image without mounting anything: docker run --rm -i --entrypoint python3 lean-bench:latest - < docker/smoke.py """ from __future__ import annotations import json import os import sys import tempfile from leanbench.config import Settings from leanbench.db import Database from leanbench.llm import ScriptedClient from leanbench.pipeline import build_checker, ingest BOOK_PAGE_ONE = "Theorem 1. For every natural number n, n + 0 = n.\n" BOOK_PAGE_TWO = "Exercise 2. Show that 2 + 2 = 4.\n" def detection(items: list[dict]) -> str: return json.dumps({"items": items}) def main() -> int: workdir = tempfile.mkdtemp(prefix="leanbench-smoke-") book = os.path.join(workdir, "book.txt") with open(book, "w", encoding="utf-8") as stream: stream.write(BOOK_PAGE_ONE + "\f" + BOOK_PAGE_TWO) client = ScriptedClient( [ detection( [ { "kind": "statement", "label": "Theorem 1", "summary": "adding zero changes nothing", "start_quote": "Theorem 1. For every natural number n", "end_quote": "n + 0 = n.", "continues_past_end": False, } ] ), detection( [ { "kind": "problem", "label": "Exercise 2", "summary": "two plus two", "start_quote": "Exercise 2. Show that", "end_quote": "2 + 2 = 4.", "continues_past_end": False, } ] ), json.dumps( { "skip": False, "imports": ["Mathlib"], "name": "add_zero_nat", "lean_code": "theorem add_zero_nat (n : Nat) : n + 0 = n := by simp", "has_proof": True, "informal": "n + 0 = n", "index_types": ["Nat"], "confidence": 1.0, } ), json.dumps( { "skip": False, "imports": ["Mathlib"], "name": "two_add_two", "lean_code": "theorem two_add_two : (2 : Nat) + 2 = 4 := by sorry", "has_proof": False, "informal": "2 + 2 = 4", "index_types": ["Nat"], "confidence": 1.0, } ), ] ) settings = Settings.from_env(dotenv=None) settings.database = os.path.join(workdir, "smoke.sqlite") settings.repair_attempts = 0 checker = build_checker(settings) if not getattr(checker, "available", lambda: False)(): print("FAIL: this image has no working Lean; verification would be skipped") return 1 database = Database(settings.database) result = ingest( book, settings=settings, client=client, database=database, checker=checker, title="smoke", ) summary = result.summary() print(json.dumps(summary, indent=2)) failures = [] if summary["detected"] != 2: failures.append(f"expected 2 detections, got {summary['detected']}") if summary["verified"] != 2: failures.append(f"Lean should have accepted both, got {summary['verified']}") if summary["proved"] != 1: failures.append(f"exactly one item carries a real proof, got {summary['proved']}") if not database.search(types=["Nat"]): failures.append("the type index is empty") problems = database.search(kind="problem") if len(problems) != 1 or problems[0]["verify_mode"] != "statement": failures.append("the open problem should be checked as a statement only") database.close() for line in failures: print("FAIL:", line) if failures: return 1 print("pipeline + lean end to end: ok") return 0 if __name__ == "__main__": sys.exit(main())