lean-bench/docker/smoke.py

"""End-to-end smoke test for the image, with the real Lean compiler attached.

The LLM is scripted (the image has no API key and CI should not need one), but
everything else is real: pages are extracted, spans are resolved to page/index,
Lean elaborates the generated code, and the items land in an indexed database.

Run it against a built image without mounting anything:

    docker run --rm -i --entrypoint python3 lean-bench:latest - < docker/smoke.py
"""
from __future__ import annotations

import json
import os
import sys
import tempfile

from leanbench.config import Settings
from leanbench.db import Database
from leanbench.llm import ScriptedClient
from leanbench.pipeline import build_checker, ingest

BOOK_PAGE_ONE = "Theorem 1. For every natural number n, n + 0 = n.\n"
BOOK_PAGE_TWO = "Exercise 2. Show that 2 + 2 = 4.\n"


def detection(items: list[dict]) -> str:
    return json.dumps({"items": items})


def main() -> int:
    workdir = tempfile.mkdtemp(prefix="leanbench-smoke-")
    book = os.path.join(workdir, "book.txt")
    with open(book, "w", encoding="utf-8") as stream:
        stream.write(BOOK_PAGE_ONE + "\f" + BOOK_PAGE_TWO)

    client = ScriptedClient(
        [
            detection(
                [
                    {
                        "kind": "statement",
                        "label": "Theorem 1",
                        "summary": "adding zero changes nothing",
                        "start_quote": "Theorem 1. For every natural number n",
                        "end_quote": "n + 0 = n.",
                        "continues_past_end": False,
                    }
                ]
            ),
            detection(
                [
                    {
                        "kind": "problem",
                        "label": "Exercise 2",
                        "summary": "two plus two",
                        "start_quote": "Exercise 2. Show that",
                        "end_quote": "2 + 2 = 4.",
                        "continues_past_end": False,
                    }
                ]
            ),
            json.dumps(
                {
                    "skip": False,
                    "imports": ["Mathlib"],
                    "name": "add_zero_nat",
                    "lean_code": "theorem add_zero_nat (n : Nat) : n + 0 = n := by simp",
                    "has_proof": True,
                    "informal": "n + 0 = n",
                    "index_types": ["Nat"],
                    "confidence": 1.0,
                }
            ),
            json.dumps(
                {
                    "skip": False,
                    "imports": ["Mathlib"],
                    "name": "two_add_two",
                    "lean_code": "theorem two_add_two : (2 : Nat) + 2 = 4 := by sorry",
                    "has_proof": False,
                    "informal": "2 + 2 = 4",
                    "index_types": ["Nat"],
                    "confidence": 1.0,
                }
            ),
        ]
    )

    settings = Settings.from_env(dotenv=None)
    settings.database = os.path.join(workdir, "smoke.sqlite")
    settings.repair_attempts = 0

    checker = build_checker(settings)
    if not getattr(checker, "available", lambda: False)():
        print("FAIL: this image has no working Lean; verification would be skipped")
        return 1

    database = Database(settings.database)
    result = ingest(
        book,
        settings=settings,
        client=client,
        database=database,
        checker=checker,
        title="smoke",
    )
    summary = result.summary()
    print(json.dumps(summary, indent=2))

    failures = []
    if summary["detected"] != 2:
        failures.append(f"expected 2 detections, got {summary['detected']}")
    if summary["verified"] != 2:
        failures.append(f"Lean should have accepted both, got {summary['verified']}")
    if summary["proved"] != 1:
        failures.append(f"exactly one item carries a real proof, got {summary['proved']}")
    if not database.search(types=["Nat"]):
        failures.append("the type index is empty")
    problems = database.search(kind="problem")
    if len(problems) != 1 or problems[0]["verify_mode"] != "statement":
        failures.append("the open problem should be checked as a statement only")
    database.close()

    for line in failures:
        print("FAIL:", line)
    if failures:
        return 1
    print("pipeline + lean end to end: ok")
    return 0


if __name__ == "__main__":
    sys.exit(main())