from __future__ import annotations from leanbench.detect import KIND_PROBLEM, KIND_STATEMENT, detect_document from leanbench.document import Document from leanbench.llm import ScriptedClient from .conftest import detection_reply def test_single_page_detection_reports_page_and_index(two_page_book: Document) -> None: client = ScriptedClient( [ detection_reply( [ { "kind": "statement", "label": "Definition 1.6.1", "summary": "divisibility", "start_quote": "Definition 1.6.1. Let a and b be integers.", "end_quote": "b = a k.", "continues_past_end": False, } ] ), detection_reply([]), ] ) report = detect_document(client, two_page_book, max_window_pages=3) assert len(report.detections) == 1 found = report.detections[0] assert found.kind == KIND_STATEMENT assert found.start_page == 1 and found.end_page == 1 page = two_page_book.page(1).text assert page[found.start_index : found.end_index].startswith("Definition 1.6.1") assert page[found.start_index : found.end_index].endswith("b = a k.") def test_item_running_off_the_page_triggers_a_second_page(two_page_book: Document) -> None: truncated = detection_reply( [ { "kind": "statement", "label": "Theorem 1.6.3", "summary": "gcd exists", "start_quote": "Theorem 1.6.3. Let a and b be nonzero integers.", "end_quote": "the set of common", "continues_past_end": True, } ] ) whole = detection_reply( [ { "kind": "statement", "label": "Theorem 1.6.3", "summary": "gcd exists", "start_quote": "Theorem 1.6.3. Let a and b be nonzero integers.", "end_quote": "the greatest common divisor.", "continues_past_end": False, }, { "kind": "problem", "label": "Exercise 1.6.5", "summary": "odd + odd is even", "start_quote": "Exercise 1.6.5. Prove that the sum of two odd", "end_quote": "odd integers is even.", "continues_past_end": False, }, ] ) client = ScriptedClient([truncated, whole]) report = detect_document(client, two_page_book, max_window_pages=3) assert len(client.calls) == 2, "the page should be re-read with its successor" theorem = report.detections[0] assert theorem.start_page == 1 assert theorem.end_page == 2, "the span must cross the page boundary" assert theorem.spans_pages == 2 assert not theorem.truncated assert two_page_book.page(2).text[: theorem.end_index].endswith("greatest common divisor.") exercise = report.detections[1] assert exercise.kind == KIND_PROBLEM assert exercise.start_page == 2 def test_window_stops_growing_at_the_limit_and_says_so(two_page_book: Document) -> None: always_truncated = detection_reply( [ { "kind": "statement", "label": "", "summary": "", "start_quote": "Definition 1.6.1. Let a and b be integers.", "end_quote": "b = a k.", "continues_past_end": True, } ] ) client = ScriptedClient([always_truncated] * 4) report = detect_document(client, two_page_book, max_window_pages=2) assert len(client.calls) == 2 assert report.detections[0].truncated is True def test_unresolvable_quotes_are_recorded_not_invented(two_page_book: Document) -> None: client = ScriptedClient( [ detection_reply( [ { "kind": "statement", "label": "Hallucination 9", "summary": "not in the book", "start_quote": "Theorem 9.9.9 about topological quantum widgets", "end_quote": "which is absurd.", "continues_past_end": False, } ] ), detection_reply([]), ] ) report = detect_document(client, two_page_book) assert report.detections == [] assert len(report.unresolved) == 1 def test_a_failing_page_does_not_kill_the_run(two_page_book: Document) -> None: client = ScriptedClient(["not json at all", detection_reply([])]) report = detect_document(client, two_page_book) assert report.pages_scanned == 2 assert any("detector failed" in u.get("reason", "") for u in report.unresolved)