Stop an empty env var from shadowing .env, and refuse a total OCR blackout
ci / test (push) Successful in 52s
ci / lean-project (push) Successful in 6s
docker / image (push) Successful in 1m13s

Two faults found by running the pipeline inside the Docker image for the first
time, both of which let a 25-page run finish "successfully" having stored nothing.

The image declares ENV DASHSCOPE_API_KEY= so the variable is visible to
docker inspect and overridable at docker run. That leaves it set-but-empty in
every container, and load_dotenv used os.environ.setdefault, so the empty string
won and the mounted .env was ignored. An empty value now counts as unset; a real
value still wins, because overriding on the command line is the point.

The consequence was worse than a missing key. Every page failed, the per-page
fallback kept the font-mangled embedded text -- exactly the text OCR is there to
replace -- and the run went on to spend a full detection pass on it. Tolerating
one unreadable page is right; tolerating every page is a configuration error
wearing a success message, so Document now raises OcrUnavailable when OCR was
attempted and not one page came back, quoting the first real failure. A page
range that needed no OCR at all is still not an error.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Ci5T3gWDhhmqDec3e1XtYZ
This commit is contained in:
2026-09-19 19:09:11 +00:00
parent 598f7fd662
commit bd26e9ce6e
4 changed files with 145 additions and 6 deletions
+46 -4
View File
@@ -4,7 +4,7 @@ from pathlib import Path
import pytest
from leanbench.document import Document
from leanbench.document import Document, OcrUnavailable
from leanbench.ocr import (
OcrCache,
OcrReader,
@@ -64,6 +64,22 @@ class ExplodingBackend:
raise RuntimeError("model refused")
class FlakyBackend:
"""Fails on chosen pages only -- the ordinary case of one unreadable page."""
name = "flaky:v1"
def __init__(self, fail_on: set[int]) -> None:
self.fail_on = fail_on
self.seen = 0
def transcribe(self, image: bytes) -> str:
self.seen += 1
if self.seen in self.fail_on:
raise RuntimeError(f"page {self.seen} refused")
return CLEAN
# -- the quality heuristic -------------------------------------------------
@@ -225,11 +241,37 @@ def test_text_mode_never_touches_the_model(mixed_book: Path) -> None:
def test_a_failed_page_keeps_its_embedded_text(mixed_book: Path, monkeypatch) -> None:
"""One bad page falls back; the rest of the book is unaffected."""
monkeypatch.setattr("leanbench.ocr.render_page", lambda *a, **k: b"png")
reader = OcrReader(backend=FlakyBackend(fail_on={2}), cache=None)
document = Document.load(mixed_book, mode="ocr", ocr=reader)
assert document.meta["ocr"]["pages_failed"] == [2]
assert document.meta["ocr"]["pages_replaced"] == [1]
assert "Š" in document.page(2).text, "we fall back, we do not lose the page"
def test_total_ocr_failure_is_loud(mixed_book: Path, monkeypatch) -> None:
"""Every page failing is a broken configuration, not a run to complete.
Silently falling back here is the expensive failure: the fallback text is the
mangled text OCR was called in to replace, so the run costs a full detection
pass and stores nothing.
"""
monkeypatch.setattr("leanbench.ocr.render_page", lambda *a, **k: b"png")
reader = OcrReader(backend=ExplodingBackend(), cache=None)
document = Document.load(mixed_book, mode="ocr", ocr=reader)
assert document.meta["ocr"]["pages_failed"] == [1, 2]
assert "Š" in document.page(2).text, "we fall back, we do not lose the page"
with pytest.raises(OcrUnavailable) as caught:
Document.load(mixed_book, mode="ocr", ocr=reader)
assert "model refused" in str(caught.value), "the real cause must reach the operator"
assert "boom:v1" in str(caught.value)
def test_auto_mode_with_nothing_to_do_does_not_raise(tmp_path: Path) -> None:
"""No page needed OCR, so nothing failed -- that is success, not an outage."""
path = tmp_path / "clean.txt"
path.write_text(PROSE, encoding="utf-8")
document = Document.load(path, mode="auto", ocr=OcrReader(backend=ExplodingBackend()))
assert document.meta["ocr"]["pages_replaced"] == []
assert document.meta["ocr"]["pages_failed"] == []
def test_offsets_stay_usable_after_ocr(mixed_book: Path, monkeypatch) -> None: