Stop an empty env var from shadowing .env, and refuse a total OCR blackout
Two faults found by running the pipeline inside the Docker image for the first time, both of which let a 25-page run finish "successfully" having stored nothing. The image declares ENV DASHSCOPE_API_KEY= so the variable is visible to docker inspect and overridable at docker run. That leaves it set-but-empty in every container, and load_dotenv used os.environ.setdefault, so the empty string won and the mounted .env was ignored. An empty value now counts as unset; a real value still wins, because overriding on the command line is the point. The consequence was worse than a missing key. Every page failed, the per-page fallback kept the font-mangled embedded text -- exactly the text OCR is there to replace -- and the run went on to spend a full detection pass on it. Tolerating one unreadable page is right; tolerating every page is a configuration error wearing a success message, so Document now raises OcrUnavailable when OCR was attempted and not one page came back, quoting the first real failure. A page range that needed no OCR at all is still not an error. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Ci5T3gWDhhmqDec3e1XtYZ
This commit is contained in:
+46
-4
@@ -4,7 +4,7 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from leanbench.document import Document
|
||||
from leanbench.document import Document, OcrUnavailable
|
||||
from leanbench.ocr import (
|
||||
OcrCache,
|
||||
OcrReader,
|
||||
@@ -64,6 +64,22 @@ class ExplodingBackend:
|
||||
raise RuntimeError("model refused")
|
||||
|
||||
|
||||
class FlakyBackend:
|
||||
"""Fails on chosen pages only -- the ordinary case of one unreadable page."""
|
||||
|
||||
name = "flaky:v1"
|
||||
|
||||
def __init__(self, fail_on: set[int]) -> None:
|
||||
self.fail_on = fail_on
|
||||
self.seen = 0
|
||||
|
||||
def transcribe(self, image: bytes) -> str:
|
||||
self.seen += 1
|
||||
if self.seen in self.fail_on:
|
||||
raise RuntimeError(f"page {self.seen} refused")
|
||||
return CLEAN
|
||||
|
||||
|
||||
# -- the quality heuristic -------------------------------------------------
|
||||
|
||||
|
||||
@@ -225,11 +241,37 @@ def test_text_mode_never_touches_the_model(mixed_book: Path) -> None:
|
||||
|
||||
|
||||
def test_a_failed_page_keeps_its_embedded_text(mixed_book: Path, monkeypatch) -> None:
|
||||
"""One bad page falls back; the rest of the book is unaffected."""
|
||||
monkeypatch.setattr("leanbench.ocr.render_page", lambda *a, **k: b"png")
|
||||
reader = OcrReader(backend=FlakyBackend(fail_on={2}), cache=None)
|
||||
document = Document.load(mixed_book, mode="ocr", ocr=reader)
|
||||
assert document.meta["ocr"]["pages_failed"] == [2]
|
||||
assert document.meta["ocr"]["pages_replaced"] == [1]
|
||||
assert "Š" in document.page(2).text, "we fall back, we do not lose the page"
|
||||
|
||||
|
||||
def test_total_ocr_failure_is_loud(mixed_book: Path, monkeypatch) -> None:
|
||||
"""Every page failing is a broken configuration, not a run to complete.
|
||||
|
||||
Silently falling back here is the expensive failure: the fallback text is the
|
||||
mangled text OCR was called in to replace, so the run costs a full detection
|
||||
pass and stores nothing.
|
||||
"""
|
||||
monkeypatch.setattr("leanbench.ocr.render_page", lambda *a, **k: b"png")
|
||||
reader = OcrReader(backend=ExplodingBackend(), cache=None)
|
||||
document = Document.load(mixed_book, mode="ocr", ocr=reader)
|
||||
assert document.meta["ocr"]["pages_failed"] == [1, 2]
|
||||
assert "Š" in document.page(2).text, "we fall back, we do not lose the page"
|
||||
with pytest.raises(OcrUnavailable) as caught:
|
||||
Document.load(mixed_book, mode="ocr", ocr=reader)
|
||||
assert "model refused" in str(caught.value), "the real cause must reach the operator"
|
||||
assert "boom:v1" in str(caught.value)
|
||||
|
||||
|
||||
def test_auto_mode_with_nothing_to_do_does_not_raise(tmp_path: Path) -> None:
|
||||
"""No page needed OCR, so nothing failed -- that is success, not an outage."""
|
||||
path = tmp_path / "clean.txt"
|
||||
path.write_text(PROSE, encoding="utf-8")
|
||||
document = Document.load(path, mode="auto", ocr=OcrReader(backend=ExplodingBackend()))
|
||||
assert document.meta["ocr"]["pages_replaced"] == []
|
||||
assert document.meta["ocr"]["pages_failed"] == []
|
||||
|
||||
|
||||
def test_offsets_stay_usable_after_ocr(mixed_book: Path, monkeypatch) -> None:
|
||||
|
||||
Reference in New Issue
Block a user