Zayden-Jung
d3555d9a52
Replaces the previous tree entirely. The pipeline is four separable pieces, each
testable on its own:
1. leanbench/llm.py provider-neutral OpenAI-compatible connector, stdlib only,
with retries, JSON coercion and a scripted client so the
rest of the code is testable without a network.
2. leanbench/detect.py per-page detection of formalizable mathematics. Case 1 is
mathematics asserted as fact (definition/theorem/lemma/...,
deliberately not sub-classified); case 2 is a problem left
open to the reader. Hits come back as spans -- start page
and index, end page and index -- so several per page is
normal. A hit cut off by the page end makes the detector
re-read that page together with the next, repeatedly, so an
item spread over three pages is found whole.
Offsets come from verbatim quotes resolved by
leanbench/anchors.py (exact -> whitespace-insensitive ->
fuzzy), because models quote well and count badly.
3. leanbench/formalize.py + leanbench/verify.py
each span becomes one Lean 4 declaration, judged by the real
compiler: full elaboration for case 1, statement legality
only for case 2, which has no proof to check. Lean's own
diagnostics feed back for a repair attempt. "verified" (Lean
accepted it) and "proved" (accepted with no sorry) are kept
strictly apart.
4. leanbench/db.py indexed SQLite plus JSONL export. Every item is indexed by
the types of its inputs, or by its own type when it is not a
proof, and records the source book (sha256) and the page it
starts on -- so `drop --source H --pages A-B` followed by a
re-ingest redoes exactly those pages and nothing else.
Also: lean/ Lake project pinned to Mathlib, docker/ image carrying Lean 4 and a
prebuilt Mathlib (the Mathlib revision and the toolchain are resolved against each
other at build time rather than both hardcoded), and .gitea/workflows for tests,
image build and push to the Gitea registry, and a manual "submit a book" run.
58 tests; they need neither network nor Lean.
117 lines
3.8 KiB
Python
117 lines
3.8 KiB
Python
"""Runtime configuration, read from the environment (and an optional .env file).
|
|
|
|
Everything the pipeline needs to talk to an LLM or to Lean is settable without
|
|
touching code, so the same image runs against any OpenAI-compatible endpoint and
|
|
any Lean project.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
|
|
DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
|
DEFAULT_MODEL = "qwen-max"
|
|
DEFAULT_API_KEY_ENV = "DASHSCOPE_API_KEY"
|
|
|
|
|
|
def load_dotenv(path: Path | str = ".env") -> None:
|
|
"""Load ``KEY=VALUE`` lines into os.environ without overwriting what is set."""
|
|
path = Path(path)
|
|
if not path.is_file():
|
|
return
|
|
for line in path.read_text(encoding="utf-8").splitlines():
|
|
line = line.strip()
|
|
if not line or line.startswith("#") or "=" not in line:
|
|
continue
|
|
key, _, value = line.partition("=")
|
|
os.environ.setdefault(key.strip(), value.strip().strip('"').strip("'"))
|
|
|
|
|
|
def _env(name: str, default: str) -> str:
|
|
value = os.environ.get(name)
|
|
return default if value is None or value == "" else value
|
|
|
|
|
|
def _env_int(name: str, default: int) -> int:
|
|
try:
|
|
return int(_env(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
def _env_float(name: str, default: float) -> float:
|
|
try:
|
|
return float(_env(name, str(default)))
|
|
except ValueError:
|
|
return default
|
|
|
|
|
|
@dataclass
|
|
class LLMSettings:
|
|
model: str = DEFAULT_MODEL
|
|
base_url: str = DEFAULT_BASE_URL
|
|
api_key_env: str = DEFAULT_API_KEY_ENV
|
|
timeout: float = 300.0
|
|
max_retries: int = 4
|
|
temperature: float = 0.0
|
|
|
|
@classmethod
|
|
def from_env(cls) -> LLMSettings:
|
|
return cls(
|
|
model=_env("LEANBENCH_LLM_MODEL", DEFAULT_MODEL),
|
|
base_url=_env("LEANBENCH_LLM_BASE_URL", DEFAULT_BASE_URL),
|
|
api_key_env=_env("LEANBENCH_LLM_API_KEY_ENV", DEFAULT_API_KEY_ENV),
|
|
timeout=_env_float("LEANBENCH_LLM_TIMEOUT", 300.0),
|
|
max_retries=_env_int("LEANBENCH_LLM_MAX_RETRIES", 4),
|
|
temperature=_env_float("LEANBENCH_LLM_TEMPERATURE", 0.0),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class LeanSettings:
|
|
"""How to invoke Lean. ``command`` is an argv prefix that is given a .lean path."""
|
|
|
|
command: list[str] = field(default_factory=lambda: ["lake", "env", "lean"])
|
|
project: str | None = None
|
|
timeout: float = 120.0
|
|
header: str = "import Mathlib\n"
|
|
|
|
@classmethod
|
|
def from_env(cls) -> LeanSettings:
|
|
raw = _env("LEANBENCH_LEAN_COMMAND", '["lake","env","lean"]')
|
|
try:
|
|
command = json.loads(raw)
|
|
if not isinstance(command, list) or not all(isinstance(x, str) for x in command):
|
|
raise ValueError
|
|
except ValueError:
|
|
command = raw.split()
|
|
return cls(
|
|
command=command,
|
|
project=os.environ.get("LEANBENCH_LEAN_PROJECT") or None,
|
|
timeout=_env_float("LEANBENCH_LEAN_TIMEOUT", 120.0),
|
|
header=_env("LEANBENCH_LEAN_HEADER", "import Mathlib\n"),
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Settings:
|
|
llm: LLMSettings = field(default_factory=LLMSettings)
|
|
lean: LeanSettings = field(default_factory=LeanSettings)
|
|
database: str = "database/leanbench.sqlite"
|
|
max_window_pages: int = 3
|
|
repair_attempts: int = 1
|
|
|
|
@classmethod
|
|
def from_env(cls, dotenv: Path | str | None = ".env") -> Settings:
|
|
if dotenv is not None:
|
|
load_dotenv(dotenv)
|
|
return cls(
|
|
llm=LLMSettings.from_env(),
|
|
lean=LeanSettings.from_env(),
|
|
database=_env("LEANBENCH_DATABASE", "database/leanbench.sqlite"),
|
|
max_window_pages=_env_int("LEANBENCH_MAX_WINDOW_PAGES", 3),
|
|
repair_attempts=_env_int("LEANBENCH_REPAIR_ATTEMPTS", 1),
|
|
)
|