Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6

tests/test_align.py

@@ -0,0 +1,148 @@
+import json
+
+from leanbench.align import align, llm_pairing
+from leanbench.llm import ScriptedClient
+from leanbench.project import project
+from leanbench.srcsimplex import SourceRole, SourceSimplex
+
+
+def src(*roles, exists=False):
+    out = []
+    counts: dict[str, int] = {}
+    for kind, symbols, desc, keywords, *rest in roles:
+        counts[kind] = counts.get(kind, 0) + 1
+        out.append(SourceRole(
+            kind, symbols, desc, keywords, rest[0] if rest else "given", f"{kind}{counts[kind]}"))
+    return SourceSimplex(out, exists)
+
+
+GROUP = project(
+    "theorem t (G : Type*) [Group G] [Fintype G] (g : G) : g ^ Fintype.card G = 1 := by sorry")
+GROUP_SRC = src(
+    ("object", ["G"], "finite group", ["finite", "group"]),
+    ("object", ["g"], "element of G", []),
+    ("fact", ["G"], "G is finite", ["finite"]),
+    ("goal", ["g", "G"], "g raised to the size of G is the identity", ["order"]),
+)
+
+
+def test_faithful_statement_is_not_flagged():
+    result = align(GROUP_SRC, GROUP)
+    assert not result.flagged, result.as_dict()
+    assert result.score > 0.8
+    assert {m.source for m in result.matches} >= {"object1", "object2", "fact1", "goal1"}
+
+
+def test_missing_hypothesis_is_named():
+    lean = project("theorem t (n : ℕ) (a : ℤ) : a ^ n = a ^ n := by rfl")
+    source = src(
+        ("object", ["n"], "positive integer", ["integer"]),
+        ("object", ["a"], "integer", ["integer"]),
+        ("fact", ["a", "n"], "a and n are coprime", ["coprime"]),
+        ("goal", ["a", "n"], "a to the n", []),
+    )
+    result = align(source, lean)
+    assert "missing_hypothesis" in result.errors
+    assert result.flagged
+
+
+def test_extra_hypothesis_is_named():
+    lean = project("theorem t (n : ℤ) (h : 0 < n) : n = n := by rfl")
+    source = src(
+        ("object", ["n"], "integer", ["integer"]),
+        ("goal", ["n"], "n equals itself", []),
+    )
+    result = align(source, lean)
+    assert "extra_hypothesis" in result.errors and result.flagged
+
+
+def test_changed_number_system_is_a_type_conflict():
+    lean = project("theorem t (n : ℕ) : n = n := by rfl")
+    source = src(
+        ("object", ["n"], "an integer", ["integer"]),
+        ("goal", ["n"], "n equals itself", []),
+    )
+    result = align(source, lean)
+    assert [c.kind for c in result.conflicts if c.kind == "type_conflict"] == ["type_conflict"]
+
+
+def test_exists_turned_into_given_is_a_quantifier_error():
+    lean = project("theorem t (m : ℕ) : m = m := by rfl")
+    source = src(
+        ("object", ["m"], "a natural number", ["natural"], "exists"),
+        ("goal", ["m"], "there is such an m", [], "given"),
+    )
+    result = align(source, lean)
+    assert "quantifier" in result.errors
+
+
+def test_existential_in_goal_is_a_clean_match():
+    lean = project("theorem t (n : ℕ) : ∃ m : ℕ, m > n := by sorry")
+    source = src(
+        ("object", ["n"], "natural number", ["natural"]),
+        ("object", ["m"], "a larger natural number", ["natural"], "exists"),
+        ("goal", ["m", "n"], "there is a larger number", []),
+        exists=True,
+    )
+    result = align(source, lean)
+    assert "quantifier" not in result.errors
+    assert "missing_object" not in result.errors
+
+
+def test_goal_exists_mismatch():
+    lean = project("theorem t (n : ℕ) (a : ℕ) : n = a := by sorry")
+    source = src(
+        ("object", ["n"], "natural number", ["natural"]),
+        ("object", ["a"], "natural number", ["natural"]),
+        ("goal", ["n", "a"], "there exist", []),
+        exists=True,
+    )
+    assert "quantifier" in align(source, lean).errors
+
+
+def test_wrong_goal_is_flagged():
+    lean = project("theorem t (G : Type*) [Group G] (g : G) : g * g = g := by sorry")
+    source = src(
+        ("object", ["G"], "group", ["group"]),
+        ("object", ["g"], "element", []),
+        ("goal", ["x"], "the subgroup is normal", ["normal", "subgroup"]),
+    )
+    assert "goal_mismatch" in align(source, lean).errors
+
+
+def test_degenerate_lean_is_reported():
+    lean = project("theorem t : 2 + 2 = 4 := by norm_num")
+    source = src(("object", ["n"], "integer", ["integer"]), ("goal", ["n"], "n", []))
+    assert "degenerate" in align(source, lean).errors
+
+
+def test_extra_object_is_reported_but_not_flagged_alone():
+    lean = project("theorem t (n : ℕ) (k : ℕ) : n = n := by rfl")
+    source = src(("object", ["n"], "natural", ["natural"]), ("goal", ["n"], "n = n", []))
+    result = align(source, lean)
+    assert "extra_object" in result.errors and not result.flagged
+
+
+def test_pairing_overrides_the_deterministic_matcher():
+    lean = project("theorem t (x : ℤ) (hx : 0 < x) : x = x := by rfl")
+    source = src(
+        ("object", ["z"], "integer", ["integer"]),
+        ("fact", [], "it is positive", []),
+        ("goal", [], "trivial", []),
+    )
+    pairing = {"object1": "x", "fact1": "hx"}
+    result = align(source, lean, pairing=pairing)
+    assert {(m.source, m.lean) for m in result.matches} >= {("object1", "x"), ("fact1", "hx")}
+    assert "missing_hypothesis" not in result.errors
+
+
+def test_llm_pairing_keeps_only_real_keys():
+    client = ScriptedClient([json.dumps({"pairs": {"object1": "x", "fact1": "nonsense"}})])
+    lean = project("theorem t (x : ℤ) (hx : 0 < x) : x = x := by rfl")
+    source = src(("object", ["z"], "integer", []), ("fact", [], "positive", []),
+                 ("goal", [], "", []))
+    assert llm_pairing(client, source, lean) == {"object1": "x"}
+
+
+def test_alignment_dict_is_json_serialisable():
+    json.dumps(align(GROUP_SRC, GROUP).as_dict())

tests/test_evalstats.py

@@ -0,0 +1,44 @@
+import math
+
+import pytest
+
+from leanbench.evalstats import auc, cohen_kappa, confusion, wilson
+
+
+def test_wilson_matches_known_values():
+    lo, hi = wilson(5, 10)
+    assert math.isclose(lo, 0.2366, abs_tol=1e-3) and math.isclose(hi, 0.7634, abs_tol=1e-3)
+    assert wilson(0, 0)[0] == 0.0 and wilson(0, 0)[1] == 1.0
+    assert wilson(10, 10)[1] == 1.0
+
+
+def test_confusion_counts_and_rates():
+    c = confusion([True, True, False, False, True], [True, False, True, False, True])
+    assert (c.tp, c.fp, c.fn, c.tn) == (2, 1, 1, 1)
+    assert math.isclose(c.recall, 2 / 3) and math.isclose(c.precision, 2 / 3)
+    assert math.isclose(c.false_flag_rate, 0.5)
+    assert c.as_dict()["n"] == 5
+
+
+def test_confusion_length_mismatch():
+    with pytest.raises(ValueError):
+        confusion([True], [True, False])
+
+
+def test_empty_classes_give_nan_not_a_crash():
+    c = confusion([False, False], [False, False])
+    assert math.isnan(c.recall) and math.isnan(c.precision)
+
+
+def test_auc():
+    assert auc([0.9, 0.8, 0.2, 0.1], [True, True, False, False]) == 1.0
+    assert auc([0.1, 0.2, 0.8, 0.9], [True, True, False, False]) == 0.0
+    assert auc([0.5, 0.5], [True, False]) == 0.5
+    assert math.isnan(auc([0.5], [True]))
+
+
+def test_kappa():
+    assert cohen_kappa(["a", "b", "a"], ["a", "b", "a"]) == 1.0
+    assert math.isclose(cohen_kappa(["a", "a", "b", "b"], ["a", "b", "a", "b"]), 0.0)
+    with pytest.raises(ValueError):
+        cohen_kappa([], [])

tests/test_lint.py

@@ -0,0 +1,34 @@
+from leanbench.lint import lint
+from leanbench.project import project
+
+
+def found(code):
+    return lint(project(code))
+
+
+def test_clean_statement_has_no_findings():
+    code = "theorem t (G : Type*) [Group G] (g : G) : g * 1 = g := by sorry"
+    assert found(code) == []
+
+
+def test_unused_binder():
+    assert "unused_binder:k" in found("theorem t (n k : ℕ) : n = n := by rfl")
+
+
+def test_carrier_type_is_not_flagged_as_unused():
+    assert not any(f.startswith("unused") for f in found(
+        "theorem t (G : Type*) [Group G] (g : G) : g = g := by rfl"))
+
+
+def test_natural_subtraction_hazard():
+    assert "nat_subtraction" in found("theorem t (a b : ℕ) : a - b + b = a := by sorry")
+    assert "nat_subtraction" not in found("theorem t (a b : ℤ) : a - b + b = a := by ring")
+
+
+def test_integer_division_hazard():
+    assert "integer_division" in found("theorem t (a : ℕ) : a / 2 * 2 = a := by sorry")
+
+
+def test_trivial_goal_and_degenerate():
+    assert "trivial_goal" in found("theorem t (n : ℕ) : True := by trivial")
+    assert "degenerate" in found("theorem t : 2 + 2 = 4 := by norm_num")

tests/test_pilot_scripts.py

@@ -0,0 +1,38 @@
+"""Offline smoke test of the pilot scripts with a stub model (no network)."""
+import importlib.util
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1] / "experiments" / "pilot"
+
+
+def load(name):
+    spec = importlib.util.spec_from_file_location(name, ROOT / f"{name}.py")
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+class Stub:
+    def complete_json(self, messages, **kw):
+        if "roles" in messages[0]["content"] or "passage" in messages[0]["content"].lower() and "faithful" not in messages[0]["content"]:
+            return {"roles": [
+                {"kind": "object", "symbols": ["G"], "description": "finite group", "keywords": ["group", "finite"]},
+                {"kind": "goal", "symbols": [], "description": "g^n = e", "keywords": ["identity"]},
+            ]}
+        return {"faithful": True, "confidence": 0.9, "reason": "ok"}
+
+
+ITEM = {"id": "x1", "source_text": "Let G be a finite group. Then g^n = e.", "label": "t",
+        "lean_code": "theorem t (G : Type*) [Group G] [Fintype G] (g : G) : g ^ Fintype.card G = 1 := sorry",
+        "verified": True}
+
+
+def test_run_item_and_score(tmp_path):
+    rp, sc = load("run_pilot"), load("score")
+    row = rp.run_item(Stub(), ITEM)
+    assert row["id"] == "x1" and row["compiled"] and "judge" in row
+    assert set(sc.flags(row)) == {"compile_only", "lint", "role", "judge"}
+    rows = [dict(row, verdict=v) for v in ("wrong", "faithful", "faithful", "wrong")]
+    out = sc.report(rows, lambda r: r["verdict"] == "wrong")
+    assert out["judge"]["n"] == 4
+    assert sc.decide(out) in {"GO", "PARTIAL", "NO-GO"}

tests/test_project.py

@@ -0,0 +1,94 @@
+from leanbench.project import classify, project
+from leanbench.schema import KIND_FACT, KIND_GOAL, KIND_OBJECT, KIND_OWN
+
+
+def roles(code):
+    return {r.name: r for r in project(code).roles}
+
+
+def test_objects_instances_and_goal():
+    p = project(
+        "theorem t (G : Type*) [Group G] [Fintype G] (g : G) : g ^ Fintype.card G = 1 := by sorry"
+    )
+    got = {r.name: r for r in p.roles}
+    assert list(got) == ["G", "g", "goal"]
+    assert got["G"].kind == KIND_OBJECT
+    assert got["G"].attrs["instances"] == ["Group G", "Fintype G"]
+    assert got["goal"].kind == KIND_GOAL
+    assert not p.degenerate
+    assert p.simplex is not None and len(p.simplex.roles) == 3
+
+
+def test_hypothesis_binders_become_facts():
+    got = roles("theorem t (n : ℕ) (a : ℤ) (h : Nat.Coprime a n) : a ≡ 1 [ZMOD n] := by sorry")
+    assert got["h"].kind == KIND_FACT
+    assert got["h"].attrs["classified_by"] == "predicate"
+    assert got["n"].kind == KIND_OBJECT
+
+
+def test_relation_symbols_mark_a_fact():
+    got = roles("theorem t (a b : ℤ) (hab : a ∣ b) (hpos : 0 < a) : a ≤ b := by sorry")
+    assert got["hab"].kind == KIND_FACT and got["hpos"].kind == KIND_FACT
+    assert got["a"].kind == KIND_OBJECT
+
+
+def test_function_type_is_data_not_a_fact():
+    kind, how = classify("ℕ → ℕ", ["f"], set())
+    assert kind == KIND_OBJECT and how == "default"
+    assert classify("Type*", ["G"], set()) == (KIND_OBJECT, "sort")
+
+
+def test_leading_forall_is_lifted_so_both_spellings_agree():
+    a = project("theorem t (n : ℕ) : n = n := by rfl")
+    b = project("theorem t : ∀ n : ℕ, n = n := by rfl")
+    assert [(r.name, r.kind) for r in a.roles] == [(r.name, r.kind) for r in b.roles]
+    assert {r.attrs.get("origin") for r in b.roles if r.name == "n"} == {"forall_in_goal"}
+
+
+def test_goal_implication_splits_into_premises():
+    got = roles("theorem t : ∀ n m : ℕ, n < m → m < 10 → n < 10 := by omega")
+    assert [n for n in got if n.startswith("premise")] == ["premise1", "premise2"]
+    assert got["goal"].type_text == "n < 10"
+
+
+def test_bounded_forall_adds_an_object_and_a_fact():
+    got = roles("theorem t : ∀ x ∈ S, f x = 0 := by sorry")
+    assert got["x"].kind == KIND_OBJECT and got["h_x"].kind == KIND_FACT
+
+
+def test_definition_has_an_own_role():
+    p = project("def divides (a b : Int) : Prop := ∃ k, b = a * k")
+    assert p.by_kind(KIND_OWN)[0].type_text == "Prop"
+    assert [r.name for r in p.roles] == ["a", "b", "own"]
+
+
+def test_structure_stops_at_where_and_ignores_extends():
+    p = project("structure Ring (R : Type*) extends AddCommGroup R, Mul R where\n  x : R")
+    assert [r.name for r in p.roles] == ["R", "own"]
+
+
+def test_bare_statement_is_degenerate_not_an_error():
+    p = project("theorem t : 2 + 2 = 4 := by norm_num")
+    assert p.degenerate and p.simplex is None
+    assert [r.name for r in p.roles] == ["goal"]
+
+
+def test_no_declaration():
+    p = project("-- just a comment")
+    assert p.degenerate and p.notes == ["no declaration found"]
+
+
+def test_instance_without_target_is_noted():
+    p = project("theorem t (n : ℕ) [Fact (1 < 2)] : n = n := by rfl")
+    assert any("constrains no named object" in n for n in p.notes)
+
+
+def test_duplicate_names_are_aliased():
+    p = project("theorem t (h : 0 < 1) (h : 1 < 2) : True := by trivial")
+    assert p.simplex is not None
+    assert "h" in p.simplex.roles and "h_2" in p.simplex.roles
+
+
+def test_comments_and_proof_body_are_ignored():
+    code = "/- doc -/\ntheorem t (n : ℕ) : n = n := by\n  -- (x : T) here is noise\n  rfl"
+    assert [r.name for r in project(code).roles] == ["n", "goal"]

tests/test_schema.py

@@ -0,0 +1,49 @@
+import pytest
+
+from leanbench.schema import KIND_FACT, KIND_OBJECT, Fact, Goal, Role, Simplex
+
+
+def test_simplex_needs_two_roles():
+    only = Role("n", KIND_OBJECT)
+    with pytest.raises(ValueError, match="at least 2"):
+        Simplex("solo", {"n": only})
+    two = Simplex("pair", {"n": only, "h": Role("h", KIND_FACT)})
+    assert [r.name for r in two.by_kind(KIND_FACT)] == ["h"]
+
+
+def test_validation_runs_in_init_not_a_dead_post_init():
+    # The archived prototype defined __post_init__ beside a hand-written __init__,
+    # so its minimum-role check never executed. Guard against that regression.
+    with pytest.raises(ValueError):
+        Simplex("empty", {})
+
+
+def test_constrained_role_rejects_values_outside_choices():
+    role = Role("parity", KIND_OBJECT, choices=["even", "odd"])
+    assert role.is_constrained()
+    assert role.validate_value("even")
+    with pytest.raises(ValueError, match="not allowed"):
+        role.check_value("prime")
+    assert Role("free", KIND_OBJECT).validate_value(object())
+
+
+def test_simplices_nest_and_walk_flattens():
+    inner = Simplex("inner", {"a": Role("a"), "b": Role("b")})
+    outer = Simplex("outer", {"inner": inner, "c": Role("c")})
+    assert sorted(r.name for r in outer.walk()) == ["a", "b", "c"]
+    assert outer.as_dict()["roles"]["inner"]["roles"]["a"]["name"] == "a"
+
+
+def test_unknown_kind_and_empty_name_are_rejected():
+    with pytest.raises(ValueError):
+        Role("x", "banana")
+    with pytest.raises(ValueError):
+        Role("")
+
+
+def test_states():
+    role = Role("n")
+    assert not Goal(role).is_resolved()
+    assert Fact(role, 3).is_resolved()
+    with pytest.raises(ValueError):
+        Fact(role, None)

tests/test_srcsimplex.py

@@ -0,0 +1,72 @@
+import json
+
+import pytest
+
+from leanbench.llm import ScriptedClient
+from leanbench.srcsimplex import (
+    SYSTEM_PROMPT,
+    extract_source_simplex,
+    parse_source_simplex,
+    passage_prompt,
+)
+
+GOOD = {
+    "roles": [
+        {"kind": "object", "symbols": ["G"], "description": "finite group",
+         "keywords": ["Finite", "group"], "introduced": "given"},
+        {"kind": "goal", "symbols": [], "description": "g^|G| = e", "keywords": []},
+    ],
+    "goal_has_exists": False,
+}
+
+
+def test_parse_lowercases_keywords_and_numbers_aliases():
+    s = parse_source_simplex(GOOD)
+    assert s.roles[0].keywords == ["finite", "group"]
+    assert [r.alias for r in s.roles] == ["object1", "goal1"]
+    assert s.goal is not None and s.goal.kind == "goal"
+
+
+@pytest.mark.parametrize("payload", [
+    {"roles": []},
+    {"roles": [{"kind": "goal"}]},
+    {"roles": [{"kind": "object"}, {"kind": "object"}]},
+    {"roles": [{"kind": "goal"}, {"kind": "goal"}]},
+    ["not", "a", "dict"],
+])
+def test_unusable_replies_are_rejected(payload):
+    with pytest.raises(ValueError):
+        parse_source_simplex(payload)
+
+
+def test_unknown_kinds_are_dropped_with_a_note():
+    payload = {"roles": GOOD["roles"] + [{"kind": "banana"}]}
+    s = parse_source_simplex(payload)
+    assert len(s.roles) == 2 and s.notes
+
+
+def test_bad_introduced_defaults_to_given():
+    payload = {"roles": [
+        {"kind": "object", "introduced": "sometimes"}, {"kind": "goal"}]}
+    assert parse_source_simplex(payload).roles[0].introduced == "given"
+
+
+def test_extract_retries_once_then_succeeds():
+    client = ScriptedClient([json.dumps({"roles": []}), json.dumps(GOOD)])
+    s = extract_source_simplex(client, "Let G be a finite group.", "Prop 1")
+    assert s.goal is not None
+    assert len(client.calls) == 2
+
+
+def test_extract_gives_up_loudly():
+    client = ScriptedClient([json.dumps({"roles": []})] * 2)
+    with pytest.raises(ValueError, match="no usable source simplex"):
+        extract_source_simplex(client, "x")
+
+
+def test_prompt_never_shows_lean():
+    messages = passage_prompt("Let   G be a group.\n", "Theorem 2")
+    assert "Lean" in SYSTEM_PROMPT  # only to forbid writing it
+    assert "Do not write Lean" in SYSTEM_PROMPT
+    assert "Label: Theorem 2" in messages[1]["content"]
+    assert "Let G be a group." in messages[1]["content"]