Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6

tests/test_align.py

@@ -0,0 +1,148 @@
+import json
+
+from leanbench.align import align, llm_pairing
+from leanbench.llm import ScriptedClient
+from leanbench.project import project
+from leanbench.srcsimplex import SourceRole, SourceSimplex
+
+
+def src(*roles, exists=False):
+    out = []
+    counts: dict[str, int] = {}
+    for kind, symbols, desc, keywords, *rest in roles:
+        counts[kind] = counts.get(kind, 0) + 1
+        out.append(SourceRole(
+            kind, symbols, desc, keywords, rest[0] if rest else "given", f"{kind}{counts[kind]}"))
+    return SourceSimplex(out, exists)
+
+
+GROUP = project(
+    "theorem t (G : Type*) [Group G] [Fintype G] (g : G) : g ^ Fintype.card G = 1 := by sorry")
+GROUP_SRC = src(
+    ("object", ["G"], "finite group", ["finite", "group"]),
+    ("object", ["g"], "element of G", []),
+    ("fact", ["G"], "G is finite", ["finite"]),
+    ("goal", ["g", "G"], "g raised to the size of G is the identity", ["order"]),
+)
+
+
+def test_faithful_statement_is_not_flagged():
+    result = align(GROUP_SRC, GROUP)
+    assert not result.flagged, result.as_dict()
+    assert result.score > 0.8
+    assert {m.source for m in result.matches} >= {"object1", "object2", "fact1", "goal1"}
+
+
+def test_missing_hypothesis_is_named():
+    lean = project("theorem t (n : ℕ) (a : ℤ) : a ^ n = a ^ n := by rfl")
+    source = src(
+        ("object", ["n"], "positive integer", ["integer"]),
+        ("object", ["a"], "integer", ["integer"]),
+        ("fact", ["a", "n"], "a and n are coprime", ["coprime"]),
+        ("goal", ["a", "n"], "a to the n", []),
+    )
+    result = align(source, lean)
+    assert "missing_hypothesis" in result.errors
+    assert result.flagged
+
+
+def test_extra_hypothesis_is_named():
+    lean = project("theorem t (n : ℤ) (h : 0 < n) : n = n := by rfl")
+    source = src(
+        ("object", ["n"], "integer", ["integer"]),
+        ("goal", ["n"], "n equals itself", []),
+    )
+    result = align(source, lean)
+    assert "extra_hypothesis" in result.errors and result.flagged
+
+
+def test_changed_number_system_is_a_type_conflict():
+    lean = project("theorem t (n : ℕ) : n = n := by rfl")
+    source = src(
+        ("object", ["n"], "an integer", ["integer"]),
+        ("goal", ["n"], "n equals itself", []),
+    )
+    result = align(source, lean)
+    assert [c.kind for c in result.conflicts if c.kind == "type_conflict"] == ["type_conflict"]
+
+
+def test_exists_turned_into_given_is_a_quantifier_error():
+    lean = project("theorem t (m : ℕ) : m = m := by rfl")
+    source = src(
+        ("object", ["m"], "a natural number", ["natural"], "exists"),
+        ("goal", ["m"], "there is such an m", [], "given"),
+    )
+    result = align(source, lean)
+    assert "quantifier" in result.errors
+
+
+def test_existential_in_goal_is_a_clean_match():
+    lean = project("theorem t (n : ℕ) : ∃ m : ℕ, m > n := by sorry")
+    source = src(
+        ("object", ["n"], "natural number", ["natural"]),
+        ("object", ["m"], "a larger natural number", ["natural"], "exists"),
+        ("goal", ["m", "n"], "there is a larger number", []),
+        exists=True,
+    )
+    result = align(source, lean)
+    assert "quantifier" not in result.errors
+    assert "missing_object" not in result.errors
+
+
+def test_goal_exists_mismatch():
+    lean = project("theorem t (n : ℕ) (a : ℕ) : n = a := by sorry")
+    source = src(
+        ("object", ["n"], "natural number", ["natural"]),
+        ("object", ["a"], "natural number", ["natural"]),
+        ("goal", ["n", "a"], "there exist", []),
+        exists=True,
+    )
+    assert "quantifier" in align(source, lean).errors
+
+
+def test_wrong_goal_is_flagged():
+    lean = project("theorem t (G : Type*) [Group G] (g : G) : g * g = g := by sorry")
+    source = src(
+        ("object", ["G"], "group", ["group"]),
+        ("object", ["g"], "element", []),
+        ("goal", ["x"], "the subgroup is normal", ["normal", "subgroup"]),
+    )
+    assert "goal_mismatch" in align(source, lean).errors
+
+
+def test_degenerate_lean_is_reported():
+    lean = project("theorem t : 2 + 2 = 4 := by norm_num")
+    source = src(("object", ["n"], "integer", ["integer"]), ("goal", ["n"], "n", []))
+    assert "degenerate" in align(source, lean).errors
+
+
+def test_extra_object_is_reported_but_not_flagged_alone():
+    lean = project("theorem t (n : ℕ) (k : ℕ) : n = n := by rfl")
+    source = src(("object", ["n"], "natural", ["natural"]), ("goal", ["n"], "n = n", []))
+    result = align(source, lean)
+    assert "extra_object" in result.errors and not result.flagged
+
+
+def test_pairing_overrides_the_deterministic_matcher():
+    lean = project("theorem t (x : ℤ) (hx : 0 < x) : x = x := by rfl")
+    source = src(
+        ("object", ["z"], "integer", ["integer"]),
+        ("fact", [], "it is positive", []),
+        ("goal", [], "trivial", []),
+    )
+    pairing = {"object1": "x", "fact1": "hx"}
+    result = align(source, lean, pairing=pairing)
+    assert {(m.source, m.lean) for m in result.matches} >= {("object1", "x"), ("fact1", "hx")}
+    assert "missing_hypothesis" not in result.errors
+
+
+def test_llm_pairing_keeps_only_real_keys():
+    client = ScriptedClient([json.dumps({"pairs": {"object1": "x", "fact1": "nonsense"}})])
+    lean = project("theorem t (x : ℤ) (hx : 0 < x) : x = x := by rfl")
+    source = src(("object", ["z"], "integer", []), ("fact", [], "positive", []),
+                 ("goal", [], "", []))
+    assert llm_pairing(client, source, lean) == {"object1": "x"}
+
+
+def test_alignment_dict_is_json_serialisable():
+    json.dumps(align(GROUP_SRC, GROUP).as_dict())