Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6

tests/test_srcsimplex.py

@@ -0,0 +1,72 @@
+import json
+
+import pytest
+
+from leanbench.llm import ScriptedClient
+from leanbench.srcsimplex import (
+    SYSTEM_PROMPT,
+    extract_source_simplex,
+    parse_source_simplex,
+    passage_prompt,
+)
+
+GOOD = {
+    "roles": [
+        {"kind": "object", "symbols": ["G"], "description": "finite group",
+         "keywords": ["Finite", "group"], "introduced": "given"},
+        {"kind": "goal", "symbols": [], "description": "g^|G| = e", "keywords": []},
+    ],
+    "goal_has_exists": False,
+}
+
+
+def test_parse_lowercases_keywords_and_numbers_aliases():
+    s = parse_source_simplex(GOOD)
+    assert s.roles[0].keywords == ["finite", "group"]
+    assert [r.alias for r in s.roles] == ["object1", "goal1"]
+    assert s.goal is not None and s.goal.kind == "goal"
+
+
+@pytest.mark.parametrize("payload", [
+    {"roles": []},
+    {"roles": [{"kind": "goal"}]},
+    {"roles": [{"kind": "object"}, {"kind": "object"}]},
+    {"roles": [{"kind": "goal"}, {"kind": "goal"}]},
+    ["not", "a", "dict"],
+])
+def test_unusable_replies_are_rejected(payload):
+    with pytest.raises(ValueError):
+        parse_source_simplex(payload)
+
+
+def test_unknown_kinds_are_dropped_with_a_note():
+    payload = {"roles": GOOD["roles"] + [{"kind": "banana"}]}
+    s = parse_source_simplex(payload)
+    assert len(s.roles) == 2 and s.notes
+
+
+def test_bad_introduced_defaults_to_given():
+    payload = {"roles": [
+        {"kind": "object", "introduced": "sometimes"}, {"kind": "goal"}]}
+    assert parse_source_simplex(payload).roles[0].introduced == "given"
+
+
+def test_extract_retries_once_then_succeeds():
+    client = ScriptedClient([json.dumps({"roles": []}), json.dumps(GOOD)])
+    s = extract_source_simplex(client, "Let G be a finite group.", "Prop 1")
+    assert s.goal is not None
+    assert len(client.calls) == 2
+
+
+def test_extract_gives_up_loudly():
+    client = ScriptedClient([json.dumps({"roles": []})] * 2)
+    with pytest.raises(ValueError, match="no usable source simplex"):
+        extract_source_simplex(client, "x")
+
+
+def test_prompt_never_shows_lean():
+    messages = passage_prompt("Let   G be a group.\n", "Theorem 2")
+    assert "Lean" in SYSTEM_PROMPT  # only to forbid writing it
+    assert "Do not write Lean" in SYSTEM_PROMPT
+    assert "Label: Theorem 2" in messages[1]["content"]
+    assert "Let G be a group." in messages[1]["content"]