claude-bot
ca98703538
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6
73 lines
2.3 KiB
Python
73 lines
2.3 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from leanbench.llm import ScriptedClient
|
|
from leanbench.srcsimplex import (
|
|
SYSTEM_PROMPT,
|
|
extract_source_simplex,
|
|
parse_source_simplex,
|
|
passage_prompt,
|
|
)
|
|
|
|
GOOD = {
|
|
"roles": [
|
|
{"kind": "object", "symbols": ["G"], "description": "finite group",
|
|
"keywords": ["Finite", "group"], "introduced": "given"},
|
|
{"kind": "goal", "symbols": [], "description": "g^|G| = e", "keywords": []},
|
|
],
|
|
"goal_has_exists": False,
|
|
}
|
|
|
|
|
|
def test_parse_lowercases_keywords_and_numbers_aliases():
|
|
s = parse_source_simplex(GOOD)
|
|
assert s.roles[0].keywords == ["finite", "group"]
|
|
assert [r.alias for r in s.roles] == ["object1", "goal1"]
|
|
assert s.goal is not None and s.goal.kind == "goal"
|
|
|
|
|
|
@pytest.mark.parametrize("payload", [
|
|
{"roles": []},
|
|
{"roles": [{"kind": "goal"}]},
|
|
{"roles": [{"kind": "object"}, {"kind": "object"}]},
|
|
{"roles": [{"kind": "goal"}, {"kind": "goal"}]},
|
|
["not", "a", "dict"],
|
|
])
|
|
def test_unusable_replies_are_rejected(payload):
|
|
with pytest.raises(ValueError):
|
|
parse_source_simplex(payload)
|
|
|
|
|
|
def test_unknown_kinds_are_dropped_with_a_note():
|
|
payload = {"roles": GOOD["roles"] + [{"kind": "banana"}]}
|
|
s = parse_source_simplex(payload)
|
|
assert len(s.roles) == 2 and s.notes
|
|
|
|
|
|
def test_bad_introduced_defaults_to_given():
|
|
payload = {"roles": [
|
|
{"kind": "object", "introduced": "sometimes"}, {"kind": "goal"}]}
|
|
assert parse_source_simplex(payload).roles[0].introduced == "given"
|
|
|
|
|
|
def test_extract_retries_once_then_succeeds():
|
|
client = ScriptedClient([json.dumps({"roles": []}), json.dumps(GOOD)])
|
|
s = extract_source_simplex(client, "Let G be a finite group.", "Prop 1")
|
|
assert s.goal is not None
|
|
assert len(client.calls) == 2
|
|
|
|
|
|
def test_extract_gives_up_loudly():
|
|
client = ScriptedClient([json.dumps({"roles": []})] * 2)
|
|
with pytest.raises(ValueError, match="no usable source simplex"):
|
|
extract_source_simplex(client, "x")
|
|
|
|
|
|
def test_prompt_never_shows_lean():
|
|
messages = passage_prompt("Let G be a group.\n", "Theorem 2")
|
|
assert "Lean" in SYSTEM_PROMPT # only to forbid writing it
|
|
assert "Do not write Lean" in SYSTEM_PROMPT
|
|
assert "Label: Theorem 2" in messages[1]["content"]
|
|
assert "Let G be a group." in messages[1]["content"]
|