import json import pytest from leanbench.llm import ScriptedClient from leanbench.srcsimplex import ( SYSTEM_PROMPT, extract_source_simplex, parse_source_simplex, passage_prompt, ) GOOD = { "roles": [ {"kind": "object", "symbols": ["G"], "description": "finite group", "keywords": ["Finite", "group"], "introduced": "given"}, {"kind": "goal", "symbols": [], "description": "g^|G| = e", "keywords": []}, ], "goal_has_exists": False, } def test_parse_lowercases_keywords_and_numbers_aliases(): s = parse_source_simplex(GOOD) assert s.roles[0].keywords == ["finite", "group"] assert [r.alias for r in s.roles] == ["object1", "goal1"] assert s.goal is not None and s.goal.kind == "goal" @pytest.mark.parametrize("payload", [ {"roles": []}, {"roles": [{"kind": "goal"}]}, {"roles": [{"kind": "object"}, {"kind": "object"}]}, {"roles": [{"kind": "goal"}, {"kind": "goal"}]}, ["not", "a", "dict"], ]) def test_unusable_replies_are_rejected(payload): with pytest.raises(ValueError): parse_source_simplex(payload) def test_unknown_kinds_are_dropped_with_a_note(): payload = {"roles": GOOD["roles"] + [{"kind": "banana"}]} s = parse_source_simplex(payload) assert len(s.roles) == 2 and s.notes def test_bad_introduced_defaults_to_given(): payload = {"roles": [ {"kind": "object", "introduced": "sometimes"}, {"kind": "goal"}]} assert parse_source_simplex(payload).roles[0].introduced == "given" def test_extract_retries_once_then_succeeds(): client = ScriptedClient([json.dumps({"roles": []}), json.dumps(GOOD)]) s = extract_source_simplex(client, "Let G be a finite group.", "Prop 1") assert s.goal is not None assert len(client.calls) == 2 def test_extract_gives_up_loudly(): client = ScriptedClient([json.dumps({"roles": []})] * 2) with pytest.raises(ValueError, match="no usable source simplex"): extract_source_simplex(client, "x") def test_prompt_never_shows_lean(): messages = passage_prompt("Let G be a group.\n", "Theorem 2") assert "Lean" in SYSTEM_PROMPT # only to forbid writing it assert "Do not write Lean" in SYSTEM_PROMPT assert "Label: Theorem 2" in messages[1]["content"] assert "Let G be a group." in messages[1]["content"]