61 lines
2.2 KiB
Python
61 lines
2.2 KiB
Python
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from logicbench.pipeline import parse_marked_answer, run_evaluation
|
|
|
|
|
|
class QueueClient:
|
|
def __init__(self, responses):
|
|
self.responses = iter(responses)
|
|
|
|
def request_json(self, **_kwargs):
|
|
return next(self.responses)
|
|
|
|
def request_text(self, **_kwargs):
|
|
return next(self.responses)
|
|
|
|
|
|
class PipelineTests(unittest.TestCase):
|
|
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
|
renderer = QueueClient([
|
|
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
|
{"problem": "Choose YES, NO, or UNKNOWN."},
|
|
{"problem": "Choose 9, 10, or 8."},
|
|
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
|
{"problem": "Choose YES, NO, or UNKNOWN."},
|
|
])
|
|
# Seed 20260922 makes the third gold answer 9.
|
|
solver = QueueClient([
|
|
"The relation follows by transitivity.\n#### LEFT_OF",
|
|
"Membership propagates through the subset.\n#### YES",
|
|
"The two increments produce nine.\n#### 9",
|
|
"The relation follows by transitivity.\n#### LEFT_OF",
|
|
"Membership propagates through the subset.\n#### YES",
|
|
])
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
summary = run_evaluation(
|
|
renderer=renderer,
|
|
solver=solver,
|
|
count=5,
|
|
length="short",
|
|
seed=20260922,
|
|
output_dir=output,
|
|
)
|
|
self.assertTrue((output / "skeletons.jsonl").is_file())
|
|
self.assertTrue((output / "results.jsonl").is_file())
|
|
self.assertEqual(summary["count"], 5)
|
|
self.assertEqual(summary["correct"], 5)
|
|
|
|
def test_answer_parser_requires_final_marker_and_permitted_label(self):
|
|
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
|
|
with self.assertRaisesRegex(RuntimeError, "must end"):
|
|
parse_marked_answer("#### YES\nextra", ("YES", "NO"))
|
|
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
|
|
parse_marked_answer("#### MAYBE", ("YES", "NO"))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|