Capture full solver output in five-case evaluation

This commit is contained in:
2026-09-22 23:07:09 +08:00
parent 3b50b8201d
commit fab07f80b0
6 changed files with 83 additions and 42 deletions
+21 -7
View File
@@ -2,7 +2,7 @@ import tempfile
import unittest
from pathlib import Path
from logicbench.pipeline import run_evaluation
from logicbench.pipeline import parse_marked_answer, run_evaluation
class QueueClient:
@@ -12,6 +12,9 @@ class QueueClient:
def request_json(self, **_kwargs):
return next(self.responses)
def request_text(self, **_kwargs):
return next(self.responses)
class PipelineTests(unittest.TestCase):
def test_end_to_end_writes_skeletons_before_scored_results(self):
@@ -19,27 +22,38 @@ class PipelineTests(unittest.TestCase):
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
{"problem": "Choose YES, NO, or UNKNOWN."},
{"problem": "Choose 9, 10, or 8."},
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
{"problem": "Choose YES, NO, or UNKNOWN."},
])
# Seed 20260922 makes the third gold answer 9.
solver = QueueClient([
{"answer": "LEFT_OF"},
{"answer": "YES"},
{"answer": "9"},
"The relation follows by transitivity.\n#### LEFT_OF",
"Membership propagates through the subset.\n#### YES",
"The two increments produce nine.\n#### 9",
"The relation follows by transitivity.\n#### LEFT_OF",
"Membership propagates through the subset.\n#### YES",
])
with tempfile.TemporaryDirectory() as directory:
output = Path(directory)
summary = run_evaluation(
renderer=renderer,
solver=solver,
count=3,
count=5,
length="short",
seed=20260922,
output_dir=output,
)
self.assertTrue((output / "skeletons.jsonl").is_file())
self.assertTrue((output / "results.jsonl").is_file())
self.assertEqual(summary["count"], 3)
self.assertEqual(summary["correct"], 3)
self.assertEqual(summary["count"], 5)
self.assertEqual(summary["correct"], 5)
def test_answer_parser_requires_final_marker_and_permitted_label(self):
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
with self.assertRaisesRegex(RuntimeError, "must end"):
parse_marked_answer("#### YES\nextra", ("YES", "NO"))
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
parse_marked_answer("#### MAYBE", ("YES", "NO"))
if __name__ == "__main__":