Preserve and recover solver outputs
This commit is contained in:
+10
-1
@@ -2,7 +2,7 @@ import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from logicbench.pipeline import parse_marked_answer, run_evaluation
|
||||
from logicbench.pipeline import parse_marked_answer, parse_solver_answer, run_evaluation
|
||||
|
||||
|
||||
class QueueClient:
|
||||
@@ -49,9 +49,11 @@ class PipelineTests(unittest.TestCase):
|
||||
)
|
||||
self.assertTrue((output / "skeletons.jsonl").is_file())
|
||||
self.assertTrue((output / "rendered.jsonl").is_file())
|
||||
self.assertTrue((output / "solver-outputs.jsonl").is_file())
|
||||
self.assertTrue((output / "results.jsonl").is_file())
|
||||
self.assertEqual(summary["count"], 5)
|
||||
self.assertEqual(summary["correct"], 5)
|
||||
self.assertEqual(summary["format_compliant"], 5)
|
||||
|
||||
def test_answer_parser_requires_final_marker_and_permitted_label(self):
|
||||
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
|
||||
@@ -60,6 +62,13 @@ class PipelineTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
|
||||
parse_marked_answer("#### MAYBE", ("YES", "NO"))
|
||||
|
||||
def test_answer_parser_recovers_unique_label_on_final_line(self):
|
||||
self.assertEqual(
|
||||
parse_solver_answer("Reasoning.\nFinal answer: YES", ("YES", "NO")),
|
||||
("YES", False),
|
||||
)
|
||||
self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True))
|
||||
|
||||
def test_rendered_problem_is_checkpointed_before_solver_failure(self):
|
||||
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
Reference in New Issue
Block a user