Checkpoint rendered cases before solving
This commit is contained in:
@@ -16,6 +16,11 @@ class QueueClient:
|
||||
return next(self.responses)
|
||||
|
||||
|
||||
class FailingSolver:
|
||||
def request_text(self, **_kwargs):
|
||||
raise RuntimeError("solver failed")
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
||||
renderer = QueueClient([
|
||||
@@ -43,6 +48,7 @@ class PipelineTests(unittest.TestCase):
|
||||
output_dir=output,
|
||||
)
|
||||
self.assertTrue((output / "skeletons.jsonl").is_file())
|
||||
self.assertTrue((output / "rendered.jsonl").is_file())
|
||||
self.assertTrue((output / "results.jsonl").is_file())
|
||||
self.assertEqual(summary["count"], 5)
|
||||
self.assertEqual(summary["correct"], 5)
|
||||
@@ -54,6 +60,21 @@ class PipelineTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
|
||||
parse_marked_answer("#### MAYBE", ("YES", "NO"))
|
||||
|
||||
def test_rendered_problem_is_checkpointed_before_solver_failure(self):
|
||||
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
with self.assertRaisesRegex(RuntimeError, "solver failed"):
|
||||
run_evaluation(
|
||||
renderer=renderer,
|
||||
solver=FailingSolver(),
|
||||
count=1,
|
||||
length="short",
|
||||
seed=1,
|
||||
output_dir=output,
|
||||
)
|
||||
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user