Checkpoint rendered cases before solving

This commit is contained in:
2026-09-23 00:27:30 +08:00
parent afcd6c614e
commit b645190ea5
5 changed files with 31 additions and 4 deletions
+21
View File
@@ -16,6 +16,11 @@ class QueueClient:
return next(self.responses)
class FailingSolver:
def request_text(self, **_kwargs):
raise RuntimeError("solver failed")
class PipelineTests(unittest.TestCase):
def test_end_to_end_writes_skeletons_before_scored_results(self):
renderer = QueueClient([
@@ -43,6 +48,7 @@ class PipelineTests(unittest.TestCase):
output_dir=output,
)
self.assertTrue((output / "skeletons.jsonl").is_file())
self.assertTrue((output / "rendered.jsonl").is_file())
self.assertTrue((output / "results.jsonl").is_file())
self.assertEqual(summary["count"], 5)
self.assertEqual(summary["correct"], 5)
@@ -54,6 +60,21 @@ class PipelineTests(unittest.TestCase):
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
parse_marked_answer("#### MAYBE", ("YES", "NO"))
def test_rendered_problem_is_checkpointed_before_solver_failure(self):
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
with tempfile.TemporaryDirectory() as directory:
output = Path(directory)
with self.assertRaisesRegex(RuntimeError, "solver failed"):
run_evaluation(
renderer=renderer,
solver=FailingSolver(),
count=1,
length="short",
seed=1,
output_dir=output,
)
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
if __name__ == "__main__":
unittest.main()