Score unparseable answers without aborting batches

2026-09-23 00:52:36 +08:00
parent 06cdd9c556
commit 6f929437a2
3 changed files with 33 additions and 4 deletions
@@ -84,6 +84,23 @@ class PipelineTests(unittest.TestCase):
                )
            self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())

+    def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
+        renderer = QueueClient([{"problem": "A compact behavioral problem."}])
+        solver = QueueClient(["The response ended before its final label."])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            summary = run_evaluation(
+                renderer=renderer,
+                solver=solver,
+                count=1,
+                length="short",
+                seed=1,
+                output_dir=output,
+            )
+            self.assertEqual(summary["parsed"], 0)
+            self.assertEqual(summary["correct"], 0)
+            self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
+

 if __name__ == "__main__":
    unittest.main()