Score unparseable answers without aborting batches

README.md

@@ -59,7 +59,7 @@ Outputs under the task's `output_dir` are:
 - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call;
 - `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering;
 - `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing;
-- `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores;
+- `results.jsonl`: rendered problems, complete solver responses, parsed labels, parse errors, format compliance, gold probabilities, and scores;
 - `summary.json`: aggregate accuracy and structural metadata.
 
 ## Gitea Actions one-shot test

logicbench/pipeline.py

@@ -33,8 +33,9 @@ class EvaluationResult:
     gold_answer: str
     gold_present_probability: float
     solver_output: str
-    model_answer: str
+    model_answer: str | None
     format_compliant: bool
+    parse_error: str | None
     correct: bool
 
 
@@ -111,7 +112,8 @@ def request_solver_output(client: JsonModel, problem: str, answer_options: tuple
         system=(
             "Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
             "query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
-            "counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
+            "counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
+            "200 words, then put "
             "the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
             "is exactly one permitted answer label. Do not use unstated real-world assumptions."
         ),
@@ -148,7 +150,13 @@ def run_evaluation(
         solver_output = request_solver_output(solver, problem, skeleton.answer_options)
         solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
         _write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
-        answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
+        try:
+            answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
+            parse_error = None
+        except RuntimeError as exc:
+            answer = None
+            format_compliant = False
+            parse_error = str(exc)
         results.append(
             EvaluationResult(
                 skeleton_id=skeleton.id,
@@ -161,17 +169,21 @@ def run_evaluation(
                 solver_output=solver_output,
                 model_answer=answer,
                 format_compliant=format_compliant,
+                parse_error=parse_error,
                 correct=answer == skeleton.gold_answer,
             )
         )
         _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
 
     correct = sum(item.correct for item in results)
+    parsed = sum(item.model_answer is not None for item in results)
     format_compliant = sum(item.format_compliant for item in results)
     summary = {
         "count": len(results),
         "correct": correct,
         "accuracy": correct / len(results),
+        "parsed": parsed,
+        "parse_rate": parsed / len(results),
         "format_compliant": format_compliant,
         "format_compliance": format_compliant / len(results),
         "length": length,

tests/test_pipeline.py

@@ -84,6 +84,23 @@ class PipelineTests(unittest.TestCase):
                 )
             self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
 
+    def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
+        renderer = QueueClient([{"problem": "A compact behavioral problem."}])
+        solver = QueueClient(["The response ended before its final label."])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            summary = run_evaluation(
+                renderer=renderer,
+                solver=solver,
+                count=1,
+                length="short",
+                seed=1,
+                output_dir=output,
+            )
+            self.assertEqual(summary["parsed"], 0)
+            self.assertEqual(summary["correct"], 0)
+            self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
+
 
 if __name__ == "__main__":
     unittest.main()