Preserve and recover solver outputs

.gitea/workflows/short-eval.yml

@@ -32,6 +32,7 @@ jobs:
         run: |
           test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true
           test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true
+          test -f artifacts/short-smoke/solver-outputs.jsonl && cat artifacts/short-smoke/solver-outputs.jsonl || true
           test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
           test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
 

README.md

@@ -58,6 +58,7 @@ Outputs under the task's `output_dir` are:
 
 - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call;
 - `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering;
+- `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing;
 - `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores;
 - `summary.json`: aggregate accuracy and structural metadata.
 

logicbench/client.py

@@ -119,7 +119,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
-                "max_output_tokens": 512,
+                "max_output_tokens": 1024,
             }
         else:
             path = "/chat/completions"
@@ -129,7 +129,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
-                "max_tokens": 512,
+                "max_tokens": 1024,
             }
         if self.config.enable_thinking is not None:
             body["enable_thinking"] = self.config.enable_thinking

logicbench/pipeline.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import json
+import re
 from dataclasses import asdict, dataclass
 from pathlib import Path
 from typing import Any, Protocol
@@ -33,6 +34,7 @@ class EvaluationResult:
     gold_present_probability: float
     solver_output: str
     model_answer: str
+    format_compliant: bool
     correct: bool
 
 
@@ -79,8 +81,33 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
     return answer
 
 
+def parse_solver_answer(output: str, answer_options: tuple[str, ...]) -> tuple[str, bool]:
+    """Prefer the marker protocol, then recover one unambiguous label from the final line."""
+    try:
+        return parse_marked_answer(output, answer_options), True
+    except RuntimeError as strict_error:
+        lines = [line.strip() for line in output.splitlines() if line.strip()]
+        if not lines:
+            raise strict_error
+        final_line = lines[-1]
+        matches = [
+            option
+            for option in answer_options
+            if re.search(rf"(?<![A-Za-z0-9_]){re.escape(option)}(?![A-Za-z0-9_])", final_line)
+        ]
+        if len(matches) == 1:
+            return matches[0], False
+        raise strict_error
+
+
 def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
-    output = client.request_text(
+    output = request_solver_output(client, problem, answer_options)
+    answer, _format_compliant = parse_solver_answer(output, answer_options)
+    return output, answer
+
+
+def request_solver_output(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
+    return client.request_text(
         system=(
             "Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
             "query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
@@ -90,7 +117,6 @@ def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ..
         ),
         user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}",
     )
-    return output, parse_marked_answer(output, answer_options)
 
 
 def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
@@ -114,11 +140,15 @@ def run_evaluation(
 
     results: list[EvaluationResult] = []
     rendered_records: list[dict[str, str]] = []
+    solver_records: list[dict[str, str]] = []
     for skeleton in skeletons:
         problem = render_skeleton(renderer, skeleton)
         rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem})
         _write_jsonl(output_dir / "rendered.jsonl", rendered_records)
-        solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
+        solver_output = request_solver_output(solver, problem, skeleton.answer_options)
+        solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
+        _write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
+        answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
         results.append(
             EvaluationResult(
                 skeleton_id=skeleton.id,
@@ -130,16 +160,20 @@ def run_evaluation(
                 gold_present_probability=skeleton.gold_present_probability,
                 solver_output=solver_output,
                 model_answer=answer,
+                format_compliant=format_compliant,
                 correct=answer == skeleton.gold_answer,
             )
         )
         _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
 
     correct = sum(item.correct for item in results)
+    format_compliant = sum(item.format_compliant for item in results)
     summary = {
         "count": len(results),
         "correct": correct,
         "accuracy": correct / len(results),
+        "format_compliant": format_compliant,
+        "format_compliance": format_compliant / len(results),
         "length": length,
         "complexity": skeletons[0].complexity,
         "seed": seed,

tests/test_pipeline.py

@@ -2,7 +2,7 @@ import tempfile
 import unittest
 from pathlib import Path
 
-from logicbench.pipeline import parse_marked_answer, run_evaluation
+from logicbench.pipeline import parse_marked_answer, parse_solver_answer, run_evaluation
 
 
 class QueueClient:
@@ -49,9 +49,11 @@ class PipelineTests(unittest.TestCase):
             )
             self.assertTrue((output / "skeletons.jsonl").is_file())
             self.assertTrue((output / "rendered.jsonl").is_file())
+            self.assertTrue((output / "solver-outputs.jsonl").is_file())
             self.assertTrue((output / "results.jsonl").is_file())
             self.assertEqual(summary["count"], 5)
             self.assertEqual(summary["correct"], 5)
+            self.assertEqual(summary["format_compliant"], 5)
 
     def test_answer_parser_requires_final_marker_and_permitted_label(self):
         self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
@@ -60,6 +62,13 @@ class PipelineTests(unittest.TestCase):
         with self.assertRaisesRegex(RuntimeError, "invalid answer"):
             parse_marked_answer("#### MAYBE", ("YES", "NO"))
 
+    def test_answer_parser_recovers_unique_label_on_final_line(self):
+        self.assertEqual(
+            parse_solver_answer("Reasoning.\nFinal answer: YES", ("YES", "NO")),
+            ("YES", False),
+        )
+        self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True))
+
     def test_rendered_problem_is_checkpointed_before_solver_failure(self):
         renderer = QueueClient([{"problem": "A compact behavioral problem."}])
         with tempfile.TemporaryDirectory() as directory: