Checkpoint rendered cases before solving

.gitea/workflows/short-eval.yml

@@ -30,6 +30,7 @@ jobs:
         if: always()
         run: |
           test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true
+          test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true
           test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
           test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
 

README.md

@@ -56,6 +56,7 @@ python3 -m logicbench run tasks/short-smoke.json
 Outputs under the task's `output_dir` are:
 
 - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call;
+- `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering;
 - `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores;
 - `summary.json`: aggregate accuracy and structural metadata.
 

logicbench/client.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import socket
 import urllib.error
 import urllib.request
 from dataclasses import dataclass
@@ -109,7 +110,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
-                "max_output_tokens": 1200,
+                "max_output_tokens": 512,
             }
         else:
             path = "/chat/completions"
@@ -119,7 +120,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
-                "max_tokens": 1200,
+                "max_tokens": 512,
             }
         return self._post(path, body)
 
@@ -141,7 +142,7 @@ class LLMClient:
             raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc
         except urllib.error.URLError as exc:
             raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
-        except TimeoutError as exc:
+        except (TimeoutError, socket.timeout) as exc:
             raise RuntimeError(
                 f"LLM API request timed out after {self.config.timeout_seconds} seconds"
             ) from exc

logicbench/pipeline.py

@@ -112,8 +112,11 @@ def run_evaluation(
     _write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
 
     results: list[EvaluationResult] = []
+    rendered_records: list[dict[str, str]] = []
     for skeleton in skeletons:
         problem = render_skeleton(renderer, skeleton)
+        rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem})
+        _write_jsonl(output_dir / "rendered.jsonl", rendered_records)
         solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
         results.append(
             EvaluationResult(
@@ -129,8 +132,8 @@ def run_evaluation(
                 correct=answer == skeleton.gold_answer,
             )
         )
+        _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
 
-    _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
     correct = sum(item.correct for item in results)
     summary = {
         "count": len(results),

tests/test_pipeline.py

@@ -16,6 +16,11 @@ class QueueClient:
         return next(self.responses)
 
 
+class FailingSolver:
+    def request_text(self, **_kwargs):
+        raise RuntimeError("solver failed")
+
+
 class PipelineTests(unittest.TestCase):
     def test_end_to_end_writes_skeletons_before_scored_results(self):
         renderer = QueueClient([
@@ -43,6 +48,7 @@ class PipelineTests(unittest.TestCase):
                 output_dir=output,
             )
             self.assertTrue((output / "skeletons.jsonl").is_file())
+            self.assertTrue((output / "rendered.jsonl").is_file())
             self.assertTrue((output / "results.jsonl").is_file())
             self.assertEqual(summary["count"], 5)
             self.assertEqual(summary["correct"], 5)
@@ -54,6 +60,21 @@ class PipelineTests(unittest.TestCase):
         with self.assertRaisesRegex(RuntimeError, "invalid answer"):
             parse_marked_answer("#### MAYBE", ("YES", "NO"))
 
+    def test_rendered_problem_is_checkpointed_before_solver_failure(self):
+        renderer = QueueClient([{"problem": "A compact behavioral problem."}])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            with self.assertRaisesRegex(RuntimeError, "solver failed"):
+                run_evaluation(
+                    renderer=renderer,
+                    solver=FailingSolver(),
+                    count=1,
+                    length="short",
+                    seed=1,
+                    output_dir=output,
+                )
+            self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
+
 
 if __name__ == "__main__":
     unittest.main()