diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index 639ffda..22827c2 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -30,6 +30,7 @@ jobs: if: always() run: | test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true + test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true diff --git a/README.md b/README.md index 471dc5e..9924d6d 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ python3 -m logicbench run tasks/short-smoke.json Outputs under the task's `output_dir` are: - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call; +- `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering; - `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores; - `summary.json`: aggregate accuracy and structural metadata. diff --git a/logicbench/client.py b/logicbench/client.py index 10ada86..f8763b0 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -2,6 +2,7 @@ from __future__ import annotations import json import os +import socket import urllib.error import urllib.request from dataclasses import dataclass @@ -109,7 +110,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], - "max_output_tokens": 1200, + "max_output_tokens": 512, } else: path = "/chat/completions" @@ -119,7 +120,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], - "max_tokens": 1200, + "max_tokens": 512, } return self._post(path, body) @@ -141,7 +142,7 @@ class LLMClient: raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc except urllib.error.URLError as exc: raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc - except TimeoutError as exc: + except (TimeoutError, socket.timeout) as exc: raise RuntimeError( f"LLM API request timed out after {self.config.timeout_seconds} seconds" ) from exc diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index 60aa360..cab359e 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -112,8 +112,11 @@ def run_evaluation( _write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons]) results: list[EvaluationResult] = [] + rendered_records: list[dict[str, str]] = [] for skeleton in skeletons: problem = render_skeleton(renderer, skeleton) + rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem}) + _write_jsonl(output_dir / "rendered.jsonl", rendered_records) solver_output, answer = solve_problem(solver, problem, skeleton.answer_options) results.append( EvaluationResult( @@ -129,8 +132,8 @@ def run_evaluation( correct=answer == skeleton.gold_answer, ) ) + _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results]) - _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results]) correct = sum(item.correct for item in results) summary = { "count": len(results), diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index c01d821..fb2774e 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -16,6 +16,11 @@ class QueueClient: return next(self.responses) +class FailingSolver: + def request_text(self, **_kwargs): + raise RuntimeError("solver failed") + + class PipelineTests(unittest.TestCase): def test_end_to_end_writes_skeletons_before_scored_results(self): renderer = QueueClient([ @@ -43,6 +48,7 @@ class PipelineTests(unittest.TestCase): output_dir=output, ) self.assertTrue((output / "skeletons.jsonl").is_file()) + self.assertTrue((output / "rendered.jsonl").is_file()) self.assertTrue((output / "results.jsonl").is_file()) self.assertEqual(summary["count"], 5) self.assertEqual(summary["correct"], 5) @@ -54,6 +60,21 @@ class PipelineTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "invalid answer"): parse_marked_answer("#### MAYBE", ("YES", "NO")) + def test_rendered_problem_is_checkpointed_before_solver_failure(self): + renderer = QueueClient([{"problem": "A compact behavioral problem."}]) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + with self.assertRaisesRegex(RuntimeError, "solver failed"): + run_evaluation( + renderer=renderer, + solver=FailingSolver(), + count=1, + length="short", + seed=1, + output_dir=output, + ) + self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text()) + if __name__ == "__main__": unittest.main()