diff --git a/README.md b/README.md index 939dc9f..7929f5a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Outputs under the task's `output_dir` are: - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call; - `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering; - `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing; -- `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores; +- `results.jsonl`: rendered problems, complete solver responses, parsed labels, parse errors, format compliance, gold probabilities, and scores; - `summary.json`: aggregate accuracy and structural metadata. ## Gitea Actions one-shot test diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index 07ff3b0..cbcfa79 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -33,8 +33,9 @@ class EvaluationResult: gold_answer: str gold_present_probability: float solver_output: str - model_answer: str + model_answer: str | None format_compliant: bool + parse_error: str | None correct: bool @@ -111,7 +112,8 @@ def request_solver_output(client: JsonModel, problem: str, answer_options: tuple system=( "Infer the stable hidden mechanism from the repeated behavioral observations, then predict which " "query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and " - "counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put " + "counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under " + "200 words, then put " "the final answer on the last non-empty line in the exact form '#### ', where " "is exactly one permitted answer label. Do not use unstated real-world assumptions." ), @@ -148,7 +150,13 @@ def run_evaluation( solver_output = request_solver_output(solver, problem, skeleton.answer_options) solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output}) _write_jsonl(output_dir / "solver-outputs.jsonl", solver_records) - answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options) + try: + answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options) + parse_error = None + except RuntimeError as exc: + answer = None + format_compliant = False + parse_error = str(exc) results.append( EvaluationResult( skeleton_id=skeleton.id, @@ -161,17 +169,21 @@ def run_evaluation( solver_output=solver_output, model_answer=answer, format_compliant=format_compliant, + parse_error=parse_error, correct=answer == skeleton.gold_answer, ) ) _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results]) correct = sum(item.correct for item in results) + parsed = sum(item.model_answer is not None for item in results) format_compliant = sum(item.format_compliant for item in results) summary = { "count": len(results), "correct": correct, "accuracy": correct / len(results), + "parsed": parsed, + "parse_rate": parsed / len(results), "format_compliant": format_compliant, "format_compliance": format_compliant / len(results), "length": length, diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 68385d0..590e461 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -84,6 +84,23 @@ class PipelineTests(unittest.TestCase): ) self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text()) + def test_unparseable_solver_output_is_scored_without_stopping_batch(self): + renderer = QueueClient([{"problem": "A compact behavioral problem."}]) + solver = QueueClient(["The response ended before its final label."]) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + summary = run_evaluation( + renderer=renderer, + solver=solver, + count=1, + length="short", + seed=1, + output_dir=output, + ) + self.assertEqual(summary["parsed"], 0) + self.assertEqual(summary["correct"], 0) + self.assertIn('"parse_error":', (output / "results.jsonl").read_text()) + if __name__ == "__main__": unittest.main()