diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index 60b2592..188c2ab 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -32,6 +32,7 @@ jobs: run: | test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true + test -f artifacts/short-smoke/solver-outputs.jsonl && cat artifacts/short-smoke/solver-outputs.jsonl || true test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true diff --git a/README.md b/README.md index 03605fb..939dc9f 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Outputs under the task's `output_dir` are: - `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call; - `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering; +- `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing; - `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores; - `summary.json`: aggregate accuracy and structural metadata. diff --git a/logicbench/client.py b/logicbench/client.py index 52eb54c..300248d 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -119,7 +119,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], - "max_output_tokens": 512, + "max_output_tokens": 1024, } else: path = "/chat/completions" @@ -129,7 +129,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], - "max_tokens": 512, + "max_tokens": 1024, } if self.config.enable_thinking is not None: body["enable_thinking"] = self.config.enable_thinking diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index 4fc65ed..07ff3b0 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import re from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Protocol @@ -33,6 +34,7 @@ class EvaluationResult: gold_present_probability: float solver_output: str model_answer: str + format_compliant: bool correct: bool @@ -79,8 +81,33 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str: return answer +def parse_solver_answer(output: str, answer_options: tuple[str, ...]) -> tuple[str, bool]: + """Prefer the marker protocol, then recover one unambiguous label from the final line.""" + try: + return parse_marked_answer(output, answer_options), True + except RuntimeError as strict_error: + lines = [line.strip() for line in output.splitlines() if line.strip()] + if not lines: + raise strict_error + final_line = lines[-1] + matches = [ + option + for option in answer_options + if re.search(rf"(? tuple[str, str]: - output = client.request_text( + output = request_solver_output(client, problem, answer_options) + answer, _format_compliant = parse_solver_answer(output, answer_options) + return output, answer + + +def request_solver_output(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str: + return client.request_text( system=( "Infer the stable hidden mechanism from the repeated behavioral observations, then predict which " "query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and " @@ -90,7 +117,6 @@ def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, .. ), user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}", ) - return output, parse_marked_answer(output, answer_options) def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: @@ -114,11 +140,15 @@ def run_evaluation( results: list[EvaluationResult] = [] rendered_records: list[dict[str, str]] = [] + solver_records: list[dict[str, str]] = [] for skeleton in skeletons: problem = render_skeleton(renderer, skeleton) rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem}) _write_jsonl(output_dir / "rendered.jsonl", rendered_records) - solver_output, answer = solve_problem(solver, problem, skeleton.answer_options) + solver_output = request_solver_output(solver, problem, skeleton.answer_options) + solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output}) + _write_jsonl(output_dir / "solver-outputs.jsonl", solver_records) + answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options) results.append( EvaluationResult( skeleton_id=skeleton.id, @@ -130,16 +160,20 @@ def run_evaluation( gold_present_probability=skeleton.gold_present_probability, solver_output=solver_output, model_answer=answer, + format_compliant=format_compliant, correct=answer == skeleton.gold_answer, ) ) _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results]) correct = sum(item.correct for item in results) + format_compliant = sum(item.format_compliant for item in results) summary = { "count": len(results), "correct": correct, "accuracy": correct / len(results), + "format_compliant": format_compliant, + "format_compliance": format_compliant / len(results), "length": length, "complexity": skeletons[0].complexity, "seed": seed, diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index fb2774e..68385d0 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,7 +2,7 @@ import tempfile import unittest from pathlib import Path -from logicbench.pipeline import parse_marked_answer, run_evaluation +from logicbench.pipeline import parse_marked_answer, parse_solver_answer, run_evaluation class QueueClient: @@ -49,9 +49,11 @@ class PipelineTests(unittest.TestCase): ) self.assertTrue((output / "skeletons.jsonl").is_file()) self.assertTrue((output / "rendered.jsonl").is_file()) + self.assertTrue((output / "solver-outputs.jsonl").is_file()) self.assertTrue((output / "results.jsonl").is_file()) self.assertEqual(summary["count"], 5) self.assertEqual(summary["correct"], 5) + self.assertEqual(summary["format_compliant"], 5) def test_answer_parser_requires_final_marker_and_permitted_label(self): self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES") @@ -60,6 +62,13 @@ class PipelineTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, "invalid answer"): parse_marked_answer("#### MAYBE", ("YES", "NO")) + def test_answer_parser_recovers_unique_label_on_final_line(self): + self.assertEqual( + parse_solver_answer("Reasoning.\nFinal answer: YES", ("YES", "NO")), + ("YES", False), + ) + self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True)) + def test_rendered_problem_is_checkpointed_before_solver_failure(self): renderer = QueueClient([{"problem": "A compact behavioral problem."}]) with tempfile.TemporaryDirectory() as directory: