diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index ddc1874..74836c7 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -23,12 +23,13 @@ jobs: - name: Run unit tests run: python3 -m unittest discover -s tests -v - - name: Run three short live cases + - name: Run five short live cases run: python3 -m logicbench run tasks/short-smoke.json - - name: Show evaluation records and summary + - name: Show raw skeletons, evaluation records, and summary if: always() run: | + test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true diff --git a/README.md b/README.md index 91a7f8c..8f547a7 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,9 @@ Outputs are written under the task's `output_dir`: ## Gitea Actions one-shot test -The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs three cases with `length: short` and has a 15-minute timeout. +The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs five cases with `length: short` and has a 15-minute timeout. -The run log prints the rendered questions and scores. The same records, original answer-bearing skeletons, and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection. +The run log prints every raw answer-bearing skeleton followed by each rendered question, the solver's complete text, its parsed `#### ` label, the gold answer, and correctness. The same records and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection. Create these repository Actions settings: diff --git a/logicbench/client.py b/logicbench/client.py index 6c94f29..d85c38a 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -79,6 +79,37 @@ class LLMClient: }, } + text = self._post(path, body) + try: + result = json.loads(text) + except json.JSONDecodeError as exc: + raise RuntimeError("LLM API returned non-JSON structured output") from exc + if not isinstance(result, dict): + raise RuntimeError("LLM API structured output must be an object") + return result + + def request_text(self, *, system: str, user: str) -> str: + if self.config.api_style == "responses": + path = "/responses" + body = { + "model": self.config.model, + "input": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + } + else: + path = "/chat/completions" + body = { + "model": self.config.model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + } + return self._post(path, body) + + def _post(self, path: str, body: dict[str, Any]) -> str: request = urllib.request.Request( self.config.base_url + path, data=json.dumps(body).encode("utf-8"), @@ -97,14 +128,7 @@ class LLMClient: except urllib.error.URLError as exc: raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc - text = self._extract_text(payload) - try: - result = json.loads(text) - except json.JSONDecodeError as exc: - raise RuntimeError("LLM API returned non-JSON structured output") from exc - if not isinstance(result, dict): - raise RuntimeError("LLM API structured output must be an object") - return result + return self._extract_text(payload) def _extract_text(self, payload: dict[str, Any]) -> str: if self.config.api_style == "chat_completions": diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index d135e1b..1e9c205 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -19,6 +19,8 @@ class JsonModel(Protocol): schema: dict[str, Any], ) -> dict[str, Any]: ... + def request_text(self, *, system: str, user: str) -> str: ... + @dataclass(frozen=True) class EvaluationResult: @@ -28,6 +30,7 @@ class EvaluationResult: rendered_problem: str answer_options: tuple[str, ...] gold_answer: str + solver_output: str model_answer: str correct: bool @@ -60,28 +63,26 @@ def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str: return problem.strip() -def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str: - schema = { - "type": "object", - "properties": { - "answer": {"type": "string", "enum": list(answer_options)}, - }, - "required": ["answer"], - "additionalProperties": False, - } - response = client.request_json( - system=( - "Solve the reasoning problem from the stated facts only. Return exactly one permitted answer " - "label. Do not use unstated real-world assumptions." - ), - user=problem, - schema_name="solver_answer", - schema=schema, - ) - answer = response.get("answer") +def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str: + lines = [line.strip() for line in output.splitlines() if line.strip()] + if not lines or not lines[-1].startswith("####"): + raise RuntimeError("solver output must end with '#### '") + answer = lines[-1][4:].strip() if answer not in answer_options: raise RuntimeError(f"solver returned invalid answer label: {answer!r}") - return str(answer) + return answer + + +def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]: + output = client.request_text( + system=( + "Solve the reasoning problem from the stated facts only. Give a concise explanation, then put " + "the final answer on the last non-empty line in the exact form '#### ', where " + "is exactly one permitted answer label. Do not use unstated real-world assumptions." + ), + user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}", + ) + return output, parse_marked_answer(output, answer_options) def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: @@ -106,7 +107,7 @@ def run_evaluation( results: list[EvaluationResult] = [] for skeleton in skeletons: problem = render_skeleton(renderer, skeleton) - answer = solve_problem(solver, problem, skeleton.answer_options) + solver_output, answer = solve_problem(solver, problem, skeleton.answer_options) results.append( EvaluationResult( skeleton_id=skeleton.id, @@ -115,6 +116,7 @@ def run_evaluation( rendered_problem=problem, answer_options=skeleton.answer_options, gold_answer=skeleton.gold_answer, + solver_output=solver_output, model_answer=answer, correct=answer == skeleton.gold_answer, ) diff --git a/tasks/short-smoke.json b/tasks/short-smoke.json index 8a5ff48..2c33661 100644 --- a/tasks/short-smoke.json +++ b/tasks/short-smoke.json @@ -1,7 +1,7 @@ { "name": "short-smoke", - "description": "Render and solve one short example from each initial skeleton family.", - "count": 3, + "description": "Render and solve five short examples across the initial skeleton families.", + "count": 5, "length": "short", "seed": 20260922, "output_dir": "artifacts/short-smoke" diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index ea04616..0ec9ec7 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -2,7 +2,7 @@ import tempfile import unittest from pathlib import Path -from logicbench.pipeline import run_evaluation +from logicbench.pipeline import parse_marked_answer, run_evaluation class QueueClient: @@ -12,6 +12,9 @@ class QueueClient: def request_json(self, **_kwargs): return next(self.responses) + def request_text(self, **_kwargs): + return next(self.responses) + class PipelineTests(unittest.TestCase): def test_end_to_end_writes_skeletons_before_scored_results(self): @@ -19,27 +22,38 @@ class PipelineTests(unittest.TestCase): {"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."}, {"problem": "Choose YES, NO, or UNKNOWN."}, {"problem": "Choose 9, 10, or 8."}, + {"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."}, + {"problem": "Choose YES, NO, or UNKNOWN."}, ]) # Seed 20260922 makes the third gold answer 9. solver = QueueClient([ - {"answer": "LEFT_OF"}, - {"answer": "YES"}, - {"answer": "9"}, + "The relation follows by transitivity.\n#### LEFT_OF", + "Membership propagates through the subset.\n#### YES", + "The two increments produce nine.\n#### 9", + "The relation follows by transitivity.\n#### LEFT_OF", + "Membership propagates through the subset.\n#### YES", ]) with tempfile.TemporaryDirectory() as directory: output = Path(directory) summary = run_evaluation( renderer=renderer, solver=solver, - count=3, + count=5, length="short", seed=20260922, output_dir=output, ) self.assertTrue((output / "skeletons.jsonl").is_file()) self.assertTrue((output / "results.jsonl").is_file()) - self.assertEqual(summary["count"], 3) - self.assertEqual(summary["correct"], 3) + self.assertEqual(summary["count"], 5) + self.assertEqual(summary["correct"], 5) + + def test_answer_parser_requires_final_marker_and_permitted_label(self): + self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES") + with self.assertRaisesRegex(RuntimeError, "must end"): + parse_marked_answer("#### YES\nextra", ("YES", "NO")) + with self.assertRaisesRegex(RuntimeError, "invalid answer"): + parse_marked_answer("#### MAYBE", ("YES", "NO")) if __name__ == "__main__":