Zayden-Jung/logic-skeleton-bench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Capture full solver output in five-case evaluation

Browse Source
This commit is contained in:
Codex Bot
2026-09-22 23:07:09 +08:00
parent 3b50b8201d
commit fab07f80b0
6 changed files with 83 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitea/workflows/short-eval.yml
+3 -2
View File
@@ -23,12 +23,13 @@ jobs:
- name: Run unit tests
run: python3 -m unittest discover -s tests -v
- name: Run three short live cases
- name: Run five short live cases
run: python3 -m logicbench run tasks/short-smoke.json
- name: Show evaluation records and summary
- name: Show raw skeletons, evaluation records, and summary
if: always()
run: |
test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true
test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
README.md
+2 -2
View File
@@ -57,9 +57,9 @@ Outputs are written under the task's `output_dir`:
## Gitea Actions one-shot test
The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs three cases with `length: short` and has a 15-minute timeout.
The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs five cases with `length: short` and has a 15-minute timeout.
The run log prints the rendered questions and scores. The same records, original answer-bearing skeletons, and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection.
The run log prints every raw answer-bearing skeleton followed by each rendered question, the solver's complete text, its parsed `#### <answer>` label, the gold answer, and correctness. The same records and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection.
Create these repository Actions settings:
logicbench/client.py
+32 -8
View File
@@ -79,6 +79,37 @@ class LLMClient:
},
}
text = self._post(path, body)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
def request_text(self, *, system: str, user: str) -> str:
if self.config.api_style == "responses":
path = "/responses"
body = {
"model": self.config.model,
"input": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
else:
path = "/chat/completions"
body = {
"model": self.config.model,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
return self._post(path, body)
def _post(self, path: str, body: dict[str, Any]) -> str:
request = urllib.request.Request(
self.config.base_url + path,
data=json.dumps(body).encode("utf-8"),
@@ -97,14 +128,7 @@ class LLMClient:
except urllib.error.URLError as exc:
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
text = self._extract_text(payload)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
return self._extract_text(payload)
def _extract_text(self, payload: dict[str, Any]) -> str:
if self.config.api_style == "chat_completions":
logicbench/pipeline.py
+23 -21
View File
@@ -19,6 +19,8 @@ class JsonModel(Protocol):
schema: dict[str, Any],
) -> dict[str, Any]: ...
def request_text(self, *, system: str, user: str) -> str: ...
@dataclass(frozen=True)
class EvaluationResult:
@@ -28,6 +30,7 @@ class EvaluationResult:
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
solver_output: str
model_answer: str
correct: bool
@@ -60,28 +63,26 @@ def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
return problem.strip()
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
schema = {
"type": "object",
"properties": {
"answer": {"type": "string", "enum": list(answer_options)},
},
"required": ["answer"],
"additionalProperties": False,
}
response = client.request_json(
system=(
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
"label. Do not use unstated real-world assumptions."
),
user=problem,
schema_name="solver_answer",
schema=schema,
)
answer = response.get("answer")
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
lines = [line.strip() for line in output.splitlines() if line.strip()]
if not lines or not lines[-1].startswith("####"):
raise RuntimeError("solver output must end with '#### <answer>'")
answer = lines[-1][4:].strip()
if answer not in answer_options:
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
return str(answer)
return answer
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
output = client.request_text(
system=(
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}",
)
return output, parse_marked_answer(output, answer_options)
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
@@ -106,7 +107,7 @@ def run_evaluation(
results: list[EvaluationResult] = []
for skeleton in skeletons:
problem = render_skeleton(renderer, skeleton)
answer = solve_problem(solver, problem, skeleton.answer_options)
solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
results.append(
EvaluationResult(
skeleton_id=skeleton.id,
@@ -115,6 +116,7 @@ def run_evaluation(
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
solver_output=solver_output,
model_answer=answer,
correct=answer == skeleton.gold_answer,
)
tasks/short-smoke.json
+2 -2
View File
@@ -1,7 +1,7 @@
{
"name": "short-smoke",
"description": "Render and solve one short example from each initial skeleton family.",
"count": 3,
"description": "Render and solve five short examples across the initial skeleton families.",
"count": 5,
"length": "short",
"seed": 20260922,
"output_dir": "artifacts/short-smoke"
tests/test_pipeline.py
+21 -7
View File
@@ -2,7 +2,7 @@ import tempfile
import unittest
from pathlib import Path
from logicbench.pipeline import run_evaluation
from logicbench.pipeline import parse_marked_answer, run_evaluation
class QueueClient:
@@ -12,6 +12,9 @@ class QueueClient:
def request_json(self, **_kwargs):
return next(self.responses)
def request_text(self, **_kwargs):
return next(self.responses)
class PipelineTests(unittest.TestCase):
def test_end_to_end_writes_skeletons_before_scored_results(self):
@@ -19,27 +22,38 @@ class PipelineTests(unittest.TestCase):
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
{"problem": "Choose YES, NO, or UNKNOWN."},
{"problem": "Choose 9, 10, or 8."},
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
{"problem": "Choose YES, NO, or UNKNOWN."},
])
# Seed 20260922 makes the third gold answer 9.
solver = QueueClient([
{"answer": "LEFT_OF"},
{"answer": "YES"},
{"answer": "9"},
"The relation follows by transitivity.\n#### LEFT_OF",
"Membership propagates through the subset.\n#### YES",
"The two increments produce nine.\n#### 9",
"The relation follows by transitivity.\n#### LEFT_OF",
"Membership propagates through the subset.\n#### YES",
])
with tempfile.TemporaryDirectory() as directory:
output = Path(directory)
summary = run_evaluation(
renderer=renderer,
solver=solver,
count=3,
count=5,
length="short",
seed=20260922,
output_dir=output,
)
self.assertTrue((output / "skeletons.jsonl").is_file())
self.assertTrue((output / "results.jsonl").is_file())
self.assertEqual(summary["count"], 3)
self.assertEqual(summary["correct"], 3)
self.assertEqual(summary["count"], 5)
self.assertEqual(summary["correct"], 5)
def test_answer_parser_requires_final_marker_and_permitted_label(self):
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
with self.assertRaisesRegex(RuntimeError, "must end"):
parse_marked_answer("#### YES\nextra", ("YES", "NO"))
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
parse_marked_answer("#### MAYBE", ("YES", "NO"))
if __name__ == "__main__":