Capture full solver output in five-case evaluation
This commit is contained in:
@@ -23,12 +23,13 @@ jobs:
|
|||||||
- name: Run unit tests
|
- name: Run unit tests
|
||||||
run: python3 -m unittest discover -s tests -v
|
run: python3 -m unittest discover -s tests -v
|
||||||
|
|
||||||
- name: Run three short live cases
|
- name: Run five short live cases
|
||||||
run: python3 -m logicbench run tasks/short-smoke.json
|
run: python3 -m logicbench run tasks/short-smoke.json
|
||||||
|
|
||||||
- name: Show evaluation records and summary
|
- name: Show raw skeletons, evaluation records, and summary
|
||||||
if: always()
|
if: always()
|
||||||
run: |
|
run: |
|
||||||
|
test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true
|
||||||
test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
|
test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
|
||||||
test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
|
test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
|
||||||
|
|
||||||
|
|||||||
@@ -57,9 +57,9 @@ Outputs are written under the task's `output_dir`:
|
|||||||
|
|
||||||
## Gitea Actions one-shot test
|
## Gitea Actions one-shot test
|
||||||
|
|
||||||
The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs three cases with `length: short` and has a 15-minute timeout.
|
The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs five cases with `length: short` and has a 15-minute timeout.
|
||||||
|
|
||||||
The run log prints the rendered questions and scores. The same records, original answer-bearing skeletons, and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection.
|
The run log prints every raw answer-bearing skeleton followed by each rendered question, the solver's complete text, its parsed `#### <answer>` label, the gold answer, and correctness. The same records and summary are retained in the `short-abstraction-evaluation` Actions artifact for inspection.
|
||||||
|
|
||||||
Create these repository Actions settings:
|
Create these repository Actions settings:
|
||||||
|
|
||||||
|
|||||||
+32
-8
@@ -79,6 +79,37 @@ class LLMClient:
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
text = self._post(path, body)
|
||||||
|
try:
|
||||||
|
result = json.loads(text)
|
||||||
|
except json.JSONDecodeError as exc:
|
||||||
|
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
||||||
|
if not isinstance(result, dict):
|
||||||
|
raise RuntimeError("LLM API structured output must be an object")
|
||||||
|
return result
|
||||||
|
|
||||||
|
def request_text(self, *, system: str, user: str) -> str:
|
||||||
|
if self.config.api_style == "responses":
|
||||||
|
path = "/responses"
|
||||||
|
body = {
|
||||||
|
"model": self.config.model,
|
||||||
|
"input": [
|
||||||
|
{"role": "system", "content": system},
|
||||||
|
{"role": "user", "content": user},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
path = "/chat/completions"
|
||||||
|
body = {
|
||||||
|
"model": self.config.model,
|
||||||
|
"messages": [
|
||||||
|
{"role": "system", "content": system},
|
||||||
|
{"role": "user", "content": user},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
return self._post(path, body)
|
||||||
|
|
||||||
|
def _post(self, path: str, body: dict[str, Any]) -> str:
|
||||||
request = urllib.request.Request(
|
request = urllib.request.Request(
|
||||||
self.config.base_url + path,
|
self.config.base_url + path,
|
||||||
data=json.dumps(body).encode("utf-8"),
|
data=json.dumps(body).encode("utf-8"),
|
||||||
@@ -97,14 +128,7 @@ class LLMClient:
|
|||||||
except urllib.error.URLError as exc:
|
except urllib.error.URLError as exc:
|
||||||
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
|
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
|
||||||
|
|
||||||
text = self._extract_text(payload)
|
return self._extract_text(payload)
|
||||||
try:
|
|
||||||
result = json.loads(text)
|
|
||||||
except json.JSONDecodeError as exc:
|
|
||||||
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
|
||||||
if not isinstance(result, dict):
|
|
||||||
raise RuntimeError("LLM API structured output must be an object")
|
|
||||||
return result
|
|
||||||
|
|
||||||
def _extract_text(self, payload: dict[str, Any]) -> str:
|
def _extract_text(self, payload: dict[str, Any]) -> str:
|
||||||
if self.config.api_style == "chat_completions":
|
if self.config.api_style == "chat_completions":
|
||||||
|
|||||||
+23
-21
@@ -19,6 +19,8 @@ class JsonModel(Protocol):
|
|||||||
schema: dict[str, Any],
|
schema: dict[str, Any],
|
||||||
) -> dict[str, Any]: ...
|
) -> dict[str, Any]: ...
|
||||||
|
|
||||||
|
def request_text(self, *, system: str, user: str) -> str: ...
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class EvaluationResult:
|
class EvaluationResult:
|
||||||
@@ -28,6 +30,7 @@ class EvaluationResult:
|
|||||||
rendered_problem: str
|
rendered_problem: str
|
||||||
answer_options: tuple[str, ...]
|
answer_options: tuple[str, ...]
|
||||||
gold_answer: str
|
gold_answer: str
|
||||||
|
solver_output: str
|
||||||
model_answer: str
|
model_answer: str
|
||||||
correct: bool
|
correct: bool
|
||||||
|
|
||||||
@@ -60,28 +63,26 @@ def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
|
|||||||
return problem.strip()
|
return problem.strip()
|
||||||
|
|
||||||
|
|
||||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
|
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||||
schema = {
|
lines = [line.strip() for line in output.splitlines() if line.strip()]
|
||||||
"type": "object",
|
if not lines or not lines[-1].startswith("####"):
|
||||||
"properties": {
|
raise RuntimeError("solver output must end with '#### <answer>'")
|
||||||
"answer": {"type": "string", "enum": list(answer_options)},
|
answer = lines[-1][4:].strip()
|
||||||
},
|
|
||||||
"required": ["answer"],
|
|
||||||
"additionalProperties": False,
|
|
||||||
}
|
|
||||||
response = client.request_json(
|
|
||||||
system=(
|
|
||||||
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
|
|
||||||
"label. Do not use unstated real-world assumptions."
|
|
||||||
),
|
|
||||||
user=problem,
|
|
||||||
schema_name="solver_answer",
|
|
||||||
schema=schema,
|
|
||||||
)
|
|
||||||
answer = response.get("answer")
|
|
||||||
if answer not in answer_options:
|
if answer not in answer_options:
|
||||||
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
|
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
|
||||||
return str(answer)
|
return answer
|
||||||
|
|
||||||
|
|
||||||
|
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
|
||||||
|
output = client.request_text(
|
||||||
|
system=(
|
||||||
|
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
|
||||||
|
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||||
|
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||||
|
),
|
||||||
|
user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}",
|
||||||
|
)
|
||||||
|
return output, parse_marked_answer(output, answer_options)
|
||||||
|
|
||||||
|
|
||||||
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||||
@@ -106,7 +107,7 @@ def run_evaluation(
|
|||||||
results: list[EvaluationResult] = []
|
results: list[EvaluationResult] = []
|
||||||
for skeleton in skeletons:
|
for skeleton in skeletons:
|
||||||
problem = render_skeleton(renderer, skeleton)
|
problem = render_skeleton(renderer, skeleton)
|
||||||
answer = solve_problem(solver, problem, skeleton.answer_options)
|
solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
|
||||||
results.append(
|
results.append(
|
||||||
EvaluationResult(
|
EvaluationResult(
|
||||||
skeleton_id=skeleton.id,
|
skeleton_id=skeleton.id,
|
||||||
@@ -115,6 +116,7 @@ def run_evaluation(
|
|||||||
rendered_problem=problem,
|
rendered_problem=problem,
|
||||||
answer_options=skeleton.answer_options,
|
answer_options=skeleton.answer_options,
|
||||||
gold_answer=skeleton.gold_answer,
|
gold_answer=skeleton.gold_answer,
|
||||||
|
solver_output=solver_output,
|
||||||
model_answer=answer,
|
model_answer=answer,
|
||||||
correct=answer == skeleton.gold_answer,
|
correct=answer == skeleton.gold_answer,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"name": "short-smoke",
|
"name": "short-smoke",
|
||||||
"description": "Render and solve one short example from each initial skeleton family.",
|
"description": "Render and solve five short examples across the initial skeleton families.",
|
||||||
"count": 3,
|
"count": 5,
|
||||||
"length": "short",
|
"length": "short",
|
||||||
"seed": 20260922,
|
"seed": 20260922,
|
||||||
"output_dir": "artifacts/short-smoke"
|
"output_dir": "artifacts/short-smoke"
|
||||||
|
|||||||
+21
-7
@@ -2,7 +2,7 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from logicbench.pipeline import run_evaluation
|
from logicbench.pipeline import parse_marked_answer, run_evaluation
|
||||||
|
|
||||||
|
|
||||||
class QueueClient:
|
class QueueClient:
|
||||||
@@ -12,6 +12,9 @@ class QueueClient:
|
|||||||
def request_json(self, **_kwargs):
|
def request_json(self, **_kwargs):
|
||||||
return next(self.responses)
|
return next(self.responses)
|
||||||
|
|
||||||
|
def request_text(self, **_kwargs):
|
||||||
|
return next(self.responses)
|
||||||
|
|
||||||
|
|
||||||
class PipelineTests(unittest.TestCase):
|
class PipelineTests(unittest.TestCase):
|
||||||
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
||||||
@@ -19,27 +22,38 @@ class PipelineTests(unittest.TestCase):
|
|||||||
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
||||||
{"problem": "Choose YES, NO, or UNKNOWN."},
|
{"problem": "Choose YES, NO, or UNKNOWN."},
|
||||||
{"problem": "Choose 9, 10, or 8."},
|
{"problem": "Choose 9, 10, or 8."},
|
||||||
|
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
||||||
|
{"problem": "Choose YES, NO, or UNKNOWN."},
|
||||||
])
|
])
|
||||||
# Seed 20260922 makes the third gold answer 9.
|
# Seed 20260922 makes the third gold answer 9.
|
||||||
solver = QueueClient([
|
solver = QueueClient([
|
||||||
{"answer": "LEFT_OF"},
|
"The relation follows by transitivity.\n#### LEFT_OF",
|
||||||
{"answer": "YES"},
|
"Membership propagates through the subset.\n#### YES",
|
||||||
{"answer": "9"},
|
"The two increments produce nine.\n#### 9",
|
||||||
|
"The relation follows by transitivity.\n#### LEFT_OF",
|
||||||
|
"Membership propagates through the subset.\n#### YES",
|
||||||
])
|
])
|
||||||
with tempfile.TemporaryDirectory() as directory:
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
output = Path(directory)
|
output = Path(directory)
|
||||||
summary = run_evaluation(
|
summary = run_evaluation(
|
||||||
renderer=renderer,
|
renderer=renderer,
|
||||||
solver=solver,
|
solver=solver,
|
||||||
count=3,
|
count=5,
|
||||||
length="short",
|
length="short",
|
||||||
seed=20260922,
|
seed=20260922,
|
||||||
output_dir=output,
|
output_dir=output,
|
||||||
)
|
)
|
||||||
self.assertTrue((output / "skeletons.jsonl").is_file())
|
self.assertTrue((output / "skeletons.jsonl").is_file())
|
||||||
self.assertTrue((output / "results.jsonl").is_file())
|
self.assertTrue((output / "results.jsonl").is_file())
|
||||||
self.assertEqual(summary["count"], 3)
|
self.assertEqual(summary["count"], 5)
|
||||||
self.assertEqual(summary["correct"], 3)
|
self.assertEqual(summary["correct"], 5)
|
||||||
|
|
||||||
|
def test_answer_parser_requires_final_marker_and_permitted_label(self):
|
||||||
|
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "must end"):
|
||||||
|
parse_marked_answer("#### YES\nextra", ("YES", "NO"))
|
||||||
|
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
|
||||||
|
parse_marked_answer("#### MAYBE", ("YES", "NO"))
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
Reference in New Issue
Block a user