Capture full solver output in five-case evaluation

This commit is contained in:
2026-09-22 23:07:09 +08:00
parent 3b50b8201d
commit fab07f80b0
6 changed files with 83 additions and 42 deletions
+32 -8
View File
@@ -79,6 +79,37 @@ class LLMClient:
},
}
text = self._post(path, body)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
def request_text(self, *, system: str, user: str) -> str:
if self.config.api_style == "responses":
path = "/responses"
body = {
"model": self.config.model,
"input": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
else:
path = "/chat/completions"
body = {
"model": self.config.model,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
return self._post(path, body)
def _post(self, path: str, body: dict[str, Any]) -> str:
request = urllib.request.Request(
self.config.base_url + path,
data=json.dumps(body).encode("utf-8"),
@@ -97,14 +128,7 @@ class LLMClient:
except urllib.error.URLError as exc:
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
text = self._extract_text(payload)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
return self._extract_text(payload)
def _extract_text(self, payload: dict[str, Any]) -> str:
if self.config.api_style == "chat_completions":
+23 -21
View File
@@ -19,6 +19,8 @@ class JsonModel(Protocol):
schema: dict[str, Any],
) -> dict[str, Any]: ...
def request_text(self, *, system: str, user: str) -> str: ...
@dataclass(frozen=True)
class EvaluationResult:
@@ -28,6 +30,7 @@ class EvaluationResult:
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
solver_output: str
model_answer: str
correct: bool
@@ -60,28 +63,26 @@ def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
return problem.strip()
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
schema = {
"type": "object",
"properties": {
"answer": {"type": "string", "enum": list(answer_options)},
},
"required": ["answer"],
"additionalProperties": False,
}
response = client.request_json(
system=(
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
"label. Do not use unstated real-world assumptions."
),
user=problem,
schema_name="solver_answer",
schema=schema,
)
answer = response.get("answer")
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
lines = [line.strip() for line in output.splitlines() if line.strip()]
if not lines or not lines[-1].startswith("####"):
raise RuntimeError("solver output must end with '#### <answer>'")
answer = lines[-1][4:].strip()
if answer not in answer_options:
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
return str(answer)
return answer
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
output = client.request_text(
system=(
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}",
)
return output, parse_marked_answer(output, answer_options)
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
@@ -106,7 +107,7 @@ def run_evaluation(
results: list[EvaluationResult] = []
for skeleton in skeletons:
problem = render_skeleton(renderer, skeleton)
answer = solve_problem(solver, problem, skeleton.answer_options)
solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
results.append(
EvaluationResult(
skeleton_id=skeleton.id,
@@ -115,6 +116,7 @@ def run_evaluation(
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
solver_output=solver_output,
model_answer=answer,
correct=answer == skeleton.gold_answer,
)