Capture full solver output in five-case evaluation
This commit is contained in:
+32
-8
@@ -79,6 +79,37 @@ class LLMClient:
|
||||
},
|
||||
}
|
||||
|
||||
text = self._post(path, body)
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("LLM API structured output must be an object")
|
||||
return result
|
||||
|
||||
def request_text(self, *, system: str, user: str) -> str:
|
||||
if self.config.api_style == "responses":
|
||||
path = "/responses"
|
||||
body = {
|
||||
"model": self.config.model,
|
||||
"input": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
else:
|
||||
path = "/chat/completions"
|
||||
body = {
|
||||
"model": self.config.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
return self._post(path, body)
|
||||
|
||||
def _post(self, path: str, body: dict[str, Any]) -> str:
|
||||
request = urllib.request.Request(
|
||||
self.config.base_url + path,
|
||||
data=json.dumps(body).encode("utf-8"),
|
||||
@@ -97,14 +128,7 @@ class LLMClient:
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
|
||||
|
||||
text = self._extract_text(payload)
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("LLM API structured output must be an object")
|
||||
return result
|
||||
return self._extract_text(payload)
|
||||
|
||||
def _extract_text(self, payload: dict[str, Any]) -> str:
|
||||
if self.config.api_style == "chat_completions":
|
||||
|
||||
+23
-21
@@ -19,6 +19,8 @@ class JsonModel(Protocol):
|
||||
schema: dict[str, Any],
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
def request_text(self, *, system: str, user: str) -> str: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvaluationResult:
|
||||
@@ -28,6 +30,7 @@ class EvaluationResult:
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
solver_output: str
|
||||
model_answer: str
|
||||
correct: bool
|
||||
|
||||
@@ -60,28 +63,26 @@ def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
|
||||
return problem.strip()
|
||||
|
||||
|
||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answer": {"type": "string", "enum": list(answer_options)},
|
||||
},
|
||||
"required": ["answer"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
|
||||
"label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
user=problem,
|
||||
schema_name="solver_answer",
|
||||
schema=schema,
|
||||
)
|
||||
answer = response.get("answer")
|
||||
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||
lines = [line.strip() for line in output.splitlines() if line.strip()]
|
||||
if not lines or not lines[-1].startswith("####"):
|
||||
raise RuntimeError("solver output must end with '#### <answer>'")
|
||||
answer = lines[-1][4:].strip()
|
||||
if answer not in answer_options:
|
||||
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
|
||||
return str(answer)
|
||||
return answer
|
||||
|
||||
|
||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
|
||||
output = client.request_text(
|
||||
system=(
|
||||
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
user=f"{problem}\n\nPermitted answer labels: {', '.join(answer_options)}",
|
||||
)
|
||||
return output, parse_marked_answer(output, answer_options)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||
@@ -106,7 +107,7 @@ def run_evaluation(
|
||||
results: list[EvaluationResult] = []
|
||||
for skeleton in skeletons:
|
||||
problem = render_skeleton(renderer, skeleton)
|
||||
answer = solve_problem(solver, problem, skeleton.answer_options)
|
||||
solver_output, answer = solve_problem(solver, problem, skeleton.answer_options)
|
||||
results.append(
|
||||
EvaluationResult(
|
||||
skeleton_id=skeleton.id,
|
||||
@@ -115,6 +116,7 @@ def run_evaluation(
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
correct=answer == skeleton.gold_answer,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user