Capture full solver output in five-case evaluation

This commit is contained in:
2026-09-22 23:07:09 +08:00
parent 3b50b8201d
commit fab07f80b0
6 changed files with 83 additions and 42 deletions
+32 -8
View File
@@ -79,6 +79,37 @@ class LLMClient:
},
}
text = self._post(path, body)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
def request_text(self, *, system: str, user: str) -> str:
if self.config.api_style == "responses":
path = "/responses"
body = {
"model": self.config.model,
"input": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
else:
path = "/chat/completions"
body = {
"model": self.config.model,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
return self._post(path, body)
def _post(self, path: str, body: dict[str, Any]) -> str:
request = urllib.request.Request(
self.config.base_url + path,
data=json.dumps(body).encode("utf-8"),
@@ -97,14 +128,7 @@ class LLMClient:
except urllib.error.URLError as exc:
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
text = self._extract_text(payload)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
return self._extract_text(payload)
def _extract_text(self, payload: dict[str, Any]) -> str:
if self.config.api_style == "chat_completions":