Capture full solver output in five-case evaluation
This commit is contained in:
+32
-8
@@ -79,6 +79,37 @@ class LLMClient:
|
||||
},
|
||||
}
|
||||
|
||||
text = self._post(path, body)
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("LLM API structured output must be an object")
|
||||
return result
|
||||
|
||||
def request_text(self, *, system: str, user: str) -> str:
|
||||
if self.config.api_style == "responses":
|
||||
path = "/responses"
|
||||
body = {
|
||||
"model": self.config.model,
|
||||
"input": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
else:
|
||||
path = "/chat/completions"
|
||||
body = {
|
||||
"model": self.config.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
return self._post(path, body)
|
||||
|
||||
def _post(self, path: str, body: dict[str, Any]) -> str:
|
||||
request = urllib.request.Request(
|
||||
self.config.base_url + path,
|
||||
data=json.dumps(body).encode("utf-8"),
|
||||
@@ -97,14 +128,7 @@ class LLMClient:
|
||||
except urllib.error.URLError as exc:
|
||||
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
|
||||
|
||||
text = self._extract_text(payload)
|
||||
try:
|
||||
result = json.loads(text)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise RuntimeError("LLM API returned non-JSON structured output") from exc
|
||||
if not isinstance(result, dict):
|
||||
raise RuntimeError("LLM API structured output must be an object")
|
||||
return result
|
||||
return self._extract_text(payload)
|
||||
|
||||
def _extract_text(self, payload: dict[str, Any]) -> str:
|
||||
if self.config.api_style == "chat_completions":
|
||||
|
||||
Reference in New Issue
Block a user