Harden long-running behavioral evaluations

logicbench/client.py

@@ -14,7 +14,7 @@ class LLMClientConfig:
     api_key: str
     base_url: str = "https://api.openai.com/v1"
     api_style: str = "responses"
-    timeout_seconds: int = 120
+    timeout_seconds: int = 240
 
     @classmethod
     def from_env(cls, role: str) -> "LLMClientConfig":
@@ -23,6 +23,7 @@ class LLMClientConfig:
         api_key = os.getenv(f"{prefix}_API_KEY", "").strip()
         base_url = os.getenv(f"{prefix}_BASE_URL", "").strip()
         api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses"
+        timeout_text = os.getenv(f"{prefix}_TIMEOUT_SECONDS", "").strip()
         if not model:
             raise ValueError(f"{prefix}_MODEL is required")
         if not api_key:
@@ -31,7 +32,16 @@ class LLMClientConfig:
             raise ValueError(f"{prefix}_BASE_URL is required")
         if api_style not in {"responses", "chat_completions"}:
             raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions")
-        return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style)
+        timeout_seconds = int(timeout_text) if timeout_text else 240
+        if timeout_seconds < 1:
+            raise ValueError(f"{prefix}_TIMEOUT_SECONDS must be positive")
+        return cls(
+            model=model,
+            api_key=api_key,
+            base_url=base_url.rstrip("/"),
+            api_style=api_style,
+            timeout_seconds=timeout_seconds,
+        )
 
 
 class LLMClient:
@@ -64,6 +74,7 @@ class LLMClient:
                         "schema": schema,
                     }
                 },
+                "max_output_tokens": 2500,
             }
         else:
             path = "/chat/completions"
@@ -77,6 +88,7 @@ class LLMClient:
                     "type": "json_schema",
                     "json_schema": {"name": schema_name, "strict": True, "schema": schema},
                 },
+                "max_tokens": 2500,
             }
 
         text = self._post(path, body)
@@ -97,6 +109,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
+                "max_output_tokens": 1200,
             }
         else:
             path = "/chat/completions"
@@ -106,6 +119,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
+                "max_tokens": 1200,
             }
         return self._post(path, body)
 
@@ -127,6 +141,10 @@ class LLMClient:
             raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc
         except urllib.error.URLError as exc:
             raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
+        except TimeoutError as exc:
+            raise RuntimeError(
+                f"LLM API request timed out after {self.config.timeout_seconds} seconds"
+            ) from exc
 
         return self._extract_text(payload)