From afcd6c614e1b2ac17cd5c75d0175f3d1902bbd06 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 23 Sep 2026 00:19:35 +0800 Subject: [PATCH] Harden long-running behavioral evaluations --- README.md | 1 + logicbench/client.py | 22 ++++++++++++++++++++-- logicbench/pipeline.py | 3 ++- tests/test_client.py | 12 ++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 34d52f3..471dc5e 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ python3 -m logicbench run tasks/short-smoke.json | `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Required independent OpenAI-compatible endpoints | | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` | | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs | +| `RENDERER_TIMEOUT_SECONDS`, `SOLVER_TIMEOUT_SECONDS` | Optional per-request timeout; default `240` | Outputs under the task's `output_dir` are: diff --git a/logicbench/client.py b/logicbench/client.py index d85c38a..10ada86 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -14,7 +14,7 @@ class LLMClientConfig: api_key: str base_url: str = "https://api.openai.com/v1" api_style: str = "responses" - timeout_seconds: int = 120 + timeout_seconds: int = 240 @classmethod def from_env(cls, role: str) -> "LLMClientConfig": @@ -23,6 +23,7 @@ class LLMClientConfig: api_key = os.getenv(f"{prefix}_API_KEY", "").strip() base_url = os.getenv(f"{prefix}_BASE_URL", "").strip() api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses" + timeout_text = os.getenv(f"{prefix}_TIMEOUT_SECONDS", "").strip() if not model: raise ValueError(f"{prefix}_MODEL is required") if not api_key: @@ -31,7 +32,16 @@ class LLMClientConfig: raise ValueError(f"{prefix}_BASE_URL is required") if api_style not in {"responses", "chat_completions"}: raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions") - return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style) + timeout_seconds = int(timeout_text) if timeout_text else 240 + if timeout_seconds < 1: + raise ValueError(f"{prefix}_TIMEOUT_SECONDS must be positive") + return cls( + model=model, + api_key=api_key, + base_url=base_url.rstrip("/"), + api_style=api_style, + timeout_seconds=timeout_seconds, + ) class LLMClient: @@ -64,6 +74,7 @@ class LLMClient: "schema": schema, } }, + "max_output_tokens": 2500, } else: path = "/chat/completions" @@ -77,6 +88,7 @@ class LLMClient: "type": "json_schema", "json_schema": {"name": schema_name, "strict": True, "schema": schema}, }, + "max_tokens": 2500, } text = self._post(path, body) @@ -97,6 +109,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], + "max_output_tokens": 1200, } else: path = "/chat/completions" @@ -106,6 +119,7 @@ class LLMClient: {"role": "system", "content": system}, {"role": "user", "content": user}, ], + "max_tokens": 1200, } return self._post(path, body) @@ -127,6 +141,10 @@ class LLMClient: raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc except urllib.error.URLError as exc: raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc + except TimeoutError as exc: + raise RuntimeError( + f"LLM API request timed out after {self.config.timeout_seconds} seconds" + ) from exc return self._extract_text(payload) diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index bccedcd..60aa360 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -55,7 +55,8 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str: "are empirical observations, not explicit system laws: do not state or invent the hidden rules, " "trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that " "all trials use one unchanged system and that the question asks which final outcome is more likely, " - "not for an exact probability. End by requiring exactly one supplied answer label." + "not for an exact probability. Present repeated histories in a compact table rather than verbose " + "paragraphs. End by requiring exactly one supplied answer label." ), user=f"Render this intermediate representation:\n{prompt}", schema_name="rendered_problem", diff --git a/tests/test_client.py b/tests/test_client.py index 4fb9ff5..3222cb6 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -22,6 +22,18 @@ class ClientConfigTests(unittest.TestCase): self.assertEqual(solver.api_key, "solver-key") self.assertEqual(renderer.base_url, "https://renderer.example/v1") self.assertEqual(solver.base_url, "https://solver.example/v1") + self.assertEqual(renderer.timeout_seconds, 240) + + def test_role_timeout_can_be_overridden(self): + values = { + "SOLVER_API_KEY": "solver-key", + "SOLVER_BASE_URL": "https://solver.example/v1", + "SOLVER_MODEL": "solver-model", + "SOLVER_TIMEOUT_SECONDS": "360", + } + with patch.dict(os.environ, values, clear=True): + solver = LLMClientConfig.from_env("solver") + self.assertEqual(solver.timeout_seconds, 360) def test_missing_role_base_url_is_rejected(self): values = {