Harden long-running behavioral evaluations

README.md

@@ -51,6 +51,7 @@ python3 -m logicbench run tasks/short-smoke.json
 | `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Required independent OpenAI-compatible endpoints |
 | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` |
 | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs |
+| `RENDERER_TIMEOUT_SECONDS`, `SOLVER_TIMEOUT_SECONDS` | Optional per-request timeout; default `240` |
 
 Outputs under the task's `output_dir` are:
 

logicbench/client.py

@@ -14,7 +14,7 @@ class LLMClientConfig:
     api_key: str
     base_url: str = "https://api.openai.com/v1"
     api_style: str = "responses"
-    timeout_seconds: int = 120
+    timeout_seconds: int = 240
 
     @classmethod
     def from_env(cls, role: str) -> "LLMClientConfig":
@@ -23,6 +23,7 @@ class LLMClientConfig:
         api_key = os.getenv(f"{prefix}_API_KEY", "").strip()
         base_url = os.getenv(f"{prefix}_BASE_URL", "").strip()
         api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses"
+        timeout_text = os.getenv(f"{prefix}_TIMEOUT_SECONDS", "").strip()
         if not model:
             raise ValueError(f"{prefix}_MODEL is required")
         if not api_key:
@@ -31,7 +32,16 @@ class LLMClientConfig:
             raise ValueError(f"{prefix}_BASE_URL is required")
         if api_style not in {"responses", "chat_completions"}:
             raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions")
-        return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style)
+        timeout_seconds = int(timeout_text) if timeout_text else 240
+        if timeout_seconds < 1:
+            raise ValueError(f"{prefix}_TIMEOUT_SECONDS must be positive")
+        return cls(
+            model=model,
+            api_key=api_key,
+            base_url=base_url.rstrip("/"),
+            api_style=api_style,
+            timeout_seconds=timeout_seconds,
+        )
 
 
 class LLMClient:
@@ -64,6 +74,7 @@ class LLMClient:
                         "schema": schema,
                     }
                 },
+                "max_output_tokens": 2500,
             }
         else:
             path = "/chat/completions"
@@ -77,6 +88,7 @@ class LLMClient:
                     "type": "json_schema",
                     "json_schema": {"name": schema_name, "strict": True, "schema": schema},
                 },
+                "max_tokens": 2500,
             }
 
         text = self._post(path, body)
@@ -97,6 +109,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
+                "max_output_tokens": 1200,
             }
         else:
             path = "/chat/completions"
@@ -106,6 +119,7 @@ class LLMClient:
                     {"role": "system", "content": system},
                     {"role": "user", "content": user},
                 ],
+                "max_tokens": 1200,
             }
         return self._post(path, body)
 
@@ -127,6 +141,10 @@ class LLMClient:
             raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc
         except urllib.error.URLError as exc:
             raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
+        except TimeoutError as exc:
+            raise RuntimeError(
+                f"LLM API request timed out after {self.config.timeout_seconds} seconds"
+            ) from exc
 
         return self._extract_text(payload)
 

logicbench/pipeline.py

@@ -55,7 +55,8 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
             "are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
             "trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
             "all trials use one unchanged system and that the question asks which final outcome is more likely, "
-            "not for an exact probability. End by requiring exactly one supplied answer label."
+            "not for an exact probability. Present repeated histories in a compact table rather than verbose "
+            "paragraphs. End by requiring exactly one supplied answer label."
         ),
         user=f"Render this intermediate representation:\n{prompt}",
         schema_name="rendered_problem",

tests/test_client.py

@@ -22,6 +22,18 @@ class ClientConfigTests(unittest.TestCase):
         self.assertEqual(solver.api_key, "solver-key")
         self.assertEqual(renderer.base_url, "https://renderer.example/v1")
         self.assertEqual(solver.base_url, "https://solver.example/v1")
+        self.assertEqual(renderer.timeout_seconds, 240)
+
+    def test_role_timeout_can_be_overridden(self):
+        values = {
+            "SOLVER_API_KEY": "solver-key",
+            "SOLVER_BASE_URL": "https://solver.example/v1",
+            "SOLVER_MODEL": "solver-model",
+            "SOLVER_TIMEOUT_SECONDS": "360",
+        }
+        with patch.dict(os.environ, values, clear=True):
+            solver = LLMClientConfig.from_env("solver")
+        self.assertEqual(solver.timeout_seconds, 360)
 
     def test_missing_role_base_url_is_rejected(self):
         values = {