Add independent thinking-mode controls

.gitea/workflows/short-eval.yml

@@ -16,6 +16,7 @@ jobs:
       SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }}
       SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }}
       SOLVER_MODEL: ${{ vars.SOLVER_MODEL }}
+      SOLVER_ENABLE_THINKING: "false"
     steps:
       - name: Check out repository
         uses: actions/checkout@v4

README.md

@@ -52,6 +52,7 @@ python3 -m logicbench run tasks/short-smoke.json
 | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` |
 | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs |
 | `RENDERER_TIMEOUT_SECONDS`, `SOLVER_TIMEOUT_SECONDS` | Optional per-request timeout; default `240` |
+| `RENDERER_ENABLE_THINKING`, `SOLVER_ENABLE_THINKING` | Optional `true`/`false` provider extension; omitted unless set |
 
 Outputs under the task's `output_dir` are:
 
@@ -70,7 +71,7 @@ Configure repository Actions with:
 2. Variables `RENDERER_BASE_URL`, `RENDERER_MODEL`, and `RENDERER_API_STYLE`.
 3. Variables `SOLVER_BASE_URL`, `SOLVER_MODEL`, and `SOLVER_API_STYLE`.
 
-For Qwen's OpenAI-compatible API, use the compatible-mode `/v1` endpoint and `chat_completions`. Keys, URLs, and models remain role-specific even when both roles currently use the same provider.
+For Qwen's OpenAI-compatible API, use the compatible-mode `/v1` endpoint and `chat_completions`. Keys, URLs, and models remain role-specific even when both roles currently use the same provider. The checked-in short workflow explicitly disables solver thinking to avoid long non-streaming requests; remove or change that workflow value when thinking-mode performance is itself under test.
 
 ## Tests
 

logicbench/client.py

@@ -16,6 +16,7 @@ class LLMClientConfig:
     base_url: str = "https://api.openai.com/v1"
     api_style: str = "responses"
     timeout_seconds: int = 240
+    enable_thinking: bool | None = None
 
     @classmethod
     def from_env(cls, role: str) -> "LLMClientConfig":
@@ -25,6 +26,7 @@ class LLMClientConfig:
         base_url = os.getenv(f"{prefix}_BASE_URL", "").strip()
         api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses"
         timeout_text = os.getenv(f"{prefix}_TIMEOUT_SECONDS", "").strip()
+        thinking_text = os.getenv(f"{prefix}_ENABLE_THINKING", "").strip().lower()
         if not model:
             raise ValueError(f"{prefix}_MODEL is required")
         if not api_key:
@@ -36,12 +38,16 @@ class LLMClientConfig:
         timeout_seconds = int(timeout_text) if timeout_text else 240
         if timeout_seconds < 1:
             raise ValueError(f"{prefix}_TIMEOUT_SECONDS must be positive")
+        if thinking_text not in {"", "true", "false"}:
+            raise ValueError(f"{prefix}_ENABLE_THINKING must be true or false")
+        enable_thinking = None if not thinking_text else thinking_text == "true"
         return cls(
             model=model,
             api_key=api_key,
             base_url=base_url.rstrip("/"),
             api_style=api_style,
             timeout_seconds=timeout_seconds,
+            enable_thinking=enable_thinking,
         )
 
 
@@ -92,6 +98,9 @@ class LLMClient:
                 "max_tokens": 2500,
             }
 
+        if self.config.enable_thinking is not None:
+            body["enable_thinking"] = self.config.enable_thinking
+
         text = self._post(path, body)
         try:
             result = json.loads(text)
@@ -122,6 +131,8 @@ class LLMClient:
                 ],
                 "max_tokens": 512,
             }
+        if self.config.enable_thinking is not None:
+            body["enable_thinking"] = self.config.enable_thinking
         return self._post(path, body)
 
     def _post(self, path: str, body: dict[str, Any]) -> str:

logicbench/pipeline.py

@@ -56,7 +56,8 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
             "trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
             "all trials use one unchanged system and that the question asks which final outcome is more likely, "
             "not for an exact probability. Present repeated histories in a compact table rather than verbose "
-            "paragraphs. End by requiring exactly one supplied answer label."
+            "paragraphs. State the supplied answer labels, but do not impose a response format or forbid an "
+            "explanation; the evaluator supplies its own output protocol to the solver."
         ),
         user=f"Render this intermediate representation:\n{prompt}",
         schema_name="rendered_problem",

tests/test_client.py

@@ -30,10 +30,12 @@ class ClientConfigTests(unittest.TestCase):
             "SOLVER_BASE_URL": "https://solver.example/v1",
             "SOLVER_MODEL": "solver-model",
             "SOLVER_TIMEOUT_SECONDS": "360",
+            "SOLVER_ENABLE_THINKING": "false",
         }
         with patch.dict(os.environ, values, clear=True):
             solver = LLMClientConfig.from_env("solver")
         self.assertEqual(solver.timeout_seconds, 360)
+        self.assertFalse(solver.enable_thinking)
 
     def test_missing_role_base_url_is_rejected(self):
         values = {