From 0f0594bc41907d93096643f0858e56250d356258 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 23 Sep 2026 00:36:45 +0800 Subject: [PATCH] Add independent thinking-mode controls --- .gitea/workflows/short-eval.yml | 1 + README.md | 3 ++- logicbench/client.py | 11 +++++++++++ logicbench/pipeline.py | 3 ++- tests/test_client.py | 2 ++ 5 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index 22827c2..60b2592 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -16,6 +16,7 @@ jobs: SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} + SOLVER_ENABLE_THINKING: "false" steps: - name: Check out repository uses: actions/checkout@v4 diff --git a/README.md b/README.md index 9924d6d..03605fb 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,7 @@ python3 -m logicbench run tasks/short-smoke.json | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` | | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs | | `RENDERER_TIMEOUT_SECONDS`, `SOLVER_TIMEOUT_SECONDS` | Optional per-request timeout; default `240` | +| `RENDERER_ENABLE_THINKING`, `SOLVER_ENABLE_THINKING` | Optional `true`/`false` provider extension; omitted unless set | Outputs under the task's `output_dir` are: @@ -70,7 +71,7 @@ Configure repository Actions with: 2. Variables `RENDERER_BASE_URL`, `RENDERER_MODEL`, and `RENDERER_API_STYLE`. 3. Variables `SOLVER_BASE_URL`, `SOLVER_MODEL`, and `SOLVER_API_STYLE`. -For Qwen's OpenAI-compatible API, use the compatible-mode `/v1` endpoint and `chat_completions`. Keys, URLs, and models remain role-specific even when both roles currently use the same provider. +For Qwen's OpenAI-compatible API, use the compatible-mode `/v1` endpoint and `chat_completions`. Keys, URLs, and models remain role-specific even when both roles currently use the same provider. The checked-in short workflow explicitly disables solver thinking to avoid long non-streaming requests; remove or change that workflow value when thinking-mode performance is itself under test. ## Tests diff --git a/logicbench/client.py b/logicbench/client.py index f8763b0..52eb54c 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -16,6 +16,7 @@ class LLMClientConfig: base_url: str = "https://api.openai.com/v1" api_style: str = "responses" timeout_seconds: int = 240 + enable_thinking: bool | None = None @classmethod def from_env(cls, role: str) -> "LLMClientConfig": @@ -25,6 +26,7 @@ class LLMClientConfig: base_url = os.getenv(f"{prefix}_BASE_URL", "").strip() api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses" timeout_text = os.getenv(f"{prefix}_TIMEOUT_SECONDS", "").strip() + thinking_text = os.getenv(f"{prefix}_ENABLE_THINKING", "").strip().lower() if not model: raise ValueError(f"{prefix}_MODEL is required") if not api_key: @@ -36,12 +38,16 @@ class LLMClientConfig: timeout_seconds = int(timeout_text) if timeout_text else 240 if timeout_seconds < 1: raise ValueError(f"{prefix}_TIMEOUT_SECONDS must be positive") + if thinking_text not in {"", "true", "false"}: + raise ValueError(f"{prefix}_ENABLE_THINKING must be true or false") + enable_thinking = None if not thinking_text else thinking_text == "true" return cls( model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style, timeout_seconds=timeout_seconds, + enable_thinking=enable_thinking, ) @@ -92,6 +98,9 @@ class LLMClient: "max_tokens": 2500, } + if self.config.enable_thinking is not None: + body["enable_thinking"] = self.config.enable_thinking + text = self._post(path, body) try: result = json.loads(text) @@ -122,6 +131,8 @@ class LLMClient: ], "max_tokens": 512, } + if self.config.enable_thinking is not None: + body["enable_thinking"] = self.config.enable_thinking return self._post(path, body) def _post(self, path: str, body: dict[str, Any]) -> str: diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index cab359e..4fc65ed 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -56,7 +56,8 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str: "trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that " "all trials use one unchanged system and that the question asks which final outcome is more likely, " "not for an exact probability. Present repeated histories in a compact table rather than verbose " - "paragraphs. End by requiring exactly one supplied answer label." + "paragraphs. State the supplied answer labels, but do not impose a response format or forbid an " + "explanation; the evaluator supplies its own output protocol to the solver." ), user=f"Render this intermediate representation:\n{prompt}", schema_name="rendered_problem", diff --git a/tests/test_client.py b/tests/test_client.py index 3222cb6..0352f0c 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -30,10 +30,12 @@ class ClientConfigTests(unittest.TestCase): "SOLVER_BASE_URL": "https://solver.example/v1", "SOLVER_MODEL": "solver-model", "SOLVER_TIMEOUT_SECONDS": "360", + "SOLVER_ENABLE_THINKING": "false", } with patch.dict(os.environ, values, clear=True): solver = LLMClientConfig.from_env("solver") self.assertEqual(solver.timeout_seconds, 360) + self.assertFalse(solver.enable_thinking) def test_missing_role_base_url_is_rejected(self): values = {