diff --git a/.env.example b/.env.example index a71e4d1..458543b 100644 --- a/.env.example +++ b/.env.example @@ -1,8 +1,4 @@ -# Provider-neutral shared fallback. Store real keys as Gitea Actions secrets. -LLM_API_KEY= -LLM_BASE_URL= - -# Renderer and solver can use different OpenAI-compatible providers/models. +# Renderer and solver are fully independent OpenAI-compatible clients. RENDERER_API_KEY= RENDERER_BASE_URL= RENDERER_API_STYLE=responses diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index e74b67a..ddc1874 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -8,8 +8,6 @@ jobs: runs-on: minecraft-and-affine-server timeout-minutes: 15 env: - LLM_API_KEY: ${{ secrets.LLM_API_KEY }} - LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} diff --git a/README.md b/README.md index 7bb3588..91a7f8c 100644 --- a/README.md +++ b/README.md @@ -31,9 +31,11 @@ python3 -m logicbench generate --count 3 --length short --seed 20260922 Run the checked-in live task: ```bash -export LLM_API_KEY='...' -export LLM_BASE_URL='your-provider-compatible-base-url' +export RENDERER_API_KEY='...' +export RENDERER_BASE_URL='your-renderer-compatible-base-url' export RENDERER_MODEL='your-renderer-model' +export SOLVER_API_KEY='...' +export SOLVER_BASE_URL='your-solver-compatible-base-url' export SOLVER_MODEL='your-solver-model' python3 -m logicbench run tasks/short-smoke.json ``` @@ -42,10 +44,8 @@ Role-specific values override the shared fallback: | Variable | Meaning | | --- | --- | -| `LLM_API_KEY` | Optional shared key when both roles use the same account/provider | -| `RENDERER_API_KEY`, `SOLVER_API_KEY` | Optional role-specific keys; each takes precedence over `LLM_API_KEY` | -| `LLM_BASE_URL` | Optional shared OpenAI-compatible endpoint | -| `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Optional role-specific endpoints; each takes precedence over `LLM_BASE_URL` | +| `RENDERER_API_KEY`, `SOLVER_API_KEY` | Required independent credentials; values may currently be identical | +| `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Required independent OpenAI-compatible endpoints; values may currently be identical | | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` | | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs | @@ -63,14 +63,13 @@ The run log prints the rendered questions and scores. The same records, original Create these repository Actions settings: -1. Secret `LLM_API_KEY` when both roles share credentials, or role-specific secrets `RENDERER_API_KEY` and `SOLVER_API_KEY` when they do not. -2. Variables `RENDERER_MODEL` and `SOLVER_MODEL`. -3. Variable `LLM_BASE_URL` for a shared provider endpoint, or role-specific variables `RENDERER_BASE_URL` and `SOLVER_BASE_URL` for different providers. -4. Optional variables `RENDERER_API_STYLE` and `SOLVER_API_STYLE`, each set to `responses` or `chat_completions` according to that provider/model. +1. Secrets `RENDERER_API_KEY` and `SOLVER_API_KEY`. +2. Variables `RENDERER_BASE_URL`, `RENDERER_MODEL`, and `RENDERER_API_STYLE`. +3. Variables `SOLVER_BASE_URL`, `SOLVER_MODEL`, and `SOLVER_API_STYLE`. The workflow targets the existing `minecraft-and-affine-server` runner label. Verify that a runner advertising this label is online, then manually dispatch **Short abstraction evaluation** from the repository's Actions page. -For a current all-Qwen run, create only the shared `LLM_API_KEY` secret with the Model Studio key. Set `LLM_BASE_URL` to the API Host shown for that key and region, set the two model variables independently, and use `chat_completions` for both API-style variables. For example, a fast renderer and stronger solver can use `qwen3.8-flash` and `qwen3.8-max`; both currently support JSON Schema structured output. Do not hard-code the DashScope key or endpoint in the workflow. +For a current all-Qwen run, put the Model Studio key into both role-specific secrets and its API Host into both role-specific base-URL variables. The values may be identical today, but the two client boundaries remain explicit. Use `chat_completions` for both API-style variables. For example, a fast renderer and stronger solver can use `qwen3.8-flash` and `qwen3.8-max`; both currently support JSON Schema structured output. Do not hard-code keys or endpoints in the workflow. ## Tests diff --git a/logicbench/client.py b/logicbench/client.py index 749bacb..6c94f29 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -20,22 +20,15 @@ class LLMClientConfig: def from_env(cls, role: str) -> "LLMClientConfig": prefix = role.upper() model = os.getenv(f"{prefix}_MODEL", "").strip() - api_key = ( - os.getenv(f"{prefix}_API_KEY", "").strip() - or os.getenv("LLM_API_KEY", "").strip() - or os.getenv("OPENAI_API_KEY", "").strip() - ) - base_url = ( - os.getenv(f"{prefix}_BASE_URL", "").strip() - or os.getenv("LLM_BASE_URL", "").strip() - or os.getenv("OPENAI_BASE_URL", "").strip() - or "https://api.openai.com/v1" - ) + api_key = os.getenv(f"{prefix}_API_KEY", "").strip() + base_url = os.getenv(f"{prefix}_BASE_URL", "").strip() api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses" if not model: raise ValueError(f"{prefix}_MODEL is required") if not api_key: - raise ValueError(f"{prefix}_API_KEY or LLM_API_KEY is required") + raise ValueError(f"{prefix}_API_KEY is required") + if not base_url: + raise ValueError(f"{prefix}_BASE_URL is required") if api_style not in {"responses", "chat_completions"}: raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions") return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style) diff --git a/tests/test_client.py b/tests/test_client.py index 5074060..4fb9ff5 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -6,33 +6,31 @@ from logicbench.client import LLMClientConfig class ClientConfigTests(unittest.TestCase): - def test_shared_provider_defaults_apply_to_both_roles(self): + def test_roles_have_independent_explicit_configuration(self): values = { - "LLM_API_KEY": "shared-key", - "LLM_BASE_URL": "https://shared.example/v1", + "RENDERER_API_KEY": "renderer-key", + "RENDERER_BASE_URL": "https://renderer.example/v1", "RENDERER_MODEL": "renderer-model", + "SOLVER_API_KEY": "solver-key", + "SOLVER_BASE_URL": "https://solver.example/v1", "SOLVER_MODEL": "solver-model", } with patch.dict(os.environ, values, clear=True): renderer = LLMClientConfig.from_env("renderer") solver = LLMClientConfig.from_env("solver") - self.assertEqual(renderer.api_key, "shared-key") - self.assertEqual(solver.api_key, "shared-key") - self.assertEqual(renderer.base_url, "https://shared.example/v1") - self.assertEqual(solver.base_url, "https://shared.example/v1") + self.assertEqual(renderer.api_key, "renderer-key") + self.assertEqual(solver.api_key, "solver-key") + self.assertEqual(renderer.base_url, "https://renderer.example/v1") + self.assertEqual(solver.base_url, "https://solver.example/v1") - def test_role_specific_values_override_shared_provider(self): + def test_missing_role_base_url_is_rejected(self): values = { - "LLM_API_KEY": "shared-key", - "LLM_BASE_URL": "https://shared.example/v1", "RENDERER_API_KEY": "renderer-key", - "RENDERER_BASE_URL": "https://renderer.example/v1/", "RENDERER_MODEL": "renderer-model", } with patch.dict(os.environ, values, clear=True): - renderer = LLMClientConfig.from_env("renderer") - self.assertEqual(renderer.api_key, "renderer-key") - self.assertEqual(renderer.base_url, "https://renderer.example/v1") + with self.assertRaisesRegex(ValueError, "RENDERER_BASE_URL is required"): + LLMClientConfig.from_env("renderer") if __name__ == "__main__":