From 40cbae289e0dfec285e9158579f078c0eff035c7 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Tue, 22 Sep 2026 22:42:34 +0800 Subject: [PATCH] Restore portable evaluation workflow for configured runner --- .env.example | 6 ++--- .gitea/workflows/short-eval.yml | 42 +++++++++++++++++++++++++++++++++ README.md | 30 +++++++++-------------- logicbench/client.py | 9 +++++-- tests/test_client.py | 39 ++++++++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 24 deletions(-) create mode 100644 .gitea/workflows/short-eval.yml create mode 100644 tests/test_client.py diff --git a/.env.example b/.env.example index a7a4be2..a71e4d1 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ -# Shared fallback. Store the real value as a Gitea Actions secret, never here. -OPENAI_API_KEY= -OPENAI_BASE_URL=https://api.openai.com/v1 +# Provider-neutral shared fallback. Store real keys as Gitea Actions secrets. +LLM_API_KEY= +LLM_BASE_URL= # Renderer and solver can use different OpenAI-compatible providers/models. RENDERER_API_KEY= diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml new file mode 100644 index 0000000..e74b67a --- /dev/null +++ b/.gitea/workflows/short-eval.yml @@ -0,0 +1,42 @@ +name: Short abstraction evaluation + +on: + workflow_dispatch: + +jobs: + short-eval: + runs-on: minecraft-and-affine-server + timeout-minutes: 15 + env: + LLM_API_KEY: ${{ secrets.LLM_API_KEY }} + LLM_BASE_URL: ${{ vars.LLM_BASE_URL }} + RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} + RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} + RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} + RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} + SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} + SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} + SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} + SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Run unit tests + run: python3 -m unittest discover -s tests -v + + - name: Run three short live cases + run: python3 -m logicbench run tasks/short-smoke.json + + - name: Show evaluation records and summary + if: always() + run: | + test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true + test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true + + - name: Preserve evaluation artifacts + if: always() + uses: actions/upload-artifact@v3 + with: + name: short-abstraction-evaluation + path: artifacts/short-smoke/ diff --git a/README.md b/README.md index a5c212e..7bb3588 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,8 @@ python3 -m logicbench generate --count 3 --length short --seed 20260922 Run the checked-in live task: ```bash -export OPENAI_API_KEY='...' +export LLM_API_KEY='...' +export LLM_BASE_URL='your-provider-compatible-base-url' export RENDERER_MODEL='your-renderer-model' export SOLVER_MODEL='your-solver-model' python3 -m logicbench run tasks/short-smoke.json @@ -41,8 +42,10 @@ Role-specific values override the shared fallback: | Variable | Meaning | | --- | --- | -| `RENDERER_API_KEY`, `SOLVER_API_KEY` | Optional separate keys; each falls back to `OPENAI_API_KEY` | -| `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Optional endpoints; each falls back to `OPENAI_BASE_URL`, then `https://api.openai.com/v1` | +| `LLM_API_KEY` | Optional shared key when both roles use the same account/provider | +| `RENDERER_API_KEY`, `SOLVER_API_KEY` | Optional role-specific keys; each takes precedence over `LLM_API_KEY` | +| `LLM_BASE_URL` | Optional shared OpenAI-compatible endpoint | +| `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Optional role-specific endpoints; each takes precedence over `LLM_BASE_URL` | | `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` | | `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs | @@ -60,25 +63,14 @@ The run log prints the rendered questions and scores. The same records, original Create these repository Actions settings: -1. Secret `OPENAI_API_KEY`, or separate secrets `RENDERER_API_KEY` and `SOLVER_API_KEY`. +1. Secret `LLM_API_KEY` when both roles share credentials, or role-specific secrets `RENDERER_API_KEY` and `SOLVER_API_KEY` when they do not. 2. Variables `RENDERER_MODEL` and `SOLVER_MODEL`. -3. Optional variables `RENDERER_BASE_URL`, `SOLVER_BASE_URL`, `RENDERER_API_STYLE`, and `SOLVER_API_STYLE`. Leave base URLs empty for OpenAI; use `responses` as the API style unless a compatible provider only supports chat completions. +3. Variable `LLM_BASE_URL` for a shared provider endpoint, or role-specific variables `RENDERER_BASE_URL` and `SOLVER_BASE_URL` for different providers. +4. Optional variables `RENDERER_API_STYLE` and `SOLVER_API_STYLE`, each set to `responses` or `chat_completions` according to that provider/model. -The workflow targets `runs-on: logic-bench`. Add this label to the existing `act_runner` using one of these methods: +The workflow targets the existing `minecraft-and-affine-server` runner label. Verify that a runner advertising this label is online, then manually dispatch **Short abstraction evaluation** from the repository's Actions page. -- In the Gitea UI, open the runner settings, edit its labels, and add `logic-bench:docker://docker.gitea.com/runner-images:ubuntu-latest`. -- Or add the same string to `runner.labels` in the runner's `config.yaml`, preserving existing labels, then restart `act_runner` with that config. -- Or start the daemon with `--labels` / `GITEA_RUNNER_LABELS`; this has higher precedence than the config file and must include every label the runner should continue advertising. - -Example config fragment: - -```yaml -runner: - labels: - - "logic-bench:docker://docker.gitea.com/runner-images:ubuntu-latest" -``` - -After restarting the runner, verify that `logic-bench` appears in its Gitea settings, then manually dispatch **Short abstraction evaluation** from the repository's Actions page. +For a current all-Qwen run, create only the shared `LLM_API_KEY` secret with the Model Studio key. Set `LLM_BASE_URL` to the API Host shown for that key and region, set the two model variables independently, and use `chat_completions` for both API-style variables. For example, a fast renderer and stronger solver can use `qwen3.8-flash` and `qwen3.8-max`; both currently support JSON Schema structured output. Do not hard-code the DashScope key or endpoint in the workflow. ## Tests diff --git a/logicbench/client.py b/logicbench/client.py index 1e6fd67..749bacb 100644 --- a/logicbench/client.py +++ b/logicbench/client.py @@ -20,9 +20,14 @@ class LLMClientConfig: def from_env(cls, role: str) -> "LLMClientConfig": prefix = role.upper() model = os.getenv(f"{prefix}_MODEL", "").strip() - api_key = os.getenv(f"{prefix}_API_KEY", "").strip() or os.getenv("OPENAI_API_KEY", "").strip() + api_key = ( + os.getenv(f"{prefix}_API_KEY", "").strip() + or os.getenv("LLM_API_KEY", "").strip() + or os.getenv("OPENAI_API_KEY", "").strip() + ) base_url = ( os.getenv(f"{prefix}_BASE_URL", "").strip() + or os.getenv("LLM_BASE_URL", "").strip() or os.getenv("OPENAI_BASE_URL", "").strip() or "https://api.openai.com/v1" ) @@ -30,7 +35,7 @@ class LLMClientConfig: if not model: raise ValueError(f"{prefix}_MODEL is required") if not api_key: - raise ValueError(f"{prefix}_API_KEY or OPENAI_API_KEY is required") + raise ValueError(f"{prefix}_API_KEY or LLM_API_KEY is required") if api_style not in {"responses", "chat_completions"}: raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions") return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style) diff --git a/tests/test_client.py b/tests/test_client.py new file mode 100644 index 0000000..5074060 --- /dev/null +++ b/tests/test_client.py @@ -0,0 +1,39 @@ +import os +import unittest +from unittest.mock import patch + +from logicbench.client import LLMClientConfig + + +class ClientConfigTests(unittest.TestCase): + def test_shared_provider_defaults_apply_to_both_roles(self): + values = { + "LLM_API_KEY": "shared-key", + "LLM_BASE_URL": "https://shared.example/v1", + "RENDERER_MODEL": "renderer-model", + "SOLVER_MODEL": "solver-model", + } + with patch.dict(os.environ, values, clear=True): + renderer = LLMClientConfig.from_env("renderer") + solver = LLMClientConfig.from_env("solver") + self.assertEqual(renderer.api_key, "shared-key") + self.assertEqual(solver.api_key, "shared-key") + self.assertEqual(renderer.base_url, "https://shared.example/v1") + self.assertEqual(solver.base_url, "https://shared.example/v1") + + def test_role_specific_values_override_shared_provider(self): + values = { + "LLM_API_KEY": "shared-key", + "LLM_BASE_URL": "https://shared.example/v1", + "RENDERER_API_KEY": "renderer-key", + "RENDERER_BASE_URL": "https://renderer.example/v1/", + "RENDERER_MODEL": "renderer-model", + } + with patch.dict(os.environ, values, clear=True): + renderer = LLMClientConfig.from_env("renderer") + self.assertEqual(renderer.api_key, "renderer-key") + self.assertEqual(renderer.base_url, "https://renderer.example/v1") + + +if __name__ == "__main__": + unittest.main()