Build initial logic skeleton benchmark

This commit is contained in:
2026-09-22 21:03:19 +08:00
commit bd5bac07de
15 changed files with 726 additions and 0 deletions
+32
View File
@@ -0,0 +1,32 @@
name: Short abstraction evaluation
on:
workflow_dispatch:
jobs:
short-eval:
runs-on: logic-bench
timeout-minutes: 15
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }}
RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }}
RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }}
RENDERER_MODEL: ${{ vars.RENDERER_MODEL }}
SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }}
SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }}
SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }}
SOLVER_MODEL: ${{ vars.SOLVER_MODEL }}
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Run unit tests
run: python3 -m unittest discover -s tests -v
- name: Run three short live cases
run: python3 -m logicbench run tasks/short-smoke.json
- name: Show evaluation summary
if: always()
run: test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true