33 lines
1.0 KiB
YAML
33 lines
1.0 KiB
YAML
name: Short abstraction evaluation
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
short-eval:
|
|
runs-on: logic-bench
|
|
timeout-minutes: 15
|
|
env:
|
|
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
|
|
RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }}
|
|
RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }}
|
|
RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }}
|
|
RENDERER_MODEL: ${{ vars.RENDERER_MODEL }}
|
|
SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }}
|
|
SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }}
|
|
SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }}
|
|
SOLVER_MODEL: ${{ vars.SOLVER_MODEL }}
|
|
steps:
|
|
- name: Check out repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Run unit tests
|
|
run: python3 -m unittest discover -s tests -v
|
|
|
|
- name: Run three short live cases
|
|
run: python3 -m logicbench run tasks/short-smoke.json
|
|
|
|
- name: Show evaluation summary
|
|
if: always()
|
|
run: test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
|