Files
logic-skeleton-bench/.gitea/workflows/short-eval.yml
T

33 lines
1.0 KiB
YAML

name: Short abstraction evaluation
on:
workflow_dispatch:
jobs:
short-eval:
runs-on: logic-bench
timeout-minutes: 15
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }}
RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }}
RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }}
RENDERER_MODEL: ${{ vars.RENDERER_MODEL }}
SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }}
SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }}
SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }}
SOLVER_MODEL: ${{ vars.SOLVER_MODEL }}
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Run unit tests
run: python3 -m unittest discover -s tests -v
- name: Run three short live cases
run: python3 -m logicbench run tasks/short-smoke.json
- name: Show evaluation summary
if: always()
run: test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true