45 lines
1.7 KiB
YAML
45 lines
1.7 KiB
YAML
name: Dynamic world reasoning evaluation
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
short-eval:
|
|
runs-on: minecraft-and-affine-server
|
|
timeout-minutes: 45
|
|
env:
|
|
RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }}
|
|
RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }}
|
|
RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }}
|
|
RENDERER_MODEL: ${{ vars.RENDERER_MODEL }}
|
|
SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }}
|
|
SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }}
|
|
SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }}
|
|
SOLVER_MODEL: ${{ vars.SOLVER_MODEL }}
|
|
SOLVER_ENABLE_THINKING: "false"
|
|
steps:
|
|
- name: Check out repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Run unit tests
|
|
run: python3 -m unittest discover -s tests -v
|
|
|
|
- name: Run five large-world live cases
|
|
run: python3 -m logicbench run tasks/short-smoke.json
|
|
|
|
- name: Show dynamic worlds, rendered histories, results, and summary
|
|
if: always()
|
|
run: |
|
|
test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true
|
|
test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true
|
|
test -f artifacts/short-smoke/solver-outputs.jsonl && cat artifacts/short-smoke/solver-outputs.jsonl || true
|
|
test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true
|
|
test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true
|
|
|
|
- name: Preserve evaluation artifacts
|
|
if: always()
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: dynamic-world-evaluation
|
|
path: artifacts/short-smoke/
|