name: Dynamic world reasoning evaluation on: workflow_dispatch: jobs: short-eval: runs-on: minecraft-and-affine-server timeout-minutes: 45 env: RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} SOLVER_ENABLE_THINKING: "false" steps: - name: Check out repository uses: actions/checkout@v4 - name: Run unit tests run: python3 -m unittest discover -s tests -v - name: Run five large-world live cases run: python3 -m logicbench run tasks/short-smoke.json - name: Show dynamic worlds, rendered histories, results, and summary if: always() run: | test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true test -f artifacts/short-smoke/rendered.jsonl && cat artifacts/short-smoke/rendered.jsonl || true test -f artifacts/short-smoke/solver-outputs.jsonl && cat artifacts/short-smoke/solver-outputs.jsonl || true test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true - name: Preserve evaluation artifacts if: always() uses: actions/upload-artifact@v3 with: name: dynamic-world-evaluation path: artifacts/short-smoke/