name: Short behavioral-system evaluation on: workflow_dispatch: jobs: short-eval: runs-on: minecraft-and-affine-server timeout-minutes: 15 env: RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} steps: - name: Check out repository uses: actions/checkout@v4 - name: Run unit tests run: python3 -m unittest discover -s tests -v - name: Run five short live cases run: python3 -m logicbench run tasks/short-smoke.json - name: Show hidden systems, behavior skeletons, results, and summary if: always() run: | test -f artifacts/short-smoke/skeletons.jsonl && cat artifacts/short-smoke/skeletons.jsonl || true test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true - name: Preserve evaluation artifacts if: always() uses: actions/upload-artifact@v3 with: name: short-abstraction-evaluation path: artifacts/short-smoke/