diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml deleted file mode 100644 index 0b572f3..0000000 --- a/.gitea/workflows/short-eval.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Short abstraction evaluation - -on: - workflow_dispatch: - -jobs: - short-eval: - runs-on: logic-bench - timeout-minutes: 15 - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} - RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} - RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} - RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} - SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} - SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} - SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} - SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Run unit tests - run: python3 -m unittest discover -s tests -v - - - name: Run three short live cases - run: python3 -m logicbench run tasks/short-smoke.json - - - name: Show evaluation records and summary - if: always() - run: | - test -f artifacts/short-smoke/results.jsonl && cat artifacts/short-smoke/results.jsonl || true - test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true - - - name: Preserve evaluation artifacts - if: always() - uses: actions/upload-artifact@v3 - with: - name: short-abstraction-evaluation - path: artifacts/short-smoke/