lean-bench/.gitea/workflows/ingest.yml

name: ingest

# Submit a book and run the whole pipeline, inside the long-lived `lean-bench`
# container the docker workflow leaves running on this runner. The book is copied
# into the `lean-bench-books` volume and the items land in `lean-bench-data`, so
# everything survives the job and is there for the next one -- `--pages` of the
# same book can be redone later without re-ingesting the rest.
#
# The source can be a path inside the repository or a URL the runner can reach.
# Set the DASHSCOPE_API_KEY secret (or whichever key your provider uses) first.

on:
  workflow_dispatch:
    inputs:
      source:
        description: "Path in the repo, or a URL, of the book to ingest"
        required: true
      pages:
        description: "Page range, e.g. 30-60 (blank = the whole book)"
        required: false
        default: ""
      title:
        description: "Human title for this source"
        required: false
        default: ""
      verify:
        description: "Run Lean verification (slow, but the point)"
        required: false
        default: "true"
      image:
        description: "Image to use if the lean-bench container is not running"
        required: false
        default: "lean-bench:latest"

env:
  SERVICE_CONTAINER: lean-bench
  DATA_VOLUME: lean-bench-data
  BOOKS_VOLUME: lean-bench-books

jobs:
  ingest:
    runs-on: ubuntu-latest
    timeout-minutes: 720
    steps:
      - uses: actions/checkout@v4

      - name: Fetch the source
        id: source
        run: |
          set -eu
          mkdir -p books out
          src='${{ inputs.source }}'
          case "$src" in
            http://*|https://*)
              name="$(basename "${src%%\?*}")"
              curl -fsSL --retry 3 -o "books/$name" "$src"
              ;;
            *)
              [ -f "$src" ] || { echo "::error::$src is not in the repository"; exit 1; }
              name="$(basename "$src")"
              cp "$src" "books/$name"
              ;;
          esac
          ls -l "books/$name"
          echo "name=$name" >> "$GITHUB_OUTPUT"

      - name: Make sure the lean-bench container is up
        run: |
          set -eu
          docker volume create "${DATA_VOLUME}" >/dev/null
          docker volume create "${BOOKS_VOLUME}" >/dev/null
          if ! docker ps --format '{{.Names}}' | grep -qx "${SERVICE_CONTAINER}"; then
            image='${{ inputs.image }}'
            if ! docker image inspect "$image" >/dev/null 2>&1; then
              echo "::error::No $image on this runner. Run the docker workflow first."
              exit 1
            fi
            docker rm -f "${SERVICE_CONTAINER}" >/dev/null 2>&1 || true
            docker run -d --name "${SERVICE_CONTAINER}" --restart unless-stopped \
              -v "${DATA_VOLUME}:/data" -v "${BOOKS_VOLUME}:/books" \
              --entrypoint sleep "$image" infinity
          fi
          docker exec "${SERVICE_CONTAINER}" lean-bench doctor --skip-llm

      - name: Ingest
        env:
          DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }}
          LEANBENCH_LLM_MODEL: ${{ vars.LEANBENCH_LLM_MODEL }}
          LEANBENCH_LLM_BASE_URL: ${{ vars.LEANBENCH_LLM_BASE_URL }}
        run: |
          set -eu
          [ -n "${DASHSCOPE_API_KEY:-}" ] || { echo "::error::Set the DASHSCOPE_API_KEY secret"; exit 1; }
          name='${{ steps.source.outputs.name }}'
          docker cp "books/$name" "${SERVICE_CONTAINER}:/books/$name"

          set -- ingest "/books/$name" \
            --export "/data/${name%.*}.items.jsonl" \
            --detections "/data/${name%.*}.detections.jsonl"
          # `cond && set -- ...` would abort the step under `set -e` whenever the
          # condition is false, so these are spelled out.
          if [ -n '${{ inputs.pages }}' ]; then set -- "$@" --pages '${{ inputs.pages }}'; fi
          if [ -n '${{ inputs.title }}' ]; then set -- "$@" --title '${{ inputs.title }}'; fi
          if [ '${{ inputs.verify }}' != 'true' ]; then set -- "$@" --no-verify; fi

          docker exec \
            -e DASHSCOPE_API_KEY \
            -e LEANBENCH_LLM_MODEL \
            -e LEANBENCH_LLM_BASE_URL \
            "${SERVICE_CONTAINER}" lean-bench "$@"
          docker exec "${SERVICE_CONTAINER}" lean-bench stats

      - name: Collect a copy of the results
        if: always()
        run: |
          set -eu
          # The originals stay in the lean-bench-data volume; these are just copies
          # for the artifact.
          docker exec "${SERVICE_CONTAINER}" sh -c 'ls -1 /data' || true
          for f in $(docker exec "${SERVICE_CONTAINER}" sh -c 'ls -1 /data' 2>/dev/null || true); do
            docker cp "${SERVICE_CONTAINER}:/data/$f" "out/$f" || true
          done
          ls -l out || true

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: leanbench-${{ steps.source.outputs.name }}
          path: out/
          retention-days: 30