74 lines
2.4 KiB
YAML
74 lines
2.4 KiB
YAML
name: Manage AutoDL GPU runner
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
inputs:
|
|
operation:
|
|
description: discover, provision, status, or shutdown
|
|
required: true
|
|
default: discover
|
|
image_uuid:
|
|
description: Required only for provision
|
|
required: false
|
|
default: ""
|
|
framework:
|
|
description: Public image framework, for example pytorch
|
|
required: false
|
|
default: ""
|
|
framework_version:
|
|
description: Public image framework version
|
|
required: false
|
|
default: ""
|
|
cuda_version:
|
|
description: Public image CUDA version
|
|
required: false
|
|
default: ""
|
|
confirm_spend:
|
|
description: Type CREATE to provision a billed instance
|
|
required: false
|
|
default: ""
|
|
|
|
jobs:
|
|
manage:
|
|
runs-on: ces-control
|
|
timeout-minutes: 180
|
|
env:
|
|
AUTODL_API_TOKEN: ${{ secrets.AUTODL_API_TOKEN }}
|
|
TRAINING_RUNNER_REGISTRATION_TOKEN: ${{ secrets.TRAINING_RUNNER_REGISTRATION_TOKEN }}
|
|
OPERATION: ${{ inputs.operation }}
|
|
IMAGE_UUID: ${{ inputs.image_uuid }}
|
|
FRAMEWORK: ${{ inputs.framework }}
|
|
FRAMEWORK_VERSION: ${{ inputs.framework_version }}
|
|
CUDA_VERSION: ${{ inputs.cuda_version }}
|
|
CONFIRM_SPEND: ${{ inputs.confirm_spend }}
|
|
steps:
|
|
- name: Select deployed controller
|
|
run: |
|
|
test -x /usr/local/bin/ces-controller
|
|
test "$(cat /opt/ces-controller/current/VERSION)" = "$GITHUB_SHA"
|
|
echo "controller_version=$GITHUB_SHA"
|
|
|
|
- name: Validate requested operation
|
|
run: |
|
|
case "$OPERATION" in
|
|
discover|provision|status|shutdown) ;;
|
|
*) echo "Unsupported operation" >&2; exit 2 ;;
|
|
esac
|
|
|
|
- name: Manage AutoDL Pro instance
|
|
run: |
|
|
/usr/local/bin/ces-controller "$OPERATION" \
|
|
--image-uuid "$IMAGE_UUID" \
|
|
--framework "$FRAMEWORK" \
|
|
--framework-version "$FRAMEWORK_VERSION" \
|
|
--cuda-version "$CUDA_VERSION" \
|
|
--confirm-spend "$CONFIRM_SPEND"
|
|
|
|
- name: Controller host details
|
|
if: always()
|
|
run: |
|
|
echo "host=$(hostname)"
|
|
date -Iseconds
|
|
test -f /root/.local/state/ces-autodl-controller/state.json && \
|
|
python3 -c 'import json; p="/root/.local/state/ces-autodl-controller/state.json"; d=json.load(open(p)); d.pop("root_password", None); print(json.dumps(d, indent=2))' || true
|