diff --git a/.gitea/workflows/gpu-train.yaml b/.gitea/workflows/gpu-train.yaml index 335b379..c09fc5c 100644 --- a/.gitea/workflows/gpu-train.yaml +++ b/.gitea/workflows/gpu-train.yaml @@ -8,8 +8,10 @@ jobs: runs-on: gpu timeout-minutes: 1440 env: - HF_ENDPOINT: https://hf-mirror.com + HF_ENDPOINT: https://huggingface.co HF_HUB_DISABLE_XET: "1" + HF_HUB_ETAG_TIMEOUT: "60" + HF_HUB_DOWNLOAD_TIMEOUT: "600" TOKENIZERS_PARALLELISM: "false" steps: - name: Checkout from Gitea @@ -42,6 +44,14 @@ jobs: --config configs/siglip2_giant.yaml \ --reuse-existing + - name: Download SigLIP2 Giant weights + run: | + source /etc/network_turbo + date -Iseconds + /root/autodl-tmp/ces-venv/bin/python -c "from huggingface_hub import snapshot_download; print(snapshot_download('google/siglip2-giant-opt-patch16-384', max_workers=4))" + date -Iseconds + unset http_proxy https_proxy + - name: Train SigLIP2 Giant extractor run: | source /etc/network_turbo @@ -62,6 +72,23 @@ jobs: - name: Details if: always() run: | + echo "::group::Run details" + echo "repository=$GITHUB_REPOSITORY" + echo "sha=$GITHUB_SHA" + echo "run_id=$GITHUB_RUN_ID" + date -Iseconds + echo "::endgroup::" + echo "::group::Hardware and software" + nvidia-smi || true + /root/autodl-tmp/ces-venv/bin/python -c "import platform, torch, transformers; print({'python': platform.python_version(), 'torch': torch.__version__, 'torch_cuda': torch.version.cuda, 'transformers': transformers.__version__, 'cuda_available': torch.cuda.is_available(), 'gpu': torch.cuda.get_device_name() if torch.cuda.is_available() else None, 'capability': torch.cuda.get_device_capability() if torch.cuda.is_available() else None})" || true + echo "::endgroup::" + echo "::group::Configuration" + cat configs/siglip2_giant.yaml + echo "::endgroup::" + echo "::group::Dataset details" + find /root/autodl-tmp/ces-data/synthetic-v1 -maxdepth 2 -type f -name '*.jsonl' -exec wc -l {} + || true + test -f /root/autodl-tmp/ces-data/synthetic-v1/manifest.json && cat /root/autodl-tmp/ces-data/synthetic-v1/manifest.json || true + echo "::endgroup::" echo "::group::Epoch details" test -f /root/autodl-tmp/ces-artifacts/siglip2-giant/training_details.jsonl && cat /root/autodl-tmp/ces-artifacts/siglip2-giant/training_details.jsonl || true echo "::endgroup::"