diff --git a/.gitea/workflows/gpu-train.yaml b/.gitea/workflows/gpu-train.yaml index a66546b..78dd1c9 100644 --- a/.gitea/workflows/gpu-train.yaml +++ b/.gitea/workflows/gpu-train.yaml @@ -8,10 +8,6 @@ jobs: runs-on: gpu timeout-minutes: 1440 env: - HF_ENDPOINT: https://huggingface.co - HF_XET_HIGH_PERFORMANCE: "1" - HF_HUB_ETAG_TIMEOUT: "60" - HF_HUB_DOWNLOAD_TIMEOUT: "600" TOKENIZERS_PARALLELISM: "false" steps: - name: Checkout from Gitea @@ -44,13 +40,19 @@ jobs: --config configs/siglip2_giant.yaml \ --reuse-existing - - name: Download SigLIP2 Giant weights + - name: Verify preloaded SigLIP2 Giant weights run: | - source /etc/network_turbo - date -Iseconds - /root/autodl-tmp/ces-venv/bin/python -c "from huggingface_hub import snapshot_download; print(snapshot_download('google/siglip2-giant-opt-patch16-384', max_workers=4))" - date -Iseconds - unset http_proxy https_proxy + MODEL_DIR=/root/autodl-tmp/ces-models/siglip2-giant + test -s "$MODEL_DIR/config.json" + test -s "$MODEL_DIR/model.safetensors.index.json" + test -s "$MODEL_DIR/model-00001-of-00002.safetensors" + test -s "$MODEL_DIR/model-00002-of-00002.safetensors" + test -s "$MODEL_DIR/preprocessor_config.json" + test -s "$MODEL_DIR/tokenizer.json" + test -s "$MODEL_DIR/tokenizer.model" + test -s "$MODEL_DIR/tokenizer_config.json" + du -sh "$MODEL_DIR" + ls -lh "$MODEL_DIR" - name: Train SigLIP2 Giant extractor run: | diff --git a/README.md b/README.md index c34a4c0..e400c85 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ uv run python train_extractor.py --config configs/mvp.yaml --backbone clip Gitea 的 `Formal GPU training` workflow 会: -- 使用 AutoDL `network_turbo` 与 `hf-mirror.com` 下载模型; +- 校验预先放在 `/root/autodl-tmp/ces-models/siglip2-giant` 的固定版本模型; - 在 `/root/autodl-tmp/ces-data/synthetic-v1` 复用完整数据集; - 每个 epoch 保存可恢复的 `last.pt`,并按验证集保存 `best.pt`; - 将运行配置、硬件、分项 loss、每轮耗时、所有验证指标和最终 benchmark 放在 @@ -78,6 +78,33 @@ Gitea 的 `Formal GPU training` workflow 会: - 将结果持久化到 `/root/autodl-tmp/ces-artifacts/siglip2-giant`; - 最后用六项研究门槛决定 workflow 是否通过。 +为避免付费 GPU 等待 7.53 GB 权重下载,先在网络更稳定的无 GPU 服务器上执行: + +```bash +python3 -m venv ces-model-download +source ces-model-download/bin/activate +python -m pip install -U "huggingface_hub[hf_xet]" +HF_ENDPOINT=https://hf-mirror.com hf download \ + google/siglip2-giant-opt-patch16-384 \ + --revision a713301b217d38485fb2204c808367d10bc3cc40 \ + --local-dir /data/ces-models/siglip2-giant +du -sh /data/ces-models/siglip2-giant +``` + +然后从该服务器断点续传到 AutoDL(替换 SSH 主机和端口): + +```bash +ssh -p root@ \ + 'mkdir -p /root/autodl-tmp/ces-models/siglip2-giant' +rsync -ahP --partial --append-verify \ + -e 'ssh -p ' \ + /data/ces-models/siglip2-giant/ \ + root@:/root/autodl-tmp/ces-models/siglip2-giant/ +``` + +如果两台服务器不能直连,使用阿里云 OSS 作为中转;目标目录保持不变。workflow +只做完整文件校验,不再从 Hugging Face 下载,因此模型传好后再手动触发正式训练。 + 手动等价命令为: ```bash diff --git a/configs/siglip2_giant.yaml b/configs/siglip2_giant.yaml index c6ea59e..cd06f0c 100644 --- a/configs/siglip2_giant.yaml +++ b/configs/siglip2_giant.yaml @@ -12,7 +12,7 @@ data: model: backbone: siglip2 clip_model: openai/clip-vit-base-patch32 - siglip2_model: google/siglip2-giant-opt-patch16-384 + siglip2_model: /root/autodl-tmp/ces-models/siglip2-giant state_dim: 512 num_visual_slots: 6 num_text_slots: 3