103 lines
4.4 KiB
YAML
103 lines
4.4 KiB
YAML
name: Formal GPU training
|
|
|
|
on:
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
extractor:
|
|
runs-on: gpu
|
|
timeout-minutes: 1440
|
|
env:
|
|
TOKENIZERS_PARALLELISM: "false"
|
|
steps:
|
|
- name: Checkout from Gitea
|
|
run: |
|
|
git init .
|
|
git remote add origin "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY.git"
|
|
git fetch --depth 1 origin "$GITHUB_SHA"
|
|
git checkout --detach FETCH_HEAD
|
|
|
|
- name: Experiment details
|
|
run: |
|
|
echo "::group::Hardware"
|
|
nvidia-smi
|
|
python -c "import torch; print(torch.__version__, torch.version.cuda, torch.cuda.get_device_name(), torch.cuda.get_device_capability())"
|
|
echo "::endgroup::"
|
|
echo "::group::Configuration"
|
|
cat configs/siglip2_giant.yaml
|
|
echo "::endgroup::"
|
|
|
|
- name: Prepare formal environment
|
|
run: |
|
|
source /etc/network_turbo
|
|
python -m venv --system-site-packages /root/autodl-tmp/ces-venv
|
|
/root/autodl-tmp/ces-venv/bin/python -m pip install numpy pillow pyyaml "transformers>=4.48,<5" accelerate
|
|
unset http_proxy https_proxy
|
|
|
|
- name: Build full synthetic dataset
|
|
run: |
|
|
/root/autodl-tmp/ces-venv/bin/python -m data.build_splits \
|
|
--config configs/siglip2_giant.yaml \
|
|
--reuse-existing
|
|
|
|
- name: Verify preloaded SigLIP2 Giant weights
|
|
run: |
|
|
MODEL_DIR=/root/autodl-tmp/ces-models/siglip2-giant
|
|
test -s "$MODEL_DIR/config.json"
|
|
test -s "$MODEL_DIR/model.safetensors.index.json"
|
|
test -s "$MODEL_DIR/model-00001-of-00002.safetensors"
|
|
test -s "$MODEL_DIR/model-00002-of-00002.safetensors"
|
|
test -s "$MODEL_DIR/preprocessor_config.json"
|
|
test -s "$MODEL_DIR/tokenizer.json"
|
|
test -s "$MODEL_DIR/tokenizer.model"
|
|
test -s "$MODEL_DIR/tokenizer_config.json"
|
|
du -sh "$MODEL_DIR"
|
|
ls -lh "$MODEL_DIR"
|
|
|
|
- name: Train SigLIP2 Giant extractor
|
|
run: |
|
|
source /etc/network_turbo
|
|
/root/autodl-tmp/ces-venv/bin/python train_extractor.py \
|
|
--config configs/siglip2_giant.yaml \
|
|
--backbone siglip2
|
|
unset http_proxy https_proxy
|
|
|
|
- name: Enforce research gates
|
|
run: |
|
|
/root/autodl-tmp/ces-venv/bin/python -m eval.benchmark_extractor \
|
|
--config configs/siglip2_giant.yaml \
|
|
--checkpoint /root/autodl-tmp/ces-artifacts/siglip2-giant/best.pt \
|
|
--backbone siglip2 \
|
|
--output /root/autodl-tmp/ces-artifacts/siglip2-giant/benchmark.json \
|
|
--enforce-gates
|
|
|
|
- name: Details
|
|
if: always()
|
|
run: |
|
|
echo "::group::Run details"
|
|
echo "repository=$GITHUB_REPOSITORY"
|
|
echo "sha=$GITHUB_SHA"
|
|
echo "run_id=$GITHUB_RUN_ID"
|
|
date -Iseconds
|
|
echo "::endgroup::"
|
|
echo "::group::Hardware and software"
|
|
nvidia-smi || true
|
|
/root/autodl-tmp/ces-venv/bin/python -c "import platform, torch, transformers; print({'python': platform.python_version(), 'torch': torch.__version__, 'torch_cuda': torch.version.cuda, 'transformers': transformers.__version__, 'cuda_available': torch.cuda.is_available(), 'gpu': torch.cuda.get_device_name() if torch.cuda.is_available() else None, 'capability': torch.cuda.get_device_capability() if torch.cuda.is_available() else None})" || true
|
|
echo "::endgroup::"
|
|
echo "::group::Configuration"
|
|
cat configs/siglip2_giant.yaml
|
|
echo "::endgroup::"
|
|
echo "::group::Dataset details"
|
|
find /root/autodl-tmp/ces-data/synthetic-v1 -maxdepth 2 -type f -name '*.jsonl' -exec wc -l {} + || true
|
|
test -f /root/autodl-tmp/ces-data/synthetic-v1/manifest.json && cat /root/autodl-tmp/ces-data/synthetic-v1/manifest.json || true
|
|
echo "::endgroup::"
|
|
echo "::group::Epoch details"
|
|
test -f /root/autodl-tmp/ces-artifacts/siglip2-giant/training_details.jsonl && cat /root/autodl-tmp/ces-artifacts/siglip2-giant/training_details.jsonl || true
|
|
echo "::endgroup::"
|
|
echo "::group::Benchmark details"
|
|
test -f /root/autodl-tmp/ces-artifacts/siglip2-giant/benchmark.json && cat /root/autodl-tmp/ces-artifacts/siglip2-giant/benchmark.json || true
|
|
echo "::endgroup::"
|
|
echo "::group::Saved artifacts"
|
|
ls -lh /root/autodl-tmp/ces-artifacts/siglip2-giant || true
|
|
echo "::endgroup::"
|