Add resumable SigLIP2 Giant formal training
CI / test (push) Successful in 13s

This commit is contained in:
2026-10-01 23:40:40 +08:00
parent 521dfb3d57
commit bfb96b76c7
11 changed files with 341 additions and 27 deletions
+6 -1
View File
@@ -17,7 +17,12 @@ class EvidenceExtractorModel(nn.Module):
model_config = config["model"]
selected = backbone_name or model_config["backbone"]
self.backbone_name = selected
self.backbone = build_backbone(selected, model_config["clip_model"])
model_name = (
model_config["siglip2_model"]
if selected == "siglip2"
else model_config["clip_model"]
)
self.backbone = build_backbone(selected, model_name)
common = {
"state_dim": model_config["state_dim"],
"num_layers": model_config["extractor_layers"],
+64 -2
View File
@@ -102,9 +102,71 @@ class CLIPFrozenBackbone(FrozenBackbone):
return tokens, values["attention_mask"].bool()
def build_backbone(name: str, clip_model: str) -> FrozenBackbone:
class SigLIP2FrozenBackbone(FrozenBackbone):
"""Billion-scale paired image/text encoder used for formal experiments."""
def __init__(self, model_name: str):
super().__init__()
try:
from transformers import AutoModel, AutoProcessor
except ImportError as error:
raise RuntimeError("Install the 'hf' extra to use SigLIP2.") from error
dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
self.model = AutoModel.from_pretrained(
model_name,
torch_dtype=dtype,
low_cpu_mem_usage=True,
)
self.processor = AutoProcessor.from_pretrained(model_name)
self.model.requires_grad_(False).eval()
self.vision_dim = self.model.config.vision_config.hidden_size
self.text_dim = self.model.config.text_config.hidden_size
def _to_model_device(self, values: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
device = next(self.model.parameters()).device
dtype = next(self.model.parameters()).dtype
moved = {}
for key, value in values.items():
moved[key] = (
value.to(device=device, dtype=dtype)
if value.is_floating_point()
else value.to(device=device)
)
return moved
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
values = self.processor(images=images.detach().cpu(), return_tensors="pt")
allowed = {"pixel_values", "pixel_attention_mask", "spatial_shapes"}
vision_values = self._to_model_device(
{key: value for key, value in values.items() if key in allowed}
)
with torch.no_grad():
tokens = self.model.vision_model(**vision_values).last_hidden_state
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
return tokens, mask
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
max_length = self.model.config.text_config.max_position_embeddings
values = self.processor(
text=texts,
padding="max_length",
truncation=True,
max_length=max_length,
return_tensors="pt",
)
text_values = self._to_model_device(
{key: value for key, value in values.items() if key in {"input_ids", "attention_mask"}}
)
with torch.no_grad():
tokens = self.model.text_model(**text_values).last_hidden_state
return tokens, text_values["attention_mask"].bool()
def build_backbone(name: str, model_name: str) -> FrozenBackbone:
if name == "tiny":
return TinyFrozenBackbone()
if name == "clip":
return CLIPFrozenBackbone(clip_model)
return CLIPFrozenBackbone(model_name)
if name == "siglip2":
return SigLIP2FrozenBackbone(model_name)
raise ValueError(f"Unknown backbone: {name}")