This commit is contained in:
@@ -17,7 +17,12 @@ class EvidenceExtractorModel(nn.Module):
|
||||
model_config = config["model"]
|
||||
selected = backbone_name or model_config["backbone"]
|
||||
self.backbone_name = selected
|
||||
self.backbone = build_backbone(selected, model_config["clip_model"])
|
||||
model_name = (
|
||||
model_config["siglip2_model"]
|
||||
if selected == "siglip2"
|
||||
else model_config["clip_model"]
|
||||
)
|
||||
self.backbone = build_backbone(selected, model_name)
|
||||
common = {
|
||||
"state_dim": model_config["state_dim"],
|
||||
"num_layers": model_config["extractor_layers"],
|
||||
|
||||
@@ -102,9 +102,71 @@ class CLIPFrozenBackbone(FrozenBackbone):
|
||||
return tokens, values["attention_mask"].bool()
|
||||
|
||||
|
||||
def build_backbone(name: str, clip_model: str) -> FrozenBackbone:
|
||||
class SigLIP2FrozenBackbone(FrozenBackbone):
|
||||
"""Billion-scale paired image/text encoder used for formal experiments."""
|
||||
|
||||
def __init__(self, model_name: str):
|
||||
super().__init__()
|
||||
try:
|
||||
from transformers import AutoModel, AutoProcessor
|
||||
except ImportError as error:
|
||||
raise RuntimeError("Install the 'hf' extra to use SigLIP2.") from error
|
||||
dtype = torch.bfloat16 if torch.cuda.is_available() else torch.float32
|
||||
self.model = AutoModel.from_pretrained(
|
||||
model_name,
|
||||
torch_dtype=dtype,
|
||||
low_cpu_mem_usage=True,
|
||||
)
|
||||
self.processor = AutoProcessor.from_pretrained(model_name)
|
||||
self.model.requires_grad_(False).eval()
|
||||
self.vision_dim = self.model.config.vision_config.hidden_size
|
||||
self.text_dim = self.model.config.text_config.hidden_size
|
||||
|
||||
def _to_model_device(self, values: dict[str, torch.Tensor]) -> dict[str, torch.Tensor]:
|
||||
device = next(self.model.parameters()).device
|
||||
dtype = next(self.model.parameters()).dtype
|
||||
moved = {}
|
||||
for key, value in values.items():
|
||||
moved[key] = (
|
||||
value.to(device=device, dtype=dtype)
|
||||
if value.is_floating_point()
|
||||
else value.to(device=device)
|
||||
)
|
||||
return moved
|
||||
|
||||
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
values = self.processor(images=images.detach().cpu(), return_tensors="pt")
|
||||
allowed = {"pixel_values", "pixel_attention_mask", "spatial_shapes"}
|
||||
vision_values = self._to_model_device(
|
||||
{key: value for key, value in values.items() if key in allowed}
|
||||
)
|
||||
with torch.no_grad():
|
||||
tokens = self.model.vision_model(**vision_values).last_hidden_state
|
||||
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
|
||||
return tokens, mask
|
||||
|
||||
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
max_length = self.model.config.text_config.max_position_embeddings
|
||||
values = self.processor(
|
||||
text=texts,
|
||||
padding="max_length",
|
||||
truncation=True,
|
||||
max_length=max_length,
|
||||
return_tensors="pt",
|
||||
)
|
||||
text_values = self._to_model_device(
|
||||
{key: value for key, value in values.items() if key in {"input_ids", "attention_mask"}}
|
||||
)
|
||||
with torch.no_grad():
|
||||
tokens = self.model.text_model(**text_values).last_hidden_state
|
||||
return tokens, text_values["attention_mask"].bool()
|
||||
|
||||
|
||||
def build_backbone(name: str, model_name: str) -> FrozenBackbone:
|
||||
if name == "tiny":
|
||||
return TinyFrozenBackbone()
|
||||
if name == "clip":
|
||||
return CLIPFrozenBackbone(clip_model)
|
||||
return CLIPFrozenBackbone(model_name)
|
||||
if name == "siglip2":
|
||||
return SigLIP2FrozenBackbone(model_name)
|
||||
raise ValueError(f"Unknown backbone: {name}")
|
||||
|
||||
Reference in New Issue
Block a user