64 lines
2.4 KiB
Python
64 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from data.schema import ENTITIES, NUM_STATE_TYPES, PREDICATES
|
|
from models.fact_prototype import CanonicalFactPrototype
|
|
from models.frozen_backbone import build_backbone
|
|
from models.slot_extractor import SlotExtractor
|
|
|
|
|
|
class EvidenceExtractorModel(nn.Module):
|
|
def __init__(self, config: dict[str, Any], backbone_name: str | None = None):
|
|
super().__init__()
|
|
model_config = config["model"]
|
|
selected = backbone_name or model_config["backbone"]
|
|
self.backbone_name = selected
|
|
model_name = (
|
|
model_config["siglip2_model"]
|
|
if selected == "siglip2"
|
|
else model_config["clip_model"]
|
|
)
|
|
self.backbone = build_backbone(selected, model_name)
|
|
common = {
|
|
"state_dim": model_config["state_dim"],
|
|
"num_layers": model_config["extractor_layers"],
|
|
"num_heads": model_config["num_heads"],
|
|
"dropout": model_config["dropout"],
|
|
"num_subjects": len(ENTITIES),
|
|
"num_predicates": len(PREDICATES),
|
|
"num_objects": len(ENTITIES),
|
|
"num_state_types": NUM_STATE_TYPES,
|
|
}
|
|
self.visual_extractor = SlotExtractor(
|
|
input_dim=self.backbone.vision_dim,
|
|
num_slots=model_config["num_visual_slots"],
|
|
**common,
|
|
)
|
|
self.text_extractor = SlotExtractor(
|
|
input_dim=self.backbone.text_dim,
|
|
num_slots=model_config["num_text_slots"],
|
|
**common,
|
|
)
|
|
prototype_config = config.get("prototype", {})
|
|
self.prototype_encoder = None
|
|
if prototype_config.get("enabled", False):
|
|
self.prototype_encoder = CanonicalFactPrototype(
|
|
state_dim=model_config["state_dim"],
|
|
num_state_types=NUM_STATE_TYPES,
|
|
num_subjects=len(ENTITIES),
|
|
num_predicates=len(PREDICATES),
|
|
num_objects=len(ENTITIES),
|
|
)
|
|
|
|
def forward(self, images: torch.Tensor, rules_text: list[str]) -> dict[str, Any]:
|
|
visual_tokens, visual_mask = self.backbone.encode_image(images)
|
|
text_tokens, text_mask = self.backbone.encode_text(rules_text)
|
|
return {
|
|
"visual": self.visual_extractor(visual_tokens, visual_mask),
|
|
"text": self.text_extractor(text_tokens, text_mask),
|
|
}
|