75 lines
2.7 KiB
Python
75 lines
2.7 KiB
Python
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
from data.schema import FACT_TYPE
|
|
|
|
|
|
def matched_states(
|
|
states: torch.Tensor,
|
|
targets: list[torch.Tensor],
|
|
assignments: list[tuple[torch.Tensor, torch.Tensor]],
|
|
state_type: int | None = None,
|
|
) -> tuple[torch.Tensor, torch.Tensor]:
|
|
vectors = []
|
|
labels = []
|
|
for batch_index, (slot_index, target_index) in enumerate(assignments):
|
|
if not len(slot_index):
|
|
continue
|
|
selected_targets = targets[batch_index].to(states.device)[target_index]
|
|
selected_states = states[batch_index, slot_index]
|
|
if state_type is not None:
|
|
keep = selected_targets[:, 0] == state_type
|
|
selected_targets = selected_targets[keep]
|
|
selected_states = selected_states[keep]
|
|
if len(selected_states):
|
|
vectors.append(selected_states)
|
|
labels.append(selected_targets)
|
|
if not vectors:
|
|
return states.new_empty((0, states.shape[-1])), torch.empty(
|
|
(0, 6), dtype=torch.long, device=states.device
|
|
)
|
|
return torch.cat(vectors), torch.cat(labels)
|
|
|
|
|
|
def _directional_multi_positive_loss(
|
|
query: torch.Tensor,
|
|
candidates: torch.Tensor,
|
|
positives: torch.Tensor,
|
|
temperature: float,
|
|
) -> torch.Tensor:
|
|
usable = positives.any(dim=1)
|
|
if not usable.any():
|
|
return query.sum() * 0.0
|
|
logits = query @ candidates.T / temperature
|
|
log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
|
|
positive_count = positives.sum(1).clamp_min(1)
|
|
per_query = -(log_prob.masked_fill(~positives, 0.0).sum(1) / positive_count)
|
|
return per_query[usable].mean()
|
|
|
|
|
|
def cross_modal_alignment_loss(
|
|
visual_states: torch.Tensor,
|
|
visual_targets: list[torch.Tensor],
|
|
visual_assignments: list[tuple[torch.Tensor, torch.Tensor]],
|
|
text_states: torch.Tensor,
|
|
text_targets: list[torch.Tensor],
|
|
text_assignments: list[tuple[torch.Tensor, torch.Tensor]],
|
|
temperature: float = 0.1,
|
|
) -> torch.Tensor:
|
|
"""Align identical FACT states across modalities; RULE states remain text-only."""
|
|
visual, visual_labels = matched_states(
|
|
visual_states, visual_targets, visual_assignments, FACT_TYPE
|
|
)
|
|
text, text_labels = matched_states(text_states, text_targets, text_assignments, FACT_TYPE)
|
|
if not len(visual) or not len(text):
|
|
return visual_states.sum() * 0.0
|
|
visual = F.normalize(visual, dim=-1)
|
|
text = F.normalize(text, dim=-1)
|
|
positives = (visual_labels[:, None, :] == text_labels[None, :, :]).all(-1)
|
|
return 0.5 * (
|
|
_directional_multi_positive_loss(visual, text, positives, temperature)
|
|
+ _directional_multi_positive_loss(text, visual, positives.T, temperature)
|
|
)
|