from __future__ import annotations import torch import torch.nn.functional as F from data.schema import FACT_TYPE def matched_states( states: torch.Tensor, targets: list[torch.Tensor], assignments: list[tuple[torch.Tensor, torch.Tensor]], state_type: int | None = None, ) -> tuple[torch.Tensor, torch.Tensor]: vectors = [] labels = [] for batch_index, (slot_index, target_index) in enumerate(assignments): if not len(slot_index): continue selected_targets = targets[batch_index].to(states.device)[target_index] selected_states = states[batch_index, slot_index] if state_type is not None: keep = selected_targets[:, 0] == state_type selected_targets = selected_targets[keep] selected_states = selected_states[keep] if len(selected_states): vectors.append(selected_states) labels.append(selected_targets) if not vectors: return states.new_empty((0, states.shape[-1])), torch.empty( (0, 6), dtype=torch.long, device=states.device ) return torch.cat(vectors), torch.cat(labels) def _directional_multi_positive_loss( query: torch.Tensor, candidates: torch.Tensor, positives: torch.Tensor, temperature: float, ) -> torch.Tensor: usable = positives.any(dim=1) if not usable.any(): return query.sum() * 0.0 logits = query @ candidates.T / temperature log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True) positive_count = positives.sum(1).clamp_min(1) per_query = -(log_prob.masked_fill(~positives, 0.0).sum(1) / positive_count) return per_query[usable].mean() def cross_modal_alignment_loss( visual_states: torch.Tensor, visual_targets: list[torch.Tensor], visual_assignments: list[tuple[torch.Tensor, torch.Tensor]], text_states: torch.Tensor, text_targets: list[torch.Tensor], text_assignments: list[tuple[torch.Tensor, torch.Tensor]], temperature: float = 0.1, ) -> torch.Tensor: """Align identical FACT states across modalities; RULE states remain text-only.""" visual, visual_labels = matched_states( visual_states, visual_targets, visual_assignments, FACT_TYPE ) text, text_labels = matched_states(text_states, text_targets, text_assignments, FACT_TYPE) if not len(visual) or not len(text): return visual_states.sum() * 0.0 visual = F.normalize(visual, dim=-1) text = F.normalize(text, dim=-1) positives = (visual_labels[:, None, :] == text_labels[None, :, :]).all(-1) return 0.5 * ( _directional_multi_positive_loss(visual, text, positives, temperature) + _directional_multi_positive_loss(text, visual, positives.T, temperature) )