Files
composable-evidence-states-…/losses/alignment_loss.py
T
2026-10-01 22:44:07 +08:00

75 lines
2.7 KiB
Python

from __future__ import annotations
import torch
import torch.nn.functional as F
from data.schema import FACT_TYPE
def matched_states(
states: torch.Tensor,
targets: list[torch.Tensor],
assignments: list[tuple[torch.Tensor, torch.Tensor]],
state_type: int | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
vectors = []
labels = []
for batch_index, (slot_index, target_index) in enumerate(assignments):
if not len(slot_index):
continue
selected_targets = targets[batch_index].to(states.device)[target_index]
selected_states = states[batch_index, slot_index]
if state_type is not None:
keep = selected_targets[:, 0] == state_type
selected_targets = selected_targets[keep]
selected_states = selected_states[keep]
if len(selected_states):
vectors.append(selected_states)
labels.append(selected_targets)
if not vectors:
return states.new_empty((0, states.shape[-1])), torch.empty(
(0, 6), dtype=torch.long, device=states.device
)
return torch.cat(vectors), torch.cat(labels)
def _directional_multi_positive_loss(
query: torch.Tensor,
candidates: torch.Tensor,
positives: torch.Tensor,
temperature: float,
) -> torch.Tensor:
usable = positives.any(dim=1)
if not usable.any():
return query.sum() * 0.0
logits = query @ candidates.T / temperature
log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
positive_count = positives.sum(1).clamp_min(1)
per_query = -(log_prob.masked_fill(~positives, 0.0).sum(1) / positive_count)
return per_query[usable].mean()
def cross_modal_alignment_loss(
visual_states: torch.Tensor,
visual_targets: list[torch.Tensor],
visual_assignments: list[tuple[torch.Tensor, torch.Tensor]],
text_states: torch.Tensor,
text_targets: list[torch.Tensor],
text_assignments: list[tuple[torch.Tensor, torch.Tensor]],
temperature: float = 0.1,
) -> torch.Tensor:
"""Align identical FACT states across modalities; RULE states remain text-only."""
visual, visual_labels = matched_states(
visual_states, visual_targets, visual_assignments, FACT_TYPE
)
text, text_labels = matched_states(text_states, text_targets, text_assignments, FACT_TYPE)
if not len(visual) or not len(text):
return visual_states.sum() * 0.0
visual = F.normalize(visual, dim=-1)
text = F.normalize(text, dim=-1)
positives = (visual_labels[:, None, :] == text_labels[None, :, :]).all(-1)
return 0.5 * (
_directional_multi_positive_loss(visual, text, positives, temperature)
+ _directional_multi_positive_loss(text, visual, positives.T, temperature)
)