37 lines
1.3 KiB
Python
37 lines
1.3 KiB
Python
from __future__ import annotations
|
|
|
|
import torch
|
|
import torch.nn.functional as F
|
|
|
|
|
|
def supervised_alignment_loss(
|
|
states: torch.Tensor,
|
|
targets: list[torch.Tensor],
|
|
assignments: list[tuple[torch.Tensor, torch.Tensor]],
|
|
temperature: float = 0.1,
|
|
) -> torch.Tensor:
|
|
vectors = []
|
|
labels = []
|
|
for batch_index, (slot_index, fact_index) in enumerate(assignments):
|
|
if len(slot_index) == 0:
|
|
continue
|
|
vectors.append(states[batch_index, slot_index])
|
|
labels.append(targets[batch_index].to(states.device)[fact_index])
|
|
if not vectors:
|
|
return states.sum() * 0.0
|
|
vectors_tensor = F.normalize(torch.cat(vectors), dim=-1)
|
|
labels_tensor = torch.cat(labels)
|
|
if len(vectors_tensor) < 2:
|
|
return vectors_tensor.sum() * 0.0
|
|
same = (labels_tensor[:, None, :] == labels_tensor[None, :, :]).all(-1)
|
|
same.fill_diagonal_(False)
|
|
logits = vectors_tensor @ vectors_tensor.T / temperature
|
|
logits.fill_diagonal_(-torch.inf)
|
|
log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
|
|
positive_count = same.sum(1)
|
|
usable = positive_count > 0
|
|
if not usable.any():
|
|
return vectors_tensor.sum() * 0.0
|
|
return -((log_prob.masked_fill(~same, 0.0).sum(1) / positive_count.clamp_min(1))[usable]).mean()
|
|
|