This commit is contained in:
+21
-11
@@ -90,19 +90,24 @@ def summarize(counts: dict[str, int]) -> dict[str, float]:
|
||||
}
|
||||
|
||||
|
||||
def null_slot_counts(
|
||||
def slot_type_counts(
|
||||
predictions: dict[str, torch.Tensor], targets: list[torch.Tensor]
|
||||
) -> tuple[int, int]:
|
||||
) -> tuple[int, int, int, int]:
|
||||
matching = hungarian_matching_loss(predictions, targets)
|
||||
predicted_types = predictions["type_logits"].argmax(-1)
|
||||
correct = 0
|
||||
total = predicted_types.numel()
|
||||
type_correct = 0
|
||||
type_total = predicted_types.numel()
|
||||
null_correct = 0
|
||||
null_total = 0
|
||||
for batch_index, (slots, target_indices) in enumerate(matching.assignments):
|
||||
expected = torch.full_like(predicted_types[batch_index], NULL_TYPE)
|
||||
if len(slots):
|
||||
expected[slots] = targets[batch_index].to(expected.device)[target_indices, 0]
|
||||
correct += int((predicted_types[batch_index] == expected).sum())
|
||||
return correct, total
|
||||
type_correct += int((predicted_types[batch_index] == expected).sum())
|
||||
null_positions = expected == NULL_TYPE
|
||||
null_correct += int((predicted_types[batch_index][null_positions] == NULL_TYPE).sum())
|
||||
null_total += int(null_positions.sum())
|
||||
return null_correct, null_total, type_correct, type_total
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
@@ -118,6 +123,8 @@ def evaluate(
|
||||
}
|
||||
null_correct = 0
|
||||
null_total = 0
|
||||
type_correct = 0
|
||||
type_total = 0
|
||||
for batch in loader:
|
||||
outputs = model(batch["images"].to(device), batch["rules_text"])
|
||||
requests = (
|
||||
@@ -134,11 +141,14 @@ def evaluate(
|
||||
("visual", "visual_targets"),
|
||||
("text", "text_targets"),
|
||||
):
|
||||
correct, count = null_slot_counts(outputs[modality], batch[target_name])
|
||||
null_correct += correct
|
||||
null_total += count
|
||||
counts = slot_type_counts(outputs[modality], batch[target_name])
|
||||
null_correct += counts[0]
|
||||
null_total += counts[1]
|
||||
type_correct += counts[2]
|
||||
type_total += counts[3]
|
||||
result = {name: summarize(counts) for name, counts in totals.items()}
|
||||
result["null_slot_accuracy"] = null_correct / max(1, null_total)
|
||||
result["slot_type_accuracy"] = type_correct / max(1, type_total)
|
||||
return result
|
||||
|
||||
|
||||
@@ -146,14 +156,14 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--backbone", choices=("tiny", "clip"), default=None)
|
||||
parser.add_argument("--backbone", choices=("tiny", "clip", "siglip2"), default=None)
|
||||
parser.add_argument("--split", default="test_id")
|
||||
args = parser.parse_args()
|
||||
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model = EvidenceExtractorModel(config, backbone_name=args.backbone).to(device)
|
||||
state = torch.load(args.checkpoint, map_location=device, weights_only=True)
|
||||
model.load_state_dict(state["model"])
|
||||
model.load_state_dict(state["model"], strict=False)
|
||||
dataset = EvidenceDataset(
|
||||
Path(config["data"]["root"]) / f"{args.split}.jsonl",
|
||||
config["data"]["image_size"],
|
||||
|
||||
Reference in New Issue
Block a user