From b3774e783c7e32d0374b631152015a40106304df Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Thu, 1 Oct 2026 22:44:07 +0800 Subject: [PATCH] Implement typed evidence latent training and benchmarks --- .gitea/workflows/ci.yaml | 1 + .gitea/workflows/gpu-smoke.yaml | 4 +- README.md | 84 ++++++++++++---- configs/mvp.yaml | 17 +++- configs/prototype.yaml | 44 +++++++++ data/build_splits.py | 6 +- data/dataset.py | 44 +++++++-- data/generate_rules.py | 14 ++- data/generate_scenes.py | 23 ++++- data/schema.py | 27 +++++- eval/benchmark_extractor.py | 163 ++++++++++++++++++++++++++++++++ eval/evaluate_slots.py | 125 +++++++++++++++++++----- losses/alignment_loss.py | 82 +++++++++++----- losses/matching_loss.py | 46 ++++++--- losses/prototype_loss.py | 25 +++++ models/evidence_model.py | 15 ++- models/fact_prototype.py | 45 +++++++++ models/slot_extractor.py | 10 +- tests/test_benchmark.py | 17 ++++ tests/test_data.py | 9 +- tests/test_model.py | 30 +++++- train_extractor.py | 35 +++++-- 22 files changed, 747 insertions(+), 119 deletions(-) create mode 100644 configs/prototype.yaml create mode 100644 eval/benchmark_extractor.py create mode 100644 losses/prototype_loss.py create mode 100644 models/fact_prototype.py create mode 100644 tests/test_benchmark.py diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 0c7031e..003c400 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -28,3 +28,4 @@ jobs: run: | .venv-ci/bin/python -m data.build_splits --config configs/mvp.yaml --small .venv-ci/bin/python train_extractor.py --config configs/mvp.yaml --backbone tiny --epochs 1 + .venv-ci/bin/python -m eval.benchmark_extractor --config configs/mvp.yaml --checkpoint artifacts/tiny/best.pt --backbone tiny --output artifacts/tiny/benchmark.json diff --git a/.gitea/workflows/gpu-smoke.yaml b/.gitea/workflows/gpu-smoke.yaml index 97ef470..3570846 100644 --- a/.gitea/workflows/gpu-smoke.yaml +++ b/.gitea/workflows/gpu-smoke.yaml @@ -26,4 +26,6 @@ jobs: - name: Build smoke dataset run: .venv-gpu/bin/python -m data.build_splits --config configs/mvp.yaml --small - name: Train extractor - run: .venv-gpu/bin/python train_extractor.py --config configs/mvp.yaml --backbone clip --epochs 1 + run: | + .venv-gpu/bin/python train_extractor.py --config configs/mvp.yaml --backbone clip --epochs 1 + .venv-gpu/bin/python -m eval.benchmark_extractor --config configs/mvp.yaml --checkpoint artifacts/clip/best.pt --backbone clip --output artifacts/clip/benchmark.json diff --git a/README.md b/README.md index 9fb00c6..a38b422 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,49 @@ # Composable Evidence States Research -This repository contains the first-stage MVP for learning fact-level evidence slots from -synthetic images and natural-language rules. The Composer is intentionally deferred until -the extractor passes its fact-recovery checks. +第一阶段 MVP:在全合成符号世界中生成图像事实与文本事实/规则,训练 Evidence +Extractor 将两种模态映射到可比较的 latent space。Composer 暂不在本阶段实现。 -## Current scope +## 数据方案 A:全合成符号世界 -- Generate deterministic scenes containing colored geometric objects and spatial relations. -- Render scenes to PNG while preserving machine-readable fact triples. -- Generate textual implication rules as normalized triples. -- Build IID and held-out-combination splits without leaking held-out rule pairs into training. -- Train visual and textual slot extractors with Hungarian set matching. -- Measure triple exact match, component accuracy, precision, recall, and F1. -- Run CPU smoke tests with a tiny frozen backbone or GPU experiments with frozen CLIP. +每张图包含两个带颜色和形状的对象以及一条空间关系。manifest 保存完整结构化标注, +同时生成两类文本状态: -## Quick start +- `FACT`:例如 `obj1 is red`,结构为 + `[FACT, subject, predicate, object, -, -]`。 +- `RULE`:例如 `Every red object belongs to class alpha`,结构为 + `[RULE, variable, antecedent predicate, antecedent object, + consequent predicate, consequent object]`。 +- `NULL`:表示未被事实或规则占用的 slot。 + +FACT 与 RULE 共享 subject、predicate、object 分类头,但类型头明确区分二者;RULE +还有独立的 consequent predicate/object 训练目标。视觉分支只产生 FACT,文本分支同时 +产生 FACT 和 RULE。因此实现不会把“事实”和“规则”当成完全相同的对象。 + +`test_ood` 使用训练集中明确排除的颜色–形状组合,并同时改变规则映射,用来检查组合泛化。 + +## 统一 latent space 的两个实验方案 + +### 方案 1(当前默认):共享分类头 + 跨模态对比学习 + +视觉和文本 extractor 使用相同的标签空间与同构分类头。Hungarian set matching 处理无序 +slots;对比损失只把语义完全相同的视觉 FACT 与文本 FACT 拉近。RULE 保持为文本独有的 +有类型状态,不参与错误的视觉规则对齐。 + +配置:`configs/mvp.yaml`,其中 `prototype.enabled: false`。 + +### 方案 2(已实现、默认关闭):Canonical Fact Prototype + +`CanonicalFactPrototype` 将类型、主语、前件谓词/宾语以及后件谓词/宾语的可学习 embedding +组合成规范原型,并用 cosine loss 把 matched latent state 拉向该原型。类型 embedding 与 +RULE 后件字段确保 FACT 和 RULE 的原型结构不同。 + +使用完整的 `configs/prototype.yaml` 即可单独训练和比较: + +```bash +uv run python train_extractor.py --config configs/prototype.yaml --backbone tiny +``` + +## 运行 ```bash uv sync --extra dev @@ -25,7 +54,7 @@ uv run python -m eval.evaluate_slots --config configs/mvp.yaml \ uv run pytest ``` -For the first real GPU experiment: +正式 GPU 实验可把 `--small` 去掉并改用冻结 CLIP: ```bash uv sync --extra dev --extra hf @@ -33,14 +62,27 @@ uv run python -m data.build_splits --config configs/mvp.yaml uv run python train_extractor.py --config configs/mvp.yaml --backbone clip ``` -The CLIP path uses `openai/clip-vit-base-patch32` by default. It is a deliberately small -backbone for validating the extractor. A Qwen-VL adapter and the Composer belong to the next -milestone after the evidence slots are shown to recover atomic facts reliably. +## 每次训练后的独立验收 -## Success gate before Composer work +以下命令运行统一 benchmark,写出 JSON,并在任一指标未达标时返回非零退出码: -- Text-rule triple F1 >= 0.95. -- Visual-fact triple F1 >= 0.90 on the IID test split. -- No severe slot collapse: predicted valid slots track the true fact count. -- Held-out rule-combination performance is reported separately from IID performance. +```bash +uv run python -m eval.benchmark_extractor \ + --config configs/mvp.yaml \ + --checkpoint artifacts/clip/best.pt \ + --backbone clip \ + --output artifacts/clip/benchmark.json \ + --enforce-gates +``` +| 指标 | 门槛 | 定义 | +| --- | ---: | --- | +| Text fact/rule F1 | 0.98 | IID 文本中 typed FACT 与 typed RULE 的六字段 exact-match micro F1 | +| Visual fact F1 | 0.90 | IID 图像 FACT 的四个有效字段 exact-match micro F1 | +| Cross-modal Recall@1 | 0.80 | 文本 FACT 检索全部视觉 FACT,最近邻标签完全相同 | +| Hard-negative Recall@1 | 0.75 | 候选仅含正确项与恰好错一个基础字段的视觉 FACT | +| OOD fact F1 | 0.80 | held-out 颜色–形状组合上的视觉 FACT exact-match F1 | +| NULL slot accuracy | 0.95 | permutation matching 后,全部视觉/文本 slots 的类型准确率,含 NULL | + +CI 的单 epoch smoke run 会执行 benchmark 来验证整条评测管线,但不会使用 +`--enforce-gates`,因为 smoke 模型并不代表正式实验是否达标。正式训练完成后使用上述强制命令。 diff --git a/configs/mvp.yaml b/configs/mvp.yaml index 7939523..5fd44d0 100644 --- a/configs/mvp.yaml +++ b/configs/mvp.yaml @@ -25,7 +25,20 @@ training: weight_decay: 0.01 epochs: 10 num_workers: 0 - valid_weight: 0.5 - alignment_weight: 0.05 + type_weight: 0.5 + alignment_weight: 0.2 + contrastive_temperature: 0.1 output_dir: artifacts/tiny +prototype: + enabled: false + weight: 0.2 + +evaluation: + gates: + text_fact_rule_f1: 0.98 + visual_fact_f1: 0.90 + cross_modal_recall_at_1: 0.80 + hard_negative_recall_at_1: 0.75 + ood_fact_f1: 0.80 + null_slot_accuracy: 0.95 diff --git a/configs/prototype.yaml b/configs/prototype.yaml new file mode 100644 index 0000000..7f66f74 --- /dev/null +++ b/configs/prototype.yaml @@ -0,0 +1,44 @@ +seed: 17 + +data: + root: datasets/mvp + image_size: 224 + train_size: 30000 + validation_size: 3000 + test_id_size: 3000 + test_ood_size: 3000 + num_objects: 2 + +model: + backbone: tiny + clip_model: openai/clip-vit-base-patch32 + state_dim: 256 + num_visual_slots: 6 + num_text_slots: 3 + extractor_layers: 2 + num_heads: 8 + dropout: 0.1 + +training: + batch_size: 32 + learning_rate: 0.0003 + weight_decay: 0.01 + epochs: 10 + num_workers: 0 + type_weight: 0.5 + alignment_weight: 0.2 + contrastive_temperature: 0.1 + output_dir: artifacts/prototype + +prototype: + enabled: true + weight: 0.2 + +evaluation: + gates: + text_fact_rule_f1: 0.98 + visual_fact_f1: 0.90 + cross_modal_recall_at_1: 0.80 + hard_negative_recall_at_1: 0.75 + ood_fact_f1: 0.80 + null_slot_accuracy: 0.95 diff --git a/data/build_splits.py b/data/build_splits.py index 43f66a7..612177d 100644 --- a/data/build_splits.py +++ b/data/build_splits.py @@ -23,7 +23,10 @@ def _write_split( path = root / f"{name}.jsonl" with path.open("w", encoding="utf-8") as handle: for scene_number in range(start, start + size): - record = add_rule(generate_scene(scene_number, seed), seed, held_out=held_out) + scene = generate_scene( + scene_number, seed, held_out_composition=held_out + ) + record = add_rule(scene, seed, held_out=held_out) relative_image = Path("images") / f"{record['scene_id']}.png" render_scene(record, root / relative_image, image_size=image_size) record["image"] = str(relative_image) @@ -67,4 +70,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/data/dataset.py b/data/dataset.py index 25fcc28..061b314 100644 --- a/data/dataset.py +++ b/data/dataset.py @@ -9,15 +9,38 @@ import torch from PIL import Image from torch.utils.data import Dataset -from data.schema import OBJECT_TO_ID, PREDICATE_TO_ID, SUBJECT_TO_ID, Fact +from data.schema import ( + FACT_TYPE, + OBJECT_TO_ID, + PREDICATE_TO_ID, + RULE_TYPE, + SUBJECT_TO_ID, + Fact, + Rule, +) -def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int]: +def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int, int, int, int]: fact = Fact.from_dict(value) return ( + FACT_TYPE, SUBJECT_TO_ID[fact.subject], PREDICATE_TO_ID[fact.predicate], OBJECT_TO_ID[fact.object], + -1, + -1, + ) + + +def rule_to_ids(value: dict[str, str]) -> tuple[int, int, int, int, int, int]: + rule = Rule.from_dict(value) + return ( + RULE_TYPE, + SUBJECT_TO_ID[rule.variable], + PREDICATE_TO_ID[rule.antecedent_predicate], + OBJECT_TO_ID[rule.antecedent_object], + PREDICATE_TO_ID[rule.consequent_predicate], + OBJECT_TO_ID[rule.consequent_object], ) @@ -25,7 +48,9 @@ class EvidenceDataset(Dataset): def __init__(self, manifest: Path, image_size: int = 224): self.root = manifest.parent self.image_size = image_size - self.records = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()] + self.records = [ + json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines() + ] def __len__(self) -> int: return len(self.records) @@ -38,11 +63,13 @@ class EvidenceDataset(Dataset): return { "image": image_tensor, "rules_text": record["rules_text"], - "visual_facts": torch.tensor( + "visual_targets": torch.tensor( [fact_to_ids(value) for value in record["visual_facts"]], dtype=torch.long ), - "text_facts": torch.tensor( - [fact_to_ids(value) for value in record["text_facts"]], dtype=torch.long + "text_targets": torch.tensor( + [fact_to_ids(value) for value in record["text_facts"]] + + [rule_to_ids(value) for value in record["text_rules"]], + dtype=torch.long, ), "scene_id": record["scene_id"], } @@ -52,8 +79,7 @@ def collate_evidence(batch: list[dict[str, Any]]) -> dict[str, Any]: return { "images": torch.stack([item["image"] for item in batch]), "rules_text": [item["rules_text"] for item in batch], - "visual_facts": [item["visual_facts"] for item in batch], - "text_facts": [item["text_facts"] for item in batch], + "visual_targets": [item["visual_targets"] for item in batch], + "text_targets": [item["text_targets"] for item in batch], "scene_id": [item["scene_id"] for item in batch], } - diff --git a/data/generate_rules.py b/data/generate_rules.py index 5fba424..d293e0c 100644 --- a/data/generate_rules.py +++ b/data/generate_rules.py @@ -3,7 +3,7 @@ from __future__ import annotations import random from typing import Any -from data.schema import CLASSES, Fact +from data.schema import CLASSES, Fact, Rule def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[str, Any]: @@ -13,16 +13,20 @@ def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[ color_index = ("red", "blue", "green", "yellow").index(color) class_index = (color_index + (1 if held_out else 0)) % len(CLASSES) class_name = CLASSES[class_index] - rule = Fact(color, "implies_class", class_name) + text_fact = Fact(target["object_id"], "color", color) + rule = Rule("var_x", "color", color, "class", class_name) result = dict(record) result.update( { - "rules_text": f"Every {color} object belongs to class {class_name}.", - "text_facts": [rule.to_dict()], + "rules_text": ( + f"{target['object_id']} is {color}. " + f"Every {color} object belongs to class {class_name}." + ), + "text_facts": [text_fact.to_dict()], + "text_rules": [rule.to_dict()], "question": f"Which object belongs to class {class_name}?", "answer": target["object_id"], "combination": f"{color}->{class_name}", } ) return result - diff --git a/data/generate_scenes.py b/data/generate_scenes.py index 2327316..a5acf09 100644 --- a/data/generate_scenes.py +++ b/data/generate_scenes.py @@ -5,6 +5,13 @@ from typing import Any from data.schema import COLORS, OBJECT_IDS, RELATIONS, SHAPES, Fact, SceneObject +HELD_OUT_SHAPES = { + "red": "triangle", + "blue": "square", + "green": "star", + "yellow": "circle", +} + def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fact: relation = rng.choice(RELATIONS) @@ -23,12 +30,23 @@ def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fa return Fact(first.object_id, relation, second.object_id), first, second -def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, Any]: +def generate_scene( + scene_id: int, seed: int, num_objects: int = 2, held_out_composition: bool = False +) -> dict[str, Any]: if num_objects != 2: raise ValueError("The stage-one MVP currently supports exactly two objects.") rng = random.Random((seed * 1_000_003) + scene_id) colors = rng.sample(COLORS, k=num_objects) - shapes = rng.sample(SHAPES, k=num_objects) + if held_out_composition: + first_shape = HELD_OUT_SHAPES[colors[0]] + second_choices = [shape for shape in SHAPES if shape != first_shape] + shapes = [first_shape, rng.choice(second_choices)] + else: + shapes = [] + for color in colors: + choices = [shape for shape in SHAPES if shape != HELD_OUT_SHAPES[color]] + choices = [shape for shape in choices if shape not in shapes] + shapes.append(rng.choice(choices)) objects = [ SceneObject(OBJECT_IDS[index], colors[index], shapes[index], 0, 0) for index in range(num_objects) @@ -49,4 +67,3 @@ def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, "objects": [item.to_dict() for item in objects], "visual_facts": [fact.to_dict() for fact in facts], } - diff --git a/data/schema.py b/data/schema.py index 420a4d1..95e5ace 100644 --- a/data/schema.py +++ b/data/schema.py @@ -8,9 +8,15 @@ SHAPES = ("circle", "square", "triangle", "star") RELATIONS = ("left_of", "right_of", "above", "below") CLASSES = ("alpha", "beta", "gamma", "delta") OBJECT_IDS = ("obj1", "obj2", "obj3") +VARIABLES = ("var_x",) -PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "implies_class") -ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES +PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "class") +ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES + VARIABLES + +NULL_TYPE = 0 +FACT_TYPE = 1 +RULE_TYPE = 2 +NUM_STATE_TYPES = 3 SUBJECT_TO_ID = {value: index for index, value in enumerate(ENTITIES)} PREDICATE_TO_ID = {value: index for index, value in enumerate(PREDICATES)} @@ -31,6 +37,22 @@ class Fact: return cls(**value) +@dataclass(frozen=True) +class Rule: + variable: str + antecedent_predicate: str + antecedent_object: str + consequent_predicate: str + consequent_object: str + + def to_dict(self) -> dict[str, str]: + return asdict(self) + + @classmethod + def from_dict(cls, value: dict[str, str]) -> "Rule": + return cls(**value) + + @dataclass(frozen=True) class SceneObject: object_id: str @@ -41,4 +63,3 @@ class SceneObject: def to_dict(self) -> dict[str, Any]: return asdict(self) - diff --git a/eval/benchmark_extractor.py b/eval/benchmark_extractor.py new file mode 100644 index 0000000..fc2be75 --- /dev/null +++ b/eval/benchmark_extractor.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import torch +import torch.nn.functional as F +import yaml +from torch.utils.data import DataLoader + +from data.dataset import EvidenceDataset, collate_evidence +from data.schema import FACT_TYPE +from eval.evaluate_slots import evaluate +from losses.alignment_loss import matched_states +from losses.matching_loss import hungarian_matching_loss +from models.evidence_model import EvidenceExtractorModel + + +def recall_at_one( + queries: torch.Tensor, + query_labels: torch.Tensor, + candidates: torch.Tensor, + candidate_labels: torch.Tensor, +) -> float: + if not len(queries) or not len(candidates): + return 0.0 + similarity = F.normalize(queries, dim=-1) @ F.normalize(candidates, dim=-1).T + retrieved = candidate_labels[similarity.argmax(dim=1)] + return float((retrieved == query_labels).all(-1).float().mean()) + + +def hard_negative_recall_at_one( + queries: torch.Tensor, + query_labels: torch.Tensor, + candidates: torch.Tensor, + candidate_labels: torch.Tensor, +) -> float: + scores = [] + normalized_candidates = F.normalize(candidates, dim=-1) + for query, label in zip(F.normalize(queries, dim=-1), query_labels): + exact = (candidate_labels == label).all(-1) + field_distance = (candidate_labels[:, 1:4] != label[1:4]).sum(-1) + hard = ( + (candidate_labels[:, 0] == FACT_TYPE) + & (field_distance == 1) + & (candidate_labels[:, 4:] == label[4:]).all(-1) + ) + pool = exact | hard + if exact.any() and hard.any(): + selected_labels = candidate_labels[pool] + nearest = (query @ normalized_candidates[pool].T).argmax() + scores.append(float((selected_labels[nearest] == label).all())) + return sum(scores) / max(1, len(scores)) + + +@torch.no_grad() +def collect_fact_states( + model: EvidenceExtractorModel, loader: DataLoader, device: torch.device +) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + visual_vectors = [] + visual_labels = [] + text_vectors = [] + text_labels = [] + model.eval() + for batch in loader: + outputs = model(batch["images"].to(device), batch["rules_text"]) + visual_match = hungarian_matching_loss(outputs["visual"], batch["visual_targets"]) + text_match = hungarian_matching_loss(outputs["text"], batch["text_targets"]) + visual, visual_target = matched_states( + outputs["visual"]["states"], + batch["visual_targets"], + visual_match.assignments, + FACT_TYPE, + ) + text, text_target = matched_states( + outputs["text"]["states"], + batch["text_targets"], + text_match.assignments, + FACT_TYPE, + ) + visual_vectors.append(visual.cpu()) + visual_labels.append(visual_target.cpu()) + text_vectors.append(text.cpu()) + text_labels.append(text_target.cpu()) + return ( + torch.cat(visual_vectors), + torch.cat(visual_labels), + torch.cat(text_vectors), + torch.cat(text_labels), + ) + + +def make_loader(config: dict[str, Any], split: str) -> DataLoader: + dataset = EvidenceDataset( + Path(config["data"]["root"]) / f"{split}.jsonl", config["data"]["image_size"] + ) + return DataLoader( + dataset, + batch_size=config["training"]["batch_size"], + collate_fn=collate_evidence, + ) + + +@torch.no_grad() +def benchmark( + model: EvidenceExtractorModel, config: dict[str, Any], device: torch.device +) -> dict[str, Any]: + iid_loader = make_loader(config, "test_id") + iid = evaluate(model, iid_loader, device) + visual, visual_labels, text, text_labels = collect_fact_states(model, iid_loader, device) + ood = evaluate(model, make_loader(config, "test_ood"), device) + metrics = { + "text_fact_rule_f1": iid["text_all"]["f1"], + "visual_fact_f1": iid["visual_fact"]["f1"], + "cross_modal_recall_at_1": recall_at_one( + text, text_labels, visual, visual_labels + ), + "hard_negative_recall_at_1": hard_negative_recall_at_one( + text, text_labels, visual, visual_labels + ), + "ood_fact_f1": ood["visual_fact"]["f1"], + "null_slot_accuracy": iid["null_slot_accuracy"], + } + thresholds = config["evaluation"]["gates"] + gates = { + name: {"value": value, "threshold": thresholds[name], "passed": value >= thresholds[name]} + for name, value in metrics.items() + } + return { + "metrics": metrics, + "gates": gates, + "passed": all(gate["passed"] for gate in gates.values()), + "details": {"test_id": iid, "test_ood": ood}, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml")) + parser.add_argument("--checkpoint", type=Path, required=True) + parser.add_argument("--backbone", choices=("tiny", "clip"), default=None) + parser.add_argument("--output", type=Path, default=None) + parser.add_argument("--enforce-gates", action="store_true") + args = parser.parse_args() + config = yaml.safe_load(args.config.read_text(encoding="utf-8")) + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + model = EvidenceExtractorModel(config, backbone_name=args.backbone).to(device) + checkpoint = torch.load(args.checkpoint, map_location=device, weights_only=True) + model.load_state_dict(checkpoint["model"]) + result = benchmark(model, config, device) + rendered = json.dumps(result, indent=2) + print(rendered) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(rendered + "\n", encoding="utf-8") + if args.enforce_gates and not result["passed"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/eval/evaluate_slots.py b/eval/evaluate_slots.py index c647c9e..ba7125a 100644 --- a/eval/evaluate_slots.py +++ b/eval/evaluate_slots.py @@ -10,29 +10,72 @@ import yaml from torch.utils.data import DataLoader from data.dataset import EvidenceDataset, collate_evidence +from data.schema import FACT_TYPE, NULL_TYPE, RULE_TYPE +from losses.matching_loss import hungarian_matching_loss from models.evidence_model import EvidenceExtractorModel -def slot_counts(predictions: dict[str, torch.Tensor], targets: list[torch.Tensor]) -> dict[str, int]: - subjects = predictions["subject_logits"].argmax(-1).cpu() - predicates = predictions["predicate_logits"].argmax(-1).cpu() - objects = predictions["object_logits"].argmax(-1).cpu() - valid = predictions["valid_logits"].argmax(-1).cpu().bool() - counts = {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0} - for batch_index, facts in enumerate(targets): - predicted = { - (int(subjects[batch_index, slot]), int(predicates[batch_index, slot]), int(objects[batch_index, slot])) - for slot in range(subjects.shape[1]) - if valid[batch_index, slot] - } - expected = {tuple(map(int, row.tolist())) for row in facts} +COUNT_KEYS = ("tp", "fp", "fn", "component_correct", "component_total") + + +def empty_counts() -> dict[str, int]: + return {key: 0 for key in COUNT_KEYS} + + +def decode_rows(predictions: dict[str, torch.Tensor], batch_index: int) -> list[tuple[int, ...]]: + types = predictions["type_logits"].argmax(-1)[batch_index].cpu() + subjects = predictions["subject_logits"].argmax(-1)[batch_index].cpu() + predicates = predictions["predicate_logits"].argmax(-1)[batch_index].cpu() + objects = predictions["object_logits"].argmax(-1)[batch_index].cpu() + consequent_predicates = predictions["consequent_predicate_logits"].argmax(-1)[ + batch_index + ].cpu() + consequent_objects = predictions["consequent_object_logits"].argmax(-1)[batch_index].cpu() + rows = [] + for slot, state_type in enumerate(types.tolist()): + if state_type == NULL_TYPE: + continue + consequence = ( + (int(consequent_predicates[slot]), int(consequent_objects[slot])) + if state_type == RULE_TYPE + else (-1, -1) + ) + rows.append( + ( + state_type, + int(subjects[slot]), + int(predicates[slot]), + int(objects[slot]), + *consequence, + ) + ) + return rows + + +def slot_counts( + predictions: dict[str, torch.Tensor], + targets: list[torch.Tensor], + state_type: int | None = None, +) -> dict[str, int]: + counts = empty_counts() + for batch_index, target_tensor in enumerate(targets): + predicted = set(decode_rows(predictions, batch_index)) + expected = {tuple(map(int, row.tolist())) for row in target_tensor} + if state_type is not None: + predicted = {row for row in predicted if row[0] == state_type} + expected = {row for row in expected if row[0] == state_type} counts["tp"] += len(predicted & expected) counts["fp"] += len(predicted - expected) counts["fn"] += len(expected - predicted) - for triple in predicted: - best = max((sum(a == b for a, b in zip(triple, target)) for target in expected), default=0) + for row in predicted: + compatible = [target for target in expected if target[0] == row[0]] + width = 6 if row[0] == RULE_TYPE else 4 + best = max( + (sum(a == b for a, b in zip(row[:width], target[:width])) for target in compatible), + default=0, + ) counts["component_correct"] += best - counts["component_total"] += 3 + counts["component_total"] += width return counts @@ -43,26 +86,61 @@ def summarize(counts: dict[str, int]) -> dict[str, float]: "precision": precision, "recall": recall, "f1": 2 * precision * recall / max(1e-12, precision + recall), - "component_accuracy": counts["component_correct"] / max(1, counts["component_total"]), + "component_accuracy": counts["component_correct"] + / max(1, counts["component_total"]), } +def null_slot_counts( + predictions: dict[str, torch.Tensor], targets: list[torch.Tensor] +) -> tuple[int, int]: + matching = hungarian_matching_loss(predictions, targets) + predicted_types = predictions["type_logits"].argmax(-1) + correct = 0 + total = predicted_types.numel() + for batch_index, (slots, target_indices) in enumerate(matching.assignments): + expected = torch.full_like(predicted_types[batch_index], NULL_TYPE) + if len(slots): + expected[slots] = targets[batch_index].to(expected.device)[target_indices, 0] + correct += int((predicted_types[batch_index] == expected).sum()) + return correct, total + + @torch.no_grad() def evaluate( model: EvidenceExtractorModel, loader: DataLoader, device: torch.device ) -> dict[str, Any]: model.eval() totals = { - "visual": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0}, - "text": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0}, + "visual_fact": empty_counts(), + "text_all": empty_counts(), + "text_fact": empty_counts(), + "text_rule": empty_counts(), } + null_correct = 0 + null_total = 0 for batch in loader: outputs = model(batch["images"].to(device), batch["rules_text"]) - for modality, target_name in (("visual", "visual_facts"), ("text", "text_facts")): - counts = slot_counts(outputs[modality], batch[target_name]) + requests = ( + ("visual_fact", "visual", "visual_targets", FACT_TYPE), + ("text_all", "text", "text_targets", None), + ("text_fact", "text", "text_targets", FACT_TYPE), + ("text_rule", "text", "text_targets", RULE_TYPE), + ) + for name, modality, target_name, state_type in requests: + counts = slot_counts(outputs[modality], batch[target_name], state_type) for key, value in counts.items(): - totals[modality][key] += value - return {modality: summarize(value) for modality, value in totals.items()} + totals[name][key] += value + for modality, target_name in ( + ("visual", "visual_targets"), + ("text", "text_targets"), + ): + correct, count = null_slot_counts(outputs[modality], batch[target_name]) + null_correct += correct + null_total += count + result = {name: summarize(counts) for name, counts in totals.items()} + result["null_slot_accuracy"] = null_correct / max(1, null_total) + return result def main() -> None: @@ -91,4 +169,3 @@ def main() -> None: if __name__ == "__main__": main() - diff --git a/losses/alignment_loss.py b/losses/alignment_loss.py index 4e82087..d9c4ed0 100644 --- a/losses/alignment_loss.py +++ b/losses/alignment_loss.py @@ -3,34 +3,72 @@ from __future__ import annotations import torch import torch.nn.functional as F +from data.schema import FACT_TYPE -def supervised_alignment_loss( + +def matched_states( states: torch.Tensor, targets: list[torch.Tensor], assignments: list[tuple[torch.Tensor, torch.Tensor]], - temperature: float = 0.1, -) -> torch.Tensor: + state_type: int | None = None, +) -> tuple[torch.Tensor, torch.Tensor]: vectors = [] labels = [] - for batch_index, (slot_index, fact_index) in enumerate(assignments): - if len(slot_index) == 0: + for batch_index, (slot_index, target_index) in enumerate(assignments): + if not len(slot_index): continue - vectors.append(states[batch_index, slot_index]) - labels.append(targets[batch_index].to(states.device)[fact_index]) + selected_targets = targets[batch_index].to(states.device)[target_index] + selected_states = states[batch_index, slot_index] + if state_type is not None: + keep = selected_targets[:, 0] == state_type + selected_targets = selected_targets[keep] + selected_states = selected_states[keep] + if len(selected_states): + vectors.append(selected_states) + labels.append(selected_targets) if not vectors: - return states.sum() * 0.0 - vectors_tensor = F.normalize(torch.cat(vectors), dim=-1) - labels_tensor = torch.cat(labels) - if len(vectors_tensor) < 2: - return vectors_tensor.sum() * 0.0 - same = (labels_tensor[:, None, :] == labels_tensor[None, :, :]).all(-1) - same.fill_diagonal_(False) - logits = vectors_tensor @ vectors_tensor.T / temperature - logits.fill_diagonal_(-torch.inf) - log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True) - positive_count = same.sum(1) - usable = positive_count > 0 - if not usable.any(): - return vectors_tensor.sum() * 0.0 - return -((log_prob.masked_fill(~same, 0.0).sum(1) / positive_count.clamp_min(1))[usable]).mean() + return states.new_empty((0, states.shape[-1])), torch.empty( + (0, 6), dtype=torch.long, device=states.device + ) + return torch.cat(vectors), torch.cat(labels) + +def _directional_multi_positive_loss( + query: torch.Tensor, + candidates: torch.Tensor, + positives: torch.Tensor, + temperature: float, +) -> torch.Tensor: + usable = positives.any(dim=1) + if not usable.any(): + return query.sum() * 0.0 + logits = query @ candidates.T / temperature + log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True) + positive_count = positives.sum(1).clamp_min(1) + per_query = -(log_prob.masked_fill(~positives, 0.0).sum(1) / positive_count) + return per_query[usable].mean() + + +def cross_modal_alignment_loss( + visual_states: torch.Tensor, + visual_targets: list[torch.Tensor], + visual_assignments: list[tuple[torch.Tensor, torch.Tensor]], + text_states: torch.Tensor, + text_targets: list[torch.Tensor], + text_assignments: list[tuple[torch.Tensor, torch.Tensor]], + temperature: float = 0.1, +) -> torch.Tensor: + """Align identical FACT states across modalities; RULE states remain text-only.""" + visual, visual_labels = matched_states( + visual_states, visual_targets, visual_assignments, FACT_TYPE + ) + text, text_labels = matched_states(text_states, text_targets, text_assignments, FACT_TYPE) + if not len(visual) or not len(text): + return visual_states.sum() * 0.0 + visual = F.normalize(visual, dim=-1) + text = F.normalize(text, dim=-1) + positives = (visual_labels[:, None, :] == text_labels[None, :, :]).all(-1) + return 0.5 * ( + _directional_multi_positive_loss(visual, text, positives, temperature) + + _directional_multi_positive_loss(text, visual, positives.T, temperature) + ) diff --git a/losses/matching_loss.py b/losses/matching_loss.py index 322f7c0..fa71f12 100644 --- a/losses/matching_loss.py +++ b/losses/matching_loss.py @@ -6,6 +6,8 @@ from itertools import permutations import torch import torch.nn.functional as F +from data.schema import NULL_TYPE, RULE_TYPE + @dataclass class MatchingOutput: @@ -32,12 +34,14 @@ def _small_linear_assignment(cost: torch.Tensor) -> tuple[list[int], list[int]]: def hungarian_matching_loss( predictions: dict[str, torch.Tensor], targets: list[torch.Tensor], - valid_weight: float = 0.5, + type_weight: float = 0.5, ) -> MatchingOutput: subject_logp = predictions["subject_logits"].log_softmax(-1) predicate_logp = predictions["predicate_logits"].log_softmax(-1) object_logp = predictions["object_logits"].log_softmax(-1) - valid_logits = predictions["valid_logits"] + type_logp = predictions["type_logits"].log_softmax(-1) + consequent_predicate_logp = predictions["consequent_predicate_logits"].log_softmax(-1) + consequent_object_logp = predictions["consequent_object_logits"].log_softmax(-1) device = subject_logp.device total = subject_logp.sum() * 0.0 assignments = [] @@ -47,29 +51,49 @@ def hungarian_matching_loss( num_slots = subject_logp.shape[1] if len(facts): cost = -( - subject_logp[batch_index, :, facts[:, 0]] - + predicate_logp[batch_index, :, facts[:, 1]] - + object_logp[batch_index, :, facts[:, 2]] + type_logp[batch_index, :, facts[:, 0]] + + subject_logp[batch_index, :, facts[:, 1]] + + predicate_logp[batch_index, :, facts[:, 2]] + + object_logp[batch_index, :, facts[:, 3]] ) + rule_columns = facts[:, 0] == RULE_TYPE + if rule_columns.any(): + cost[:, rule_columns] -= ( + consequent_predicate_logp[batch_index, :, facts[rule_columns, 4]] + + consequent_object_logp[batch_index, :, facts[rule_columns, 5]] + ) slot_index, fact_index = _small_linear_assignment(cost) slots = torch.as_tensor(slot_index, dtype=torch.long, device=device) matched = torch.as_tensor(fact_index, dtype=torch.long, device=device) selected = facts[matched] total = total + F.cross_entropy( - predictions["subject_logits"][batch_index, slots], selected[:, 0] + predictions["subject_logits"][batch_index, slots], selected[:, 1] ) total = total + F.cross_entropy( - predictions["predicate_logits"][batch_index, slots], selected[:, 1] + predictions["predicate_logits"][batch_index, slots], selected[:, 2] ) total = total + F.cross_entropy( - predictions["object_logits"][batch_index, slots], selected[:, 2] + predictions["object_logits"][batch_index, slots], selected[:, 3] ) + selected_rules = selected[:, 0] == RULE_TYPE + if selected_rules.any(): + total = total + F.cross_entropy( + predictions["consequent_predicate_logits"][batch_index, slots[selected_rules]], + selected[selected_rules, 4], + ) + total = total + F.cross_entropy( + predictions["consequent_object_logits"][batch_index, slots[selected_rules]], + selected[selected_rules, 5], + ) else: slots = torch.empty(0, dtype=torch.long, device=device) matched = torch.empty(0, dtype=torch.long, device=device) - valid_target = torch.zeros(num_slots, dtype=torch.long, device=device) - valid_target[slots] = 1 - total = total + valid_weight * F.cross_entropy(valid_logits[batch_index], valid_target) + type_target = torch.full((num_slots,), NULL_TYPE, dtype=torch.long, device=device) + if len(slots): + type_target[slots] = facts[matched, 0] + total = total + type_weight * F.cross_entropy( + predictions["type_logits"][batch_index], type_target + ) assignments.append((slots, matched)) return MatchingOutput(total / len(targets), assignments) diff --git a/losses/prototype_loss.py b/losses/prototype_loss.py new file mode 100644 index 0000000..6e71d47 --- /dev/null +++ b/losses/prototype_loss.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +import torch +import torch.nn.functional as F + +from models.fact_prototype import CanonicalFactPrototype + + +def canonical_prototype_loss( + states: torch.Tensor, + targets: list[torch.Tensor], + assignments: list[tuple[torch.Tensor, torch.Tensor]], + prototype_encoder: CanonicalFactPrototype, +) -> torch.Tensor: + vectors = [] + labels = [] + for batch_index, (slot_index, target_index) in enumerate(assignments): + if len(slot_index): + vectors.append(states[batch_index, slot_index]) + labels.append(targets[batch_index].to(states.device)[target_index]) + if not vectors: + return states.sum() * 0.0 + state_tensor = F.normalize(torch.cat(vectors), dim=-1) + prototype_tensor = prototype_encoder(torch.cat(labels)) + return (1.0 - (state_tensor * prototype_tensor).sum(-1)).mean() diff --git a/models/evidence_model.py b/models/evidence_model.py index 577582a..b4a8943 100644 --- a/models/evidence_model.py +++ b/models/evidence_model.py @@ -5,7 +5,8 @@ from typing import Any import torch from torch import nn -from data.schema import ENTITIES, PREDICATES +from data.schema import ENTITIES, NUM_STATE_TYPES, PREDICATES +from models.fact_prototype import CanonicalFactPrototype from models.frozen_backbone import build_backbone from models.slot_extractor import SlotExtractor @@ -25,6 +26,7 @@ class EvidenceExtractorModel(nn.Module): "num_subjects": len(ENTITIES), "num_predicates": len(PREDICATES), "num_objects": len(ENTITIES), + "num_state_types": NUM_STATE_TYPES, } self.visual_extractor = SlotExtractor( input_dim=self.backbone.vision_dim, @@ -36,6 +38,16 @@ class EvidenceExtractorModel(nn.Module): num_slots=model_config["num_text_slots"], **common, ) + prototype_config = config.get("prototype", {}) + self.prototype_encoder = None + if prototype_config.get("enabled", False): + self.prototype_encoder = CanonicalFactPrototype( + state_dim=model_config["state_dim"], + num_state_types=NUM_STATE_TYPES, + num_subjects=len(ENTITIES), + num_predicates=len(PREDICATES), + num_objects=len(ENTITIES), + ) def forward(self, images: torch.Tensor, rules_text: list[str]) -> dict[str, Any]: visual_tokens, visual_mask = self.backbone.encode_image(images) @@ -44,4 +56,3 @@ class EvidenceExtractorModel(nn.Module): "visual": self.visual_extractor(visual_tokens, visual_mask), "text": self.text_extractor(text_tokens, text_mask), } - diff --git a/models/fact_prototype.py b/models/fact_prototype.py new file mode 100644 index 0000000..55ca55c --- /dev/null +++ b/models/fact_prototype.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import torch +import torch.nn.functional as F +from torch import nn + + +class CanonicalFactPrototype(nn.Module): + """Compose a typed symbolic state into a learned canonical prototype.""" + + def __init__( + self, + state_dim: int, + num_state_types: int, + num_subjects: int, + num_predicates: int, + num_objects: int, + ): + super().__init__() + self.type_embedding = nn.Embedding(num_state_types, state_dim) + self.subject_embedding = nn.Embedding(num_subjects + 1, state_dim, padding_idx=0) + self.predicate_embedding = nn.Embedding(num_predicates + 1, state_dim, padding_idx=0) + self.object_embedding = nn.Embedding(num_objects + 1, state_dim, padding_idx=0) + self.consequent_predicate_embedding = nn.Embedding( + num_predicates + 1, state_dim, padding_idx=0 + ) + self.consequent_object_embedding = nn.Embedding( + num_objects + 1, state_dim, padding_idx=0 + ) + self.projection = nn.Sequential( + nn.LayerNorm(state_dim), + nn.Linear(state_dim, state_dim), + nn.GELU(), + nn.Linear(state_dim, state_dim), + ) + + def forward(self, targets: torch.Tensor) -> torch.Tensor: + shifted = targets[:, 1:].clamp_min(-1) + 1 + value = self.type_embedding(targets[:, 0]) + value = value + self.subject_embedding(shifted[:, 0]) + value = value + self.predicate_embedding(shifted[:, 1]) + value = value + self.object_embedding(shifted[:, 2]) + value = value + self.consequent_predicate_embedding(shifted[:, 3]) + value = value + self.consequent_object_embedding(shifted[:, 4]) + return F.normalize(self.projection(value), dim=-1) diff --git a/models/slot_extractor.py b/models/slot_extractor.py index 221f568..9ef2ce5 100644 --- a/models/slot_extractor.py +++ b/models/slot_extractor.py @@ -41,6 +41,7 @@ class SlotExtractor(nn.Module): num_subjects: int, num_predicates: int, num_objects: int, + num_state_types: int, ): super().__init__() self.input_projection = nn.Linear(input_dim, state_dim) @@ -51,7 +52,9 @@ class SlotExtractor(nn.Module): self.subject_head = nn.Linear(state_dim, num_subjects) self.predicate_head = nn.Linear(state_dim, num_predicates) self.object_head = nn.Linear(state_dim, num_objects) - self.valid_head = nn.Linear(state_dim, 2) + self.type_head = nn.Linear(state_dim, num_state_types) + self.consequent_predicate_head = nn.Linear(state_dim, num_predicates) + self.consequent_object_head = nn.Linear(state_dim, num_objects) def forward(self, tokens: torch.Tensor, token_mask: torch.Tensor) -> dict[str, torch.Tensor]: tokens = self.input_projection(tokens) @@ -63,6 +66,7 @@ class SlotExtractor(nn.Module): "subject_logits": self.subject_head(slots), "predicate_logits": self.predicate_head(slots), "object_logits": self.object_head(slots), - "valid_logits": self.valid_head(slots), + "type_logits": self.type_head(slots), + "consequent_predicate_logits": self.consequent_predicate_head(slots), + "consequent_object_logits": self.consequent_object_head(slots), } - diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py new file mode 100644 index 0000000..779f228 --- /dev/null +++ b/tests/test_benchmark.py @@ -0,0 +1,17 @@ +import torch + +from data.schema import FACT_TYPE +from eval.benchmark_extractor import hard_negative_recall_at_one, recall_at_one + + +def test_retrieval_metrics_distinguish_hard_negative(): + labels = torch.tensor( + [ + [FACT_TYPE, 0, 0, 4, -1, -1], + [FACT_TYPE, 0, 0, 5, -1, -1], + ] + ) + candidates = torch.tensor([[1.0, 0.0], [0.0, 1.0]]) + queries = torch.tensor([[0.9, 0.1], [0.1, 0.9]]) + assert recall_at_one(queries, labels, candidates, labels) == 1.0 + assert hard_negative_recall_at_one(queries, labels, candidates, labels) == 1.0 diff --git a/tests/test_data.py b/tests/test_data.py index e1e97a9..c6acf0c 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -1,5 +1,5 @@ from data.generate_rules import add_rule -from data.generate_scenes import generate_scene +from data.generate_scenes import HELD_OUT_SHAPES, generate_scene def test_scene_generation_is_deterministic(): @@ -15,4 +15,11 @@ def test_held_out_rule_changes_combination(): held_out = add_rule(scene, seed=17, held_out=True) assert train["combination"] != held_out["combination"] assert train["answer"] == held_out["answer"] + assert len(train["text_facts"]) == 1 + assert len(train["text_rules"]) == 1 + +def test_ood_scene_uses_held_out_visual_composition(): + scene = generate_scene(5, seed=17, held_out_composition=True) + first = scene["objects"][0] + assert first["shape"] == HELD_OUT_SHAPES[first["color"]] diff --git a/tests/test_model.py b/tests/test_model.py index 6b8ec2c..4ca5a51 100644 --- a/tests/test_model.py +++ b/tests/test_model.py @@ -1,6 +1,10 @@ import torch +from data.schema import FACT_TYPE, RULE_TYPE +from losses.alignment_loss import cross_modal_alignment_loss from losses.matching_loss import hungarian_matching_loss +from losses.prototype_loss import canonical_prototype_loss +from models.fact_prototype import CanonicalFactPrototype from models.slot_extractor import SlotExtractor @@ -15,13 +19,37 @@ def test_slot_extractor_and_matching_loss_backward(): num_subjects=12, num_predicates=7, num_objects=12, + num_state_types=3, ) tokens = torch.randn(2, 10, 32) mask = torch.ones(2, 10, dtype=torch.bool) predictions = extractor(tokens, mask) - targets = [torch.tensor([[0, 0, 4], [1, 1, 5]]), torch.tensor([[1, 0, 6]])] + targets = [ + torch.tensor( + [ + [FACT_TYPE, 0, 0, 4, -1, -1], + [RULE_TYPE, 1, 1, 5, 6, 7], + ] + ), + torch.tensor([[FACT_TYPE, 1, 0, 6, -1, -1]]), + ] result = hungarian_matching_loss(predictions, targets) assert torch.isfinite(result.loss) result.loss.backward() assert extractor.slot_queries.grad is not None + +def test_cross_modal_alignment_and_prototype_losses(): + assignments = [(torch.tensor([0]), torch.tensor([0]))] + targets = [torch.tensor([[FACT_TYPE, 0, 0, 4, -1, -1]])] + visual = torch.randn(1, 2, 16, requires_grad=True) + text = torch.randn(1, 2, 16, requires_grad=True) + alignment = cross_modal_alignment_loss( + visual, targets, assignments, text, targets, assignments + ) + prototype = CanonicalFactPrototype(16, 3, 12, 7, 12) + prototype_value = canonical_prototype_loss(visual, targets, assignments, prototype) + loss = alignment + prototype_value + assert torch.isfinite(loss) + loss.backward() + assert visual.grad is not None diff --git a/train_extractor.py b/train_extractor.py index d57f8a9..a09efa7 100644 --- a/train_extractor.py +++ b/train_extractor.py @@ -13,8 +13,9 @@ from torch.utils.data import DataLoader from data.dataset import EvidenceDataset, collate_evidence from eval.evaluate_slots import evaluate -from losses.alignment_loss import supervised_alignment_loss +from losses.alignment_loss import cross_modal_alignment_loss from losses.matching_loss import hungarian_matching_loss +from losses.prototype_loss import canonical_prototype_loss from models.evidence_model import EvidenceExtractorModel @@ -63,23 +64,40 @@ def train(config: dict[str, Any], backbone_name: str, epochs: int | None = None) optimizer.zero_grad(set_to_none=True) outputs = model(batch["images"].to(device), batch["rules_text"]) visual = hungarian_matching_loss( - outputs["visual"], batch["visual_facts"], config["training"]["valid_weight"] + outputs["visual"], batch["visual_targets"], config["training"]["type_weight"] ) text = hungarian_matching_loss( - outputs["text"], batch["text_facts"], config["training"]["valid_weight"] + outputs["text"], batch["text_targets"], config["training"]["type_weight"] ) - alignment = supervised_alignment_loss( - outputs["visual"]["states"], batch["visual_facts"], visual.assignments - ) + supervised_alignment_loss( - outputs["text"]["states"], batch["text_facts"], text.assignments + alignment = cross_modal_alignment_loss( + outputs["visual"]["states"], + batch["visual_targets"], + visual.assignments, + outputs["text"]["states"], + batch["text_targets"], + text.assignments, + config["training"]["contrastive_temperature"], ) loss = visual.loss + text.loss + config["training"]["alignment_weight"] * alignment + if model.prototype_encoder is not None: + prototype_loss = canonical_prototype_loss( + outputs["visual"]["states"], + batch["visual_targets"], + visual.assignments, + model.prototype_encoder, + ) + canonical_prototype_loss( + outputs["text"]["states"], + batch["text_targets"], + text.assignments, + model.prototype_encoder, + ) + loss = loss + config["prototype"]["weight"] * prototype_loss loss.backward() torch.nn.utils.clip_grad_norm_(trainable, 1.0) optimizer.step() running_loss += float(loss.detach()) metrics = evaluate(model, validation_loader, device) - score = metrics["visual"]["f1"] + metrics["text"]["f1"] + score = metrics["visual_fact"]["f1"] + metrics["text_all"]["f1"] summary = { "epoch": epoch, "loss": running_loss / max(1, len(train_loader)), @@ -107,4 +125,3 @@ def main() -> None: if __name__ == "__main__": main() -