Build stage-one evidence extractor MVP
CI / test (push) Failing after 11m13s

This commit is contained in:
2026-10-01 22:16:29 +08:00
commit d30892168e
26 changed files with 2250 additions and 0 deletions
+28
View File
@@ -0,0 +1,28 @@
name: CI
on:
push:
pull_request:
jobs:
test:
runs-on: ces-ci
env:
CUDA_VISIBLE_DEVICES: ""
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- name: Reuse runner PyTorch and install lightweight dependencies
run: |
uv venv --system-site-packages --python python .venv-ci
uv pip install --python .venv-ci/bin/python numpy pillow pyyaml scipy pytest ruff
- name: Lint
run: .venv-ci/bin/ruff check .
- name: Unit tests
run: .venv-ci/bin/pytest
- name: End-to-end CPU smoke test
run: |
.venv-ci/bin/python -m data.build_splits --config configs/mvp.yaml --small
.venv-ci/bin/python train_extractor.py --config configs/mvp.yaml --backbone tiny --epochs 1
+27
View File
@@ -0,0 +1,27 @@
name: GPU smoke
on:
workflow_dispatch:
jobs:
extractor:
runs-on: gpu
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: astral-sh/setup-uv@v6
with:
enable-cache: true
- name: Verify CUDA
run: nvidia-smi
- name: Reuse runner PyTorch and install lightweight dependencies
run: |
uv venv --system-site-packages --python python .venv-gpu
uv pip install --python .venv-gpu/bin/python numpy pillow pyyaml scipy transformers pytest ruff
- name: Verify PyTorch sees the GPU
run: |
.venv-gpu/bin/python -c "import torch; assert torch.cuda.is_available(); print(torch.__version__, torch.version.cuda, torch.cuda.get_device_name(), torch.cuda.get_device_capability())"
- name: Build smoke dataset
run: .venv-gpu/bin/python -m data.build_splits --config configs/mvp.yaml --small
- name: Train extractor
run: .venv-gpu/bin/python train_extractor.py --config configs/mvp.yaml --backbone clip --epochs 1
+10
View File
@@ -0,0 +1,10 @@
.venv/
__pycache__/
*.py[cod]
.pytest_cache/
.ruff_cache/
artifacts/
checkpoints/
datasets/
wandb/
+46
View File
@@ -0,0 +1,46 @@
# Composable Evidence States Research
This repository contains the first-stage MVP for learning fact-level evidence slots from
synthetic images and natural-language rules. The Composer is intentionally deferred until
the extractor passes its fact-recovery checks.
## Current scope
- Generate deterministic scenes containing colored geometric objects and spatial relations.
- Render scenes to PNG while preserving machine-readable fact triples.
- Generate textual implication rules as normalized triples.
- Build IID and held-out-combination splits without leaking held-out rule pairs into training.
- Train visual and textual slot extractors with Hungarian set matching.
- Measure triple exact match, component accuracy, precision, recall, and F1.
- Run CPU smoke tests with a tiny frozen backbone or GPU experiments with frozen CLIP.
## Quick start
```bash
uv sync --extra dev
uv run python -m data.build_splits --config configs/mvp.yaml --small
uv run python train_extractor.py --config configs/mvp.yaml --backbone tiny --epochs 2
uv run python -m eval.evaluate_slots --config configs/mvp.yaml \
--checkpoint artifacts/tiny/best.pt --backbone tiny
uv run pytest
```
For the first real GPU experiment:
```bash
uv sync --extra dev --extra hf
uv run python -m data.build_splits --config configs/mvp.yaml
uv run python train_extractor.py --config configs/mvp.yaml --backbone clip
```
The CLIP path uses `openai/clip-vit-base-patch32` by default. It is a deliberately small
backbone for validating the extractor. A Qwen-VL adapter and the Composer belong to the next
milestone after the evidence slots are shown to recover atomic facts reliably.
## Success gate before Composer work
- Text-rule triple F1 >= 0.95.
- Visual-fact triple F1 >= 0.90 on the IID test split.
- No severe slot collapse: predicted valid slots track the true fact count.
- Held-out rule-combination performance is reported separately from IID performance.
+31
View File
@@ -0,0 +1,31 @@
seed: 17
data:
root: datasets/mvp
image_size: 224
train_size: 30000
validation_size: 3000
test_id_size: 3000
test_ood_size: 3000
num_objects: 2
model:
backbone: tiny
clip_model: openai/clip-vit-base-patch32
state_dim: 256
num_visual_slots: 6
num_text_slots: 3
extractor_layers: 2
num_heads: 8
dropout: 0.1
training:
batch_size: 32
learning_rate: 0.0003
weight_decay: 0.01
epochs: 10
num_workers: 0
valid_weight: 0.5
alignment_weight: 0.05
output_dir: artifacts/tiny
+2
View File
@@ -0,0 +1,2 @@
"""Synthetic data generation and loading."""
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
import yaml
from data.generate_rules import add_rule
from data.generate_scenes import generate_scene
from data.render_scene import render_scene
def _write_split(
name: str,
size: int,
start: int,
root: Path,
seed: int,
image_size: int,
held_out: bool,
) -> None:
path = root / f"{name}.jsonl"
with path.open("w", encoding="utf-8") as handle:
for scene_number in range(start, start + size):
record = add_rule(generate_scene(scene_number, seed), seed, held_out=held_out)
relative_image = Path("images") / f"{record['scene_id']}.png"
render_scene(record, root / relative_image, image_size=image_size)
record["image"] = str(relative_image)
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def build(config_path: Path, small: bool = False) -> Path:
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
root = Path(config["data"]["root"])
root.mkdir(parents=True, exist_ok=True)
sizes = {
"train": config["data"]["train_size"],
"validation": config["data"]["validation_size"],
"test_id": config["data"]["test_id_size"],
"test_ood": config["data"]["test_ood_size"],
}
if small:
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
cursor = 0
for split, size in sizes.items():
_write_split(
split,
size,
cursor,
root,
config["seed"],
config["data"]["image_size"],
held_out=split == "test_ood",
)
cursor += size
return root
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
parser.add_argument("--small", action="store_true")
args = parser.parse_args()
print(build(args.config, small=args.small))
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from data.schema import OBJECT_TO_ID, PREDICATE_TO_ID, SUBJECT_TO_ID, Fact
def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int]:
fact = Fact.from_dict(value)
return (
SUBJECT_TO_ID[fact.subject],
PREDICATE_TO_ID[fact.predicate],
OBJECT_TO_ID[fact.object],
)
class EvidenceDataset(Dataset):
def __init__(self, manifest: Path, image_size: int = 224):
self.root = manifest.parent
self.image_size = image_size
self.records = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
def __len__(self) -> int:
return len(self.records)
def __getitem__(self, index: int) -> dict[str, Any]:
record = self.records[index]
image = Image.open(self.root / record["image"]).convert("RGB")
image = image.resize((self.image_size, self.image_size))
image_tensor = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1).float() / 255.0
return {
"image": image_tensor,
"rules_text": record["rules_text"],
"visual_facts": torch.tensor(
[fact_to_ids(value) for value in record["visual_facts"]], dtype=torch.long
),
"text_facts": torch.tensor(
[fact_to_ids(value) for value in record["text_facts"]], dtype=torch.long
),
"scene_id": record["scene_id"],
}
def collate_evidence(batch: list[dict[str, Any]]) -> dict[str, Any]:
return {
"images": torch.stack([item["image"] for item in batch]),
"rules_text": [item["rules_text"] for item in batch],
"visual_facts": [item["visual_facts"] for item in batch],
"text_facts": [item["text_facts"] for item in batch],
"scene_id": [item["scene_id"] for item in batch],
}
+28
View File
@@ -0,0 +1,28 @@
from __future__ import annotations
import random
from typing import Any
from data.schema import CLASSES, Fact
def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[str, Any]:
rng = random.Random((seed * 9_999_991) + int(record["scene_id"].split("-")[-1]))
target = rng.choice(record["objects"])
color = target["color"]
color_index = ("red", "blue", "green", "yellow").index(color)
class_index = (color_index + (1 if held_out else 0)) % len(CLASSES)
class_name = CLASSES[class_index]
rule = Fact(color, "implies_class", class_name)
result = dict(record)
result.update(
{
"rules_text": f"Every {color} object belongs to class {class_name}.",
"text_facts": [rule.to_dict()],
"question": f"Which object belongs to class {class_name}?",
"answer": target["object_id"],
"combination": f"{color}->{class_name}",
}
)
return result
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import random
from typing import Any
from data.schema import COLORS, OBJECT_IDS, RELATIONS, SHAPES, Fact, SceneObject
def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fact:
relation = rng.choice(RELATIONS)
if relation == "left_of":
first = SceneObject(first.object_id, first.color, first.shape, 55, 112)
second = SceneObject(second.object_id, second.color, second.shape, 169, 112)
elif relation == "right_of":
first = SceneObject(first.object_id, first.color, first.shape, 169, 112)
second = SceneObject(second.object_id, second.color, second.shape, 55, 112)
elif relation == "above":
first = SceneObject(first.object_id, first.color, first.shape, 112, 55)
second = SceneObject(second.object_id, second.color, second.shape, 112, 169)
else:
first = SceneObject(first.object_id, first.color, first.shape, 112, 169)
second = SceneObject(second.object_id, second.color, second.shape, 112, 55)
return Fact(first.object_id, relation, second.object_id), first, second
def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, Any]:
if num_objects != 2:
raise ValueError("The stage-one MVP currently supports exactly two objects.")
rng = random.Random((seed * 1_000_003) + scene_id)
colors = rng.sample(COLORS, k=num_objects)
shapes = rng.sample(SHAPES, k=num_objects)
objects = [
SceneObject(OBJECT_IDS[index], colors[index], shapes[index], 0, 0)
for index in range(num_objects)
]
relation, first, second = _relation(objects[0], objects[1], rng)
objects = [first, second]
facts = []
for item in objects:
facts.extend(
[
Fact(item.object_id, "color", item.color),
Fact(item.object_id, "shape", item.shape),
]
)
facts.append(relation)
return {
"scene_id": f"scene-{scene_id:07d}",
"objects": [item.to_dict() for item in objects],
"visual_facts": [fact.to_dict() for fact in facts],
}
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import math
from pathlib import Path
from typing import Any
from PIL import Image, ImageDraw
RGB = {
"red": (220, 55, 55),
"blue": (55, 100, 220),
"green": (45, 165, 85),
"yellow": (235, 195, 45),
}
def _polygon(cx: int, cy: int, radius: int, sides: int, rotation: float = -math.pi / 2):
return [
(
cx + radius * math.cos(rotation + (2 * math.pi * index / sides)),
cy + radius * math.sin(rotation + (2 * math.pi * index / sides)),
)
for index in range(sides)
]
def render_scene(record: dict[str, Any], output_path: Path, image_size: int = 224) -> None:
image = Image.new("RGB", (image_size, image_size), (248, 248, 248))
draw = ImageDraw.Draw(image)
scale = image_size / 224
radius = int(28 * scale)
for item in record["objects"]:
x, y = int(item["x"] * scale), int(item["y"] * scale)
color = RGB[item["color"]]
box = (x - radius, y - radius, x + radius, y + radius)
if item["shape"] == "circle":
draw.ellipse(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
elif item["shape"] == "square":
draw.rectangle(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
elif item["shape"] == "triangle":
draw.polygon(_polygon(x, y, radius, 3), fill=color, outline=(20, 20, 20))
else:
points = []
for index in range(10):
current_radius = radius if index % 2 == 0 else radius * 0.45
angle = -math.pi / 2 + index * math.pi / 5
points.append((x + current_radius * math.cos(angle), y + current_radius * math.sin(angle)))
draw.polygon(points, fill=color, outline=(20, 20, 20))
draw.text((x - radius, y + radius + int(4 * scale)), item["object_id"], fill=(20, 20, 20))
output_path.parent.mkdir(parents=True, exist_ok=True)
image.save(output_path)
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from dataclasses import asdict, dataclass
from typing import Any
COLORS = ("red", "blue", "green", "yellow")
SHAPES = ("circle", "square", "triangle", "star")
RELATIONS = ("left_of", "right_of", "above", "below")
CLASSES = ("alpha", "beta", "gamma", "delta")
OBJECT_IDS = ("obj1", "obj2", "obj3")
PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "implies_class")
ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES
SUBJECT_TO_ID = {value: index for index, value in enumerate(ENTITIES)}
PREDICATE_TO_ID = {value: index for index, value in enumerate(PREDICATES)}
OBJECT_TO_ID = SUBJECT_TO_ID
@dataclass(frozen=True)
class Fact:
subject: str
predicate: str
object: str
def to_dict(self) -> dict[str, str]:
return asdict(self)
@classmethod
def from_dict(cls, value: dict[str, str]) -> "Fact":
return cls(**value)
@dataclass(frozen=True)
class SceneObject:
object_id: str
color: str
shape: str
x: int
y: int
def to_dict(self) -> dict[str, Any]:
return asdict(self)
+2
View File
@@ -0,0 +1,2 @@
"""Evaluation entry points."""
+94
View File
@@ -0,0 +1,94 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
import torch
import yaml
from torch.utils.data import DataLoader
from data.dataset import EvidenceDataset, collate_evidence
from models.evidence_model import EvidenceExtractorModel
def slot_counts(predictions: dict[str, torch.Tensor], targets: list[torch.Tensor]) -> dict[str, int]:
subjects = predictions["subject_logits"].argmax(-1).cpu()
predicates = predictions["predicate_logits"].argmax(-1).cpu()
objects = predictions["object_logits"].argmax(-1).cpu()
valid = predictions["valid_logits"].argmax(-1).cpu().bool()
counts = {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0}
for batch_index, facts in enumerate(targets):
predicted = {
(int(subjects[batch_index, slot]), int(predicates[batch_index, slot]), int(objects[batch_index, slot]))
for slot in range(subjects.shape[1])
if valid[batch_index, slot]
}
expected = {tuple(map(int, row.tolist())) for row in facts}
counts["tp"] += len(predicted & expected)
counts["fp"] += len(predicted - expected)
counts["fn"] += len(expected - predicted)
for triple in predicted:
best = max((sum(a == b for a, b in zip(triple, target)) for target in expected), default=0)
counts["component_correct"] += best
counts["component_total"] += 3
return counts
def summarize(counts: dict[str, int]) -> dict[str, float]:
precision = counts["tp"] / max(1, counts["tp"] + counts["fp"])
recall = counts["tp"] / max(1, counts["tp"] + counts["fn"])
return {
"precision": precision,
"recall": recall,
"f1": 2 * precision * recall / max(1e-12, precision + recall),
"component_accuracy": counts["component_correct"] / max(1, counts["component_total"]),
}
@torch.no_grad()
def evaluate(
model: EvidenceExtractorModel, loader: DataLoader, device: torch.device
) -> dict[str, Any]:
model.eval()
totals = {
"visual": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0},
"text": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0},
}
for batch in loader:
outputs = model(batch["images"].to(device), batch["rules_text"])
for modality, target_name in (("visual", "visual_facts"), ("text", "text_facts")):
counts = slot_counts(outputs[modality], batch[target_name])
for key, value in counts.items():
totals[modality][key] += value
return {modality: summarize(value) for modality, value in totals.items()}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
parser.add_argument("--checkpoint", type=Path, required=True)
parser.add_argument("--backbone", choices=("tiny", "clip"), default=None)
parser.add_argument("--split", default="test_id")
args = parser.parse_args()
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = EvidenceExtractorModel(config, backbone_name=args.backbone).to(device)
state = torch.load(args.checkpoint, map_location=device, weights_only=True)
model.load_state_dict(state["model"])
dataset = EvidenceDataset(
Path(config["data"]["root"]) / f"{args.split}.jsonl",
config["data"]["image_size"],
)
loader = DataLoader(
dataset,
batch_size=config["training"]["batch_size"],
collate_fn=collate_evidence,
)
print(json.dumps(evaluate(model, loader, device), indent=2))
if __name__ == "__main__":
main()
+2
View File
@@ -0,0 +1,2 @@
"""Training losses for evidence extraction."""
+36
View File
@@ -0,0 +1,36 @@
from __future__ import annotations
import torch
import torch.nn.functional as F
def supervised_alignment_loss(
states: torch.Tensor,
targets: list[torch.Tensor],
assignments: list[tuple[torch.Tensor, torch.Tensor]],
temperature: float = 0.1,
) -> torch.Tensor:
vectors = []
labels = []
for batch_index, (slot_index, fact_index) in enumerate(assignments):
if len(slot_index) == 0:
continue
vectors.append(states[batch_index, slot_index])
labels.append(targets[batch_index].to(states.device)[fact_index])
if not vectors:
return states.sum() * 0.0
vectors_tensor = F.normalize(torch.cat(vectors), dim=-1)
labels_tensor = torch.cat(labels)
if len(vectors_tensor) < 2:
return vectors_tensor.sum() * 0.0
same = (labels_tensor[:, None, :] == labels_tensor[None, :, :]).all(-1)
same.fill_diagonal_(False)
logits = vectors_tensor @ vectors_tensor.T / temperature
logits.fill_diagonal_(-torch.inf)
log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
positive_count = same.sum(1)
usable = positive_count > 0
if not usable.any():
return vectors_tensor.sum() * 0.0
return -((log_prob.masked_fill(~same, 0.0).sum(1) / positive_count.clamp_min(1))[usable]).mean()
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from dataclasses import dataclass
import torch
import torch.nn.functional as F
from scipy.optimize import linear_sum_assignment
@dataclass
class MatchingOutput:
loss: torch.Tensor
assignments: list[tuple[torch.Tensor, torch.Tensor]]
def hungarian_matching_loss(
predictions: dict[str, torch.Tensor],
targets: list[torch.Tensor],
valid_weight: float = 0.5,
) -> MatchingOutput:
subject_logp = predictions["subject_logits"].log_softmax(-1)
predicate_logp = predictions["predicate_logits"].log_softmax(-1)
object_logp = predictions["object_logits"].log_softmax(-1)
valid_logits = predictions["valid_logits"]
device = subject_logp.device
total = subject_logp.sum() * 0.0
assignments = []
for batch_index, facts in enumerate(targets):
facts = facts.to(device)
num_slots = subject_logp.shape[1]
if len(facts):
cost = -(
subject_logp[batch_index, :, facts[:, 0]]
+ predicate_logp[batch_index, :, facts[:, 1]]
+ object_logp[batch_index, :, facts[:, 2]]
)
slot_index, fact_index = linear_sum_assignment(cost.detach().cpu().numpy())
slots = torch.as_tensor(slot_index, dtype=torch.long, device=device)
matched = torch.as_tensor(fact_index, dtype=torch.long, device=device)
selected = facts[matched]
total = total + F.cross_entropy(
predictions["subject_logits"][batch_index, slots], selected[:, 0]
)
total = total + F.cross_entropy(
predictions["predicate_logits"][batch_index, slots], selected[:, 1]
)
total = total + F.cross_entropy(
predictions["object_logits"][batch_index, slots], selected[:, 2]
)
else:
slots = torch.empty(0, dtype=torch.long, device=device)
matched = torch.empty(0, dtype=torch.long, device=device)
valid_target = torch.zeros(num_slots, dtype=torch.long, device=device)
valid_target[slots] = 1
total = total + valid_weight * F.cross_entropy(valid_logits[batch_index], valid_target)
assignments.append((slots, matched))
return MatchingOutput(total / len(targets), assignments)
+2
View File
@@ -0,0 +1,2 @@
"""Evidence extraction models."""
+47
View File
@@ -0,0 +1,47 @@
from __future__ import annotations
from typing import Any
import torch
from torch import nn
from data.schema import ENTITIES, PREDICATES
from models.frozen_backbone import build_backbone
from models.slot_extractor import SlotExtractor
class EvidenceExtractorModel(nn.Module):
def __init__(self, config: dict[str, Any], backbone_name: str | None = None):
super().__init__()
model_config = config["model"]
selected = backbone_name or model_config["backbone"]
self.backbone_name = selected
self.backbone = build_backbone(selected, model_config["clip_model"])
common = {
"state_dim": model_config["state_dim"],
"num_layers": model_config["extractor_layers"],
"num_heads": model_config["num_heads"],
"dropout": model_config["dropout"],
"num_subjects": len(ENTITIES),
"num_predicates": len(PREDICATES),
"num_objects": len(ENTITIES),
}
self.visual_extractor = SlotExtractor(
input_dim=self.backbone.vision_dim,
num_slots=model_config["num_visual_slots"],
**common,
)
self.text_extractor = SlotExtractor(
input_dim=self.backbone.text_dim,
num_slots=model_config["num_text_slots"],
**common,
)
def forward(self, images: torch.Tensor, rules_text: list[str]) -> dict[str, Any]:
visual_tokens, visual_mask = self.backbone.encode_image(images)
text_tokens, text_mask = self.backbone.encode_text(rules_text)
return {
"visual": self.visual_extractor(visual_tokens, visual_mask),
"text": self.text_extractor(text_tokens, text_mask),
}
+106
View File
@@ -0,0 +1,106 @@
from __future__ import annotations
import hashlib
from abc import ABC, abstractmethod
import torch
from torch import nn
class FrozenBackbone(nn.Module, ABC):
vision_dim: int
text_dim: int
@abstractmethod
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: ...
@abstractmethod
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]: ...
def train(self, mode: bool = True) -> "FrozenBackbone":
# The extractor trains around the backbone; pretrained backbone behavior stays fixed.
return super().train(False)
class TinyFrozenBackbone(FrozenBackbone):
"""Offline backbone used to exercise the complete pipeline in CPU CI."""
def __init__(self, hidden_dim: int = 64, patch_size: int = 16):
super().__init__()
generator = torch.Generator().manual_seed(1234)
self.patch = nn.Conv2d(3, hidden_dim, patch_size, stride=patch_size, bias=False)
self.byte_embedding = nn.Embedding(257, hidden_dim, padding_idx=0)
with torch.no_grad():
self.patch.weight.normal_(generator=generator, std=0.04)
self.byte_embedding.weight.normal_(generator=generator, std=0.04)
self.byte_embedding.weight[0].zero_()
self.requires_grad_(False)
self.vision_dim = hidden_dim
self.text_dim = hidden_dim
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
with torch.no_grad():
tokens = self.patch(images).flatten(2).transpose(1, 2)
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
return tokens, mask
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
encoded = []
for text in texts:
# Hash whole whitespace-delimited tokens so the offline representation preserves words.
ids = []
for token in text.lower().replace(".", "").split():
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=2).digest()
ids.append((int.from_bytes(digest, "little") % 256) + 1)
encoded.append(ids or [1])
length = max(len(row) for row in encoded)
input_ids = torch.zeros(len(encoded), length, dtype=torch.long, device=self.patch.weight.device)
mask = torch.zeros_like(input_ids, dtype=torch.bool)
for index, row in enumerate(encoded):
input_ids[index, : len(row)] = torch.tensor(row, device=input_ids.device)
mask[index, : len(row)] = True
with torch.no_grad():
tokens = self.byte_embedding(input_ids)
return tokens, mask
class CLIPFrozenBackbone(FrozenBackbone):
"""Frozen CLIP token encoder for the first real GPU experiment."""
def __init__(self, model_name: str):
super().__init__()
try:
from transformers import AutoTokenizer, CLIPModel
except ImportError as error:
raise RuntimeError("Install the 'hf' extra to use the CLIP backbone.") from error
self.model = CLIPModel.from_pretrained(model_name)
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
self.model.requires_grad_(False).eval()
self.vision_dim = self.model.config.vision_config.hidden_size
self.text_dim = self.model.config.text_config.hidden_size
mean = torch.tensor([0.48145466, 0.4578275, 0.40821073])
std = torch.tensor([0.26862954, 0.26130258, 0.27577711])
self.register_buffer("image_mean", mean.view(1, 3, 1, 1), persistent=False)
self.register_buffer("image_std", std.view(1, 3, 1, 1), persistent=False)
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
pixels = (images - self.image_mean) / self.image_std
with torch.no_grad():
tokens = self.model.vision_model(pixel_values=pixels).last_hidden_state
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
return tokens, mask
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
values = self.tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
values = {key: value.to(self.image_mean.device) for key, value in values.items()}
with torch.no_grad():
tokens = self.model.text_model(**values).last_hidden_state
return tokens, values["attention_mask"].bool()
def build_backbone(name: str, clip_model: str) -> FrozenBackbone:
if name == "tiny":
return TinyFrozenBackbone()
if name == "clip":
return CLIPFrozenBackbone(clip_model)
raise ValueError(f"Unknown backbone: {name}")
+68
View File
@@ -0,0 +1,68 @@
from __future__ import annotations
import torch
from torch import nn
class CrossAttentionBlock(nn.Module):
def __init__(self, state_dim: int, num_heads: int, dropout: float):
super().__init__()
self.cross_attention = nn.MultiheadAttention(
state_dim, num_heads, dropout=dropout, batch_first=True
)
self.norm1 = nn.LayerNorm(state_dim)
self.ffn = nn.Sequential(
nn.Linear(state_dim, state_dim * 4),
nn.GELU(),
nn.Dropout(dropout),
nn.Linear(state_dim * 4, state_dim),
)
self.norm2 = nn.LayerNorm(state_dim)
def forward(
self, slots: torch.Tensor, tokens: torch.Tensor, token_mask: torch.Tensor
) -> torch.Tensor:
update, _ = self.cross_attention(
slots, tokens, tokens, key_padding_mask=~token_mask, need_weights=False
)
slots = self.norm1(slots + update)
return self.norm2(slots + self.ffn(slots))
class SlotExtractor(nn.Module):
def __init__(
self,
input_dim: int,
state_dim: int,
num_slots: int,
num_layers: int,
num_heads: int,
dropout: float,
num_subjects: int,
num_predicates: int,
num_objects: int,
):
super().__init__()
self.input_projection = nn.Linear(input_dim, state_dim)
self.slot_queries = nn.Parameter(torch.randn(num_slots, state_dim) * 0.02)
self.blocks = nn.ModuleList(
[CrossAttentionBlock(state_dim, num_heads, dropout) for _ in range(num_layers)]
)
self.subject_head = nn.Linear(state_dim, num_subjects)
self.predicate_head = nn.Linear(state_dim, num_predicates)
self.object_head = nn.Linear(state_dim, num_objects)
self.valid_head = nn.Linear(state_dim, 2)
def forward(self, tokens: torch.Tensor, token_mask: torch.Tensor) -> dict[str, torch.Tensor]:
tokens = self.input_projection(tokens)
slots = self.slot_queries.unsqueeze(0).expand(tokens.shape[0], -1, -1)
for block in self.blocks:
slots = block(slots, tokens, token_mask)
return {
"states": slots,
"subject_logits": self.subject_head(slots),
"predicate_logits": self.predicate_head(slots),
"object_logits": self.object_head(slots),
"valid_logits": self.valid_head(slots),
}
+29
View File
@@ -0,0 +1,29 @@
[project]
name = "composable-evidence-states-research"
version = "0.1.0"
description = "Research MVP for extracting composable evidence states from images and text."
readme = "README.md"
requires-python = ">=3.10,<3.13"
dependencies = [
"numpy>=1.26",
"pillow>=10.0",
"pyyaml>=6.0",
"scipy>=1.11",
"torch>=2.3",
]
[project.optional-dependencies]
hf = ["transformers>=4.48"]
dev = ["pytest>=8.0", "ruff>=0.8"]
[tool.pytest.ini_options]
testpaths = ["tests"]
addopts = "-q"
[tool.ruff]
line-length = 100
target-version = "py310"
[tool.ruff.lint]
select = ["E", "F", "I"]
+18
View File
@@ -0,0 +1,18 @@
from data.generate_rules import add_rule
from data.generate_scenes import generate_scene
def test_scene_generation_is_deterministic():
first = generate_scene(7, seed=17)
second = generate_scene(7, seed=17)
assert first == second
assert len(first["visual_facts"]) == 5
def test_held_out_rule_changes_combination():
scene = generate_scene(3, seed=17)
train = add_rule(scene, seed=17, held_out=False)
held_out = add_rule(scene, seed=17, held_out=True)
assert train["combination"] != held_out["combination"]
assert train["answer"] == held_out["answer"]
+27
View File
@@ -0,0 +1,27 @@
import torch
from losses.matching_loss import hungarian_matching_loss
from models.slot_extractor import SlotExtractor
def test_slot_extractor_and_matching_loss_backward():
extractor = SlotExtractor(
input_dim=32,
state_dim=64,
num_slots=4,
num_layers=2,
num_heads=4,
dropout=0.0,
num_subjects=12,
num_predicates=7,
num_objects=12,
)
tokens = torch.randn(2, 10, 32)
mask = torch.ones(2, 10, dtype=torch.bool)
predictions = extractor(tokens, mask)
targets = [torch.tensor([[0, 0, 4], [1, 1, 5]]), torch.tensor([[1, 0, 6]])]
result = hungarian_matching_loss(predictions, targets)
assert torch.isfinite(result.loss)
result.loss.backward()
assert extractor.slot_queries.grad is not None
+110
View File
@@ -0,0 +1,110 @@
from __future__ import annotations
import argparse
import json
import random
from pathlib import Path
from typing import Any
import numpy as np
import torch
import yaml
from torch.utils.data import DataLoader
from data.dataset import EvidenceDataset, collate_evidence
from eval.evaluate_slots import evaluate
from losses.alignment_loss import supervised_alignment_loss
from losses.matching_loss import hungarian_matching_loss
from models.evidence_model import EvidenceExtractorModel
def set_seed(seed: int) -> None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
if torch.cuda.is_available():
torch.cuda.manual_seed_all(seed)
def train(config: dict[str, Any], backbone_name: str, epochs: int | None = None) -> Path:
set_seed(config["seed"])
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
data_root = Path(config["data"]["root"])
train_dataset = EvidenceDataset(data_root / "train.jsonl", config["data"]["image_size"])
validation_dataset = EvidenceDataset(
data_root / "validation.jsonl", config["data"]["image_size"]
)
loader_options = {
"batch_size": config["training"]["batch_size"],
"num_workers": config["training"]["num_workers"],
"collate_fn": collate_evidence,
}
train_loader = DataLoader(train_dataset, shuffle=True, **loader_options)
validation_loader = DataLoader(validation_dataset, **loader_options)
model = EvidenceExtractorModel(config, backbone_name=backbone_name).to(device)
trainable = [parameter for parameter in model.parameters() if parameter.requires_grad]
optimizer = torch.optim.AdamW(
trainable,
lr=config["training"]["learning_rate"],
weight_decay=config["training"]["weight_decay"],
)
output_dir = Path(config["training"]["output_dir"])
if backbone_name != config["model"]["backbone"]:
output_dir = output_dir.parent / backbone_name
output_dir.mkdir(parents=True, exist_ok=True)
best_path = output_dir / "best.pt"
best_score = -1.0
epoch_count = epochs or config["training"]["epochs"]
for epoch in range(1, epoch_count + 1):
model.train()
running_loss = 0.0
for batch in train_loader:
optimizer.zero_grad(set_to_none=True)
outputs = model(batch["images"].to(device), batch["rules_text"])
visual = hungarian_matching_loss(
outputs["visual"], batch["visual_facts"], config["training"]["valid_weight"]
)
text = hungarian_matching_loss(
outputs["text"], batch["text_facts"], config["training"]["valid_weight"]
)
alignment = supervised_alignment_loss(
outputs["visual"]["states"], batch["visual_facts"], visual.assignments
) + supervised_alignment_loss(
outputs["text"]["states"], batch["text_facts"], text.assignments
)
loss = visual.loss + text.loss + config["training"]["alignment_weight"] * alignment
loss.backward()
torch.nn.utils.clip_grad_norm_(trainable, 1.0)
optimizer.step()
running_loss += float(loss.detach())
metrics = evaluate(model, validation_loader, device)
score = metrics["visual"]["f1"] + metrics["text"]["f1"]
summary = {
"epoch": epoch,
"loss": running_loss / max(1, len(train_loader)),
"metrics": metrics,
}
print(json.dumps(summary))
if score > best_score:
best_score = score
torch.save(
{"model": model.state_dict(), "epoch": epoch, "metrics": metrics}, best_path
)
return best_path
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
parser.add_argument("--backbone", choices=("tiny", "clip"), default=None)
parser.add_argument("--epochs", type=int, default=None)
args = parser.parse_args()
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))
backbone_name = args.backbone or config["model"]["backbone"]
print(train(config, backbone_name, epochs=args.epochs))
if __name__ == "__main__":
main()
Generated
+1200
View File
File diff suppressed because it is too large Load Diff