This commit is contained in:
@@ -0,0 +1,28 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ces-ci
|
||||
env:
|
||||
CUDA_VISIBLE_DEVICES: ""
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Reuse runner PyTorch and install lightweight dependencies
|
||||
run: |
|
||||
uv venv --system-site-packages --python python .venv-ci
|
||||
uv pip install --python .venv-ci/bin/python numpy pillow pyyaml scipy pytest ruff
|
||||
- name: Lint
|
||||
run: .venv-ci/bin/ruff check .
|
||||
- name: Unit tests
|
||||
run: .venv-ci/bin/pytest
|
||||
- name: End-to-end CPU smoke test
|
||||
run: |
|
||||
.venv-ci/bin/python -m data.build_splits --config configs/mvp.yaml --small
|
||||
.venv-ci/bin/python train_extractor.py --config configs/mvp.yaml --backbone tiny --epochs 1
|
||||
@@ -0,0 +1,27 @@
|
||||
name: GPU smoke
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
extractor:
|
||||
runs-on: gpu
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: astral-sh/setup-uv@v6
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Verify CUDA
|
||||
run: nvidia-smi
|
||||
- name: Reuse runner PyTorch and install lightweight dependencies
|
||||
run: |
|
||||
uv venv --system-site-packages --python python .venv-gpu
|
||||
uv pip install --python .venv-gpu/bin/python numpy pillow pyyaml scipy transformers pytest ruff
|
||||
- name: Verify PyTorch sees the GPU
|
||||
run: |
|
||||
.venv-gpu/bin/python -c "import torch; assert torch.cuda.is_available(); print(torch.__version__, torch.version.cuda, torch.cuda.get_device_name(), torch.cuda.get_device_capability())"
|
||||
- name: Build smoke dataset
|
||||
run: .venv-gpu/bin/python -m data.build_splits --config configs/mvp.yaml --small
|
||||
- name: Train extractor
|
||||
run: .venv-gpu/bin/python train_extractor.py --config configs/mvp.yaml --backbone clip --epochs 1
|
||||
+10
@@ -0,0 +1,10 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.ruff_cache/
|
||||
artifacts/
|
||||
checkpoints/
|
||||
datasets/
|
||||
wandb/
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# Composable Evidence States Research
|
||||
|
||||
This repository contains the first-stage MVP for learning fact-level evidence slots from
|
||||
synthetic images and natural-language rules. The Composer is intentionally deferred until
|
||||
the extractor passes its fact-recovery checks.
|
||||
|
||||
## Current scope
|
||||
|
||||
- Generate deterministic scenes containing colored geometric objects and spatial relations.
|
||||
- Render scenes to PNG while preserving machine-readable fact triples.
|
||||
- Generate textual implication rules as normalized triples.
|
||||
- Build IID and held-out-combination splits without leaking held-out rule pairs into training.
|
||||
- Train visual and textual slot extractors with Hungarian set matching.
|
||||
- Measure triple exact match, component accuracy, precision, recall, and F1.
|
||||
- Run CPU smoke tests with a tiny frozen backbone or GPU experiments with frozen CLIP.
|
||||
|
||||
## Quick start
|
||||
|
||||
```bash
|
||||
uv sync --extra dev
|
||||
uv run python -m data.build_splits --config configs/mvp.yaml --small
|
||||
uv run python train_extractor.py --config configs/mvp.yaml --backbone tiny --epochs 2
|
||||
uv run python -m eval.evaluate_slots --config configs/mvp.yaml \
|
||||
--checkpoint artifacts/tiny/best.pt --backbone tiny
|
||||
uv run pytest
|
||||
```
|
||||
|
||||
For the first real GPU experiment:
|
||||
|
||||
```bash
|
||||
uv sync --extra dev --extra hf
|
||||
uv run python -m data.build_splits --config configs/mvp.yaml
|
||||
uv run python train_extractor.py --config configs/mvp.yaml --backbone clip
|
||||
```
|
||||
|
||||
The CLIP path uses `openai/clip-vit-base-patch32` by default. It is a deliberately small
|
||||
backbone for validating the extractor. A Qwen-VL adapter and the Composer belong to the next
|
||||
milestone after the evidence slots are shown to recover atomic facts reliably.
|
||||
|
||||
## Success gate before Composer work
|
||||
|
||||
- Text-rule triple F1 >= 0.95.
|
||||
- Visual-fact triple F1 >= 0.90 on the IID test split.
|
||||
- No severe slot collapse: predicted valid slots track the true fact count.
|
||||
- Held-out rule-combination performance is reported separately from IID performance.
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
seed: 17
|
||||
|
||||
data:
|
||||
root: datasets/mvp
|
||||
image_size: 224
|
||||
train_size: 30000
|
||||
validation_size: 3000
|
||||
test_id_size: 3000
|
||||
test_ood_size: 3000
|
||||
num_objects: 2
|
||||
|
||||
model:
|
||||
backbone: tiny
|
||||
clip_model: openai/clip-vit-base-patch32
|
||||
state_dim: 256
|
||||
num_visual_slots: 6
|
||||
num_text_slots: 3
|
||||
extractor_layers: 2
|
||||
num_heads: 8
|
||||
dropout: 0.1
|
||||
|
||||
training:
|
||||
batch_size: 32
|
||||
learning_rate: 0.0003
|
||||
weight_decay: 0.01
|
||||
epochs: 10
|
||||
num_workers: 0
|
||||
valid_weight: 0.5
|
||||
alignment_weight: 0.05
|
||||
output_dir: artifacts/tiny
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Synthetic data generation and loading."""
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from data.generate_rules import add_rule
|
||||
from data.generate_scenes import generate_scene
|
||||
from data.render_scene import render_scene
|
||||
|
||||
|
||||
def _write_split(
|
||||
name: str,
|
||||
size: int,
|
||||
start: int,
|
||||
root: Path,
|
||||
seed: int,
|
||||
image_size: int,
|
||||
held_out: bool,
|
||||
) -> None:
|
||||
path = root / f"{name}.jsonl"
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for scene_number in range(start, start + size):
|
||||
record = add_rule(generate_scene(scene_number, seed), seed, held_out=held_out)
|
||||
relative_image = Path("images") / f"{record['scene_id']}.png"
|
||||
render_scene(record, root / relative_image, image_size=image_size)
|
||||
record["image"] = str(relative_image)
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def build(config_path: Path, small: bool = False) -> Path:
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||
root = Path(config["data"]["root"])
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
sizes = {
|
||||
"train": config["data"]["train_size"],
|
||||
"validation": config["data"]["validation_size"],
|
||||
"test_id": config["data"]["test_id_size"],
|
||||
"test_ood": config["data"]["test_ood_size"],
|
||||
}
|
||||
if small:
|
||||
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
|
||||
cursor = 0
|
||||
for split, size in sizes.items():
|
||||
_write_split(
|
||||
split,
|
||||
size,
|
||||
cursor,
|
||||
root,
|
||||
config["seed"],
|
||||
config["data"]["image_size"],
|
||||
held_out=split == "test_ood",
|
||||
)
|
||||
cursor += size
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--small", action="store_true")
|
||||
args = parser.parse_args()
|
||||
print(build(args.config, small=args.small))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
from data.schema import OBJECT_TO_ID, PREDICATE_TO_ID, SUBJECT_TO_ID, Fact
|
||||
|
||||
|
||||
def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int]:
|
||||
fact = Fact.from_dict(value)
|
||||
return (
|
||||
SUBJECT_TO_ID[fact.subject],
|
||||
PREDICATE_TO_ID[fact.predicate],
|
||||
OBJECT_TO_ID[fact.object],
|
||||
)
|
||||
|
||||
|
||||
class EvidenceDataset(Dataset):
|
||||
def __init__(self, manifest: Path, image_size: int = 224):
|
||||
self.root = manifest.parent
|
||||
self.image_size = image_size
|
||||
self.records = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.records)
|
||||
|
||||
def __getitem__(self, index: int) -> dict[str, Any]:
|
||||
record = self.records[index]
|
||||
image = Image.open(self.root / record["image"]).convert("RGB")
|
||||
image = image.resize((self.image_size, self.image_size))
|
||||
image_tensor = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1).float() / 255.0
|
||||
return {
|
||||
"image": image_tensor,
|
||||
"rules_text": record["rules_text"],
|
||||
"visual_facts": torch.tensor(
|
||||
[fact_to_ids(value) for value in record["visual_facts"]], dtype=torch.long
|
||||
),
|
||||
"text_facts": torch.tensor(
|
||||
[fact_to_ids(value) for value in record["text_facts"]], dtype=torch.long
|
||||
),
|
||||
"scene_id": record["scene_id"],
|
||||
}
|
||||
|
||||
|
||||
def collate_evidence(batch: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {
|
||||
"images": torch.stack([item["image"] for item in batch]),
|
||||
"rules_text": [item["rules_text"] for item in batch],
|
||||
"visual_facts": [item["visual_facts"] for item in batch],
|
||||
"text_facts": [item["text_facts"] for item in batch],
|
||||
"scene_id": [item["scene_id"] for item in batch],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
from data.schema import CLASSES, Fact
|
||||
|
||||
|
||||
def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[str, Any]:
|
||||
rng = random.Random((seed * 9_999_991) + int(record["scene_id"].split("-")[-1]))
|
||||
target = rng.choice(record["objects"])
|
||||
color = target["color"]
|
||||
color_index = ("red", "blue", "green", "yellow").index(color)
|
||||
class_index = (color_index + (1 if held_out else 0)) % len(CLASSES)
|
||||
class_name = CLASSES[class_index]
|
||||
rule = Fact(color, "implies_class", class_name)
|
||||
result = dict(record)
|
||||
result.update(
|
||||
{
|
||||
"rules_text": f"Every {color} object belongs to class {class_name}.",
|
||||
"text_facts": [rule.to_dict()],
|
||||
"question": f"Which object belongs to class {class_name}?",
|
||||
"answer": target["object_id"],
|
||||
"combination": f"{color}->{class_name}",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
from data.schema import COLORS, OBJECT_IDS, RELATIONS, SHAPES, Fact, SceneObject
|
||||
|
||||
|
||||
def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fact:
|
||||
relation = rng.choice(RELATIONS)
|
||||
if relation == "left_of":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 55, 112)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 169, 112)
|
||||
elif relation == "right_of":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 169, 112)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 55, 112)
|
||||
elif relation == "above":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 112, 55)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 112, 169)
|
||||
else:
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 112, 169)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 112, 55)
|
||||
return Fact(first.object_id, relation, second.object_id), first, second
|
||||
|
||||
|
||||
def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, Any]:
|
||||
if num_objects != 2:
|
||||
raise ValueError("The stage-one MVP currently supports exactly two objects.")
|
||||
rng = random.Random((seed * 1_000_003) + scene_id)
|
||||
colors = rng.sample(COLORS, k=num_objects)
|
||||
shapes = rng.sample(SHAPES, k=num_objects)
|
||||
objects = [
|
||||
SceneObject(OBJECT_IDS[index], colors[index], shapes[index], 0, 0)
|
||||
for index in range(num_objects)
|
||||
]
|
||||
relation, first, second = _relation(objects[0], objects[1], rng)
|
||||
objects = [first, second]
|
||||
facts = []
|
||||
for item in objects:
|
||||
facts.extend(
|
||||
[
|
||||
Fact(item.object_id, "color", item.color),
|
||||
Fact(item.object_id, "shape", item.shape),
|
||||
]
|
||||
)
|
||||
facts.append(relation)
|
||||
return {
|
||||
"scene_id": f"scene-{scene_id:07d}",
|
||||
"objects": [item.to_dict() for item in objects],
|
||||
"visual_facts": [fact.to_dict() for fact in facts],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
RGB = {
|
||||
"red": (220, 55, 55),
|
||||
"blue": (55, 100, 220),
|
||||
"green": (45, 165, 85),
|
||||
"yellow": (235, 195, 45),
|
||||
}
|
||||
|
||||
|
||||
def _polygon(cx: int, cy: int, radius: int, sides: int, rotation: float = -math.pi / 2):
|
||||
return [
|
||||
(
|
||||
cx + radius * math.cos(rotation + (2 * math.pi * index / sides)),
|
||||
cy + radius * math.sin(rotation + (2 * math.pi * index / sides)),
|
||||
)
|
||||
for index in range(sides)
|
||||
]
|
||||
|
||||
|
||||
def render_scene(record: dict[str, Any], output_path: Path, image_size: int = 224) -> None:
|
||||
image = Image.new("RGB", (image_size, image_size), (248, 248, 248))
|
||||
draw = ImageDraw.Draw(image)
|
||||
scale = image_size / 224
|
||||
radius = int(28 * scale)
|
||||
for item in record["objects"]:
|
||||
x, y = int(item["x"] * scale), int(item["y"] * scale)
|
||||
color = RGB[item["color"]]
|
||||
box = (x - radius, y - radius, x + radius, y + radius)
|
||||
if item["shape"] == "circle":
|
||||
draw.ellipse(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
|
||||
elif item["shape"] == "square":
|
||||
draw.rectangle(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
|
||||
elif item["shape"] == "triangle":
|
||||
draw.polygon(_polygon(x, y, radius, 3), fill=color, outline=(20, 20, 20))
|
||||
else:
|
||||
points = []
|
||||
for index in range(10):
|
||||
current_radius = radius if index % 2 == 0 else radius * 0.45
|
||||
angle = -math.pi / 2 + index * math.pi / 5
|
||||
points.append((x + current_radius * math.cos(angle), y + current_radius * math.sin(angle)))
|
||||
draw.polygon(points, fill=color, outline=(20, 20, 20))
|
||||
draw.text((x - radius, y + radius + int(4 * scale)), item["object_id"], fill=(20, 20, 20))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output_path)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
COLORS = ("red", "blue", "green", "yellow")
|
||||
SHAPES = ("circle", "square", "triangle", "star")
|
||||
RELATIONS = ("left_of", "right_of", "above", "below")
|
||||
CLASSES = ("alpha", "beta", "gamma", "delta")
|
||||
OBJECT_IDS = ("obj1", "obj2", "obj3")
|
||||
|
||||
PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "implies_class")
|
||||
ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES
|
||||
|
||||
SUBJECT_TO_ID = {value: index for index, value in enumerate(ENTITIES)}
|
||||
PREDICATE_TO_ID = {value: index for index, value in enumerate(PREDICATES)}
|
||||
OBJECT_TO_ID = SUBJECT_TO_ID
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fact:
|
||||
subject: str
|
||||
predicate: str
|
||||
object: str
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, str]) -> "Fact":
|
||||
return cls(**value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SceneObject:
|
||||
object_id: str
|
||||
color: str
|
||||
shape: str
|
||||
x: int
|
||||
y: int
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Evaluation entry points."""
|
||||
|
||||
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import yaml
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from data.dataset import EvidenceDataset, collate_evidence
|
||||
from models.evidence_model import EvidenceExtractorModel
|
||||
|
||||
|
||||
def slot_counts(predictions: dict[str, torch.Tensor], targets: list[torch.Tensor]) -> dict[str, int]:
|
||||
subjects = predictions["subject_logits"].argmax(-1).cpu()
|
||||
predicates = predictions["predicate_logits"].argmax(-1).cpu()
|
||||
objects = predictions["object_logits"].argmax(-1).cpu()
|
||||
valid = predictions["valid_logits"].argmax(-1).cpu().bool()
|
||||
counts = {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0}
|
||||
for batch_index, facts in enumerate(targets):
|
||||
predicted = {
|
||||
(int(subjects[batch_index, slot]), int(predicates[batch_index, slot]), int(objects[batch_index, slot]))
|
||||
for slot in range(subjects.shape[1])
|
||||
if valid[batch_index, slot]
|
||||
}
|
||||
expected = {tuple(map(int, row.tolist())) for row in facts}
|
||||
counts["tp"] += len(predicted & expected)
|
||||
counts["fp"] += len(predicted - expected)
|
||||
counts["fn"] += len(expected - predicted)
|
||||
for triple in predicted:
|
||||
best = max((sum(a == b for a, b in zip(triple, target)) for target in expected), default=0)
|
||||
counts["component_correct"] += best
|
||||
counts["component_total"] += 3
|
||||
return counts
|
||||
|
||||
|
||||
def summarize(counts: dict[str, int]) -> dict[str, float]:
|
||||
precision = counts["tp"] / max(1, counts["tp"] + counts["fp"])
|
||||
recall = counts["tp"] / max(1, counts["tp"] + counts["fn"])
|
||||
return {
|
||||
"precision": precision,
|
||||
"recall": recall,
|
||||
"f1": 2 * precision * recall / max(1e-12, precision + recall),
|
||||
"component_accuracy": counts["component_correct"] / max(1, counts["component_total"]),
|
||||
}
|
||||
|
||||
|
||||
@torch.no_grad()
|
||||
def evaluate(
|
||||
model: EvidenceExtractorModel, loader: DataLoader, device: torch.device
|
||||
) -> dict[str, Any]:
|
||||
model.eval()
|
||||
totals = {
|
||||
"visual": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0},
|
||||
"text": {"tp": 0, "fp": 0, "fn": 0, "component_correct": 0, "component_total": 0},
|
||||
}
|
||||
for batch in loader:
|
||||
outputs = model(batch["images"].to(device), batch["rules_text"])
|
||||
for modality, target_name in (("visual", "visual_facts"), ("text", "text_facts")):
|
||||
counts = slot_counts(outputs[modality], batch[target_name])
|
||||
for key, value in counts.items():
|
||||
totals[modality][key] += value
|
||||
return {modality: summarize(value) for modality, value in totals.items()}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--checkpoint", type=Path, required=True)
|
||||
parser.add_argument("--backbone", choices=("tiny", "clip"), default=None)
|
||||
parser.add_argument("--split", default="test_id")
|
||||
args = parser.parse_args()
|
||||
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
model = EvidenceExtractorModel(config, backbone_name=args.backbone).to(device)
|
||||
state = torch.load(args.checkpoint, map_location=device, weights_only=True)
|
||||
model.load_state_dict(state["model"])
|
||||
dataset = EvidenceDataset(
|
||||
Path(config["data"]["root"]) / f"{args.split}.jsonl",
|
||||
config["data"]["image_size"],
|
||||
)
|
||||
loader = DataLoader(
|
||||
dataset,
|
||||
batch_size=config["training"]["batch_size"],
|
||||
collate_fn=collate_evidence,
|
||||
)
|
||||
print(json.dumps(evaluate(model, loader, device), indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Training losses for evidence extraction."""
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def supervised_alignment_loss(
|
||||
states: torch.Tensor,
|
||||
targets: list[torch.Tensor],
|
||||
assignments: list[tuple[torch.Tensor, torch.Tensor]],
|
||||
temperature: float = 0.1,
|
||||
) -> torch.Tensor:
|
||||
vectors = []
|
||||
labels = []
|
||||
for batch_index, (slot_index, fact_index) in enumerate(assignments):
|
||||
if len(slot_index) == 0:
|
||||
continue
|
||||
vectors.append(states[batch_index, slot_index])
|
||||
labels.append(targets[batch_index].to(states.device)[fact_index])
|
||||
if not vectors:
|
||||
return states.sum() * 0.0
|
||||
vectors_tensor = F.normalize(torch.cat(vectors), dim=-1)
|
||||
labels_tensor = torch.cat(labels)
|
||||
if len(vectors_tensor) < 2:
|
||||
return vectors_tensor.sum() * 0.0
|
||||
same = (labels_tensor[:, None, :] == labels_tensor[None, :, :]).all(-1)
|
||||
same.fill_diagonal_(False)
|
||||
logits = vectors_tensor @ vectors_tensor.T / temperature
|
||||
logits.fill_diagonal_(-torch.inf)
|
||||
log_prob = logits - torch.logsumexp(logits, dim=1, keepdim=True)
|
||||
positive_count = same.sum(1)
|
||||
usable = positive_count > 0
|
||||
if not usable.any():
|
||||
return vectors_tensor.sum() * 0.0
|
||||
return -((log_prob.masked_fill(~same, 0.0).sum(1) / positive_count.clamp_min(1))[usable]).mean()
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from scipy.optimize import linear_sum_assignment
|
||||
|
||||
|
||||
@dataclass
|
||||
class MatchingOutput:
|
||||
loss: torch.Tensor
|
||||
assignments: list[tuple[torch.Tensor, torch.Tensor]]
|
||||
|
||||
|
||||
def hungarian_matching_loss(
|
||||
predictions: dict[str, torch.Tensor],
|
||||
targets: list[torch.Tensor],
|
||||
valid_weight: float = 0.5,
|
||||
) -> MatchingOutput:
|
||||
subject_logp = predictions["subject_logits"].log_softmax(-1)
|
||||
predicate_logp = predictions["predicate_logits"].log_softmax(-1)
|
||||
object_logp = predictions["object_logits"].log_softmax(-1)
|
||||
valid_logits = predictions["valid_logits"]
|
||||
device = subject_logp.device
|
||||
total = subject_logp.sum() * 0.0
|
||||
assignments = []
|
||||
|
||||
for batch_index, facts in enumerate(targets):
|
||||
facts = facts.to(device)
|
||||
num_slots = subject_logp.shape[1]
|
||||
if len(facts):
|
||||
cost = -(
|
||||
subject_logp[batch_index, :, facts[:, 0]]
|
||||
+ predicate_logp[batch_index, :, facts[:, 1]]
|
||||
+ object_logp[batch_index, :, facts[:, 2]]
|
||||
)
|
||||
slot_index, fact_index = linear_sum_assignment(cost.detach().cpu().numpy())
|
||||
slots = torch.as_tensor(slot_index, dtype=torch.long, device=device)
|
||||
matched = torch.as_tensor(fact_index, dtype=torch.long, device=device)
|
||||
selected = facts[matched]
|
||||
total = total + F.cross_entropy(
|
||||
predictions["subject_logits"][batch_index, slots], selected[:, 0]
|
||||
)
|
||||
total = total + F.cross_entropy(
|
||||
predictions["predicate_logits"][batch_index, slots], selected[:, 1]
|
||||
)
|
||||
total = total + F.cross_entropy(
|
||||
predictions["object_logits"][batch_index, slots], selected[:, 2]
|
||||
)
|
||||
else:
|
||||
slots = torch.empty(0, dtype=torch.long, device=device)
|
||||
matched = torch.empty(0, dtype=torch.long, device=device)
|
||||
valid_target = torch.zeros(num_slots, dtype=torch.long, device=device)
|
||||
valid_target[slots] = 1
|
||||
total = total + valid_weight * F.cross_entropy(valid_logits[batch_index], valid_target)
|
||||
assignments.append((slots, matched))
|
||||
|
||||
return MatchingOutput(total / len(targets), assignments)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
"""Evidence extraction models."""
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from data.schema import ENTITIES, PREDICATES
|
||||
from models.frozen_backbone import build_backbone
|
||||
from models.slot_extractor import SlotExtractor
|
||||
|
||||
|
||||
class EvidenceExtractorModel(nn.Module):
|
||||
def __init__(self, config: dict[str, Any], backbone_name: str | None = None):
|
||||
super().__init__()
|
||||
model_config = config["model"]
|
||||
selected = backbone_name or model_config["backbone"]
|
||||
self.backbone_name = selected
|
||||
self.backbone = build_backbone(selected, model_config["clip_model"])
|
||||
common = {
|
||||
"state_dim": model_config["state_dim"],
|
||||
"num_layers": model_config["extractor_layers"],
|
||||
"num_heads": model_config["num_heads"],
|
||||
"dropout": model_config["dropout"],
|
||||
"num_subjects": len(ENTITIES),
|
||||
"num_predicates": len(PREDICATES),
|
||||
"num_objects": len(ENTITIES),
|
||||
}
|
||||
self.visual_extractor = SlotExtractor(
|
||||
input_dim=self.backbone.vision_dim,
|
||||
num_slots=model_config["num_visual_slots"],
|
||||
**common,
|
||||
)
|
||||
self.text_extractor = SlotExtractor(
|
||||
input_dim=self.backbone.text_dim,
|
||||
num_slots=model_config["num_text_slots"],
|
||||
**common,
|
||||
)
|
||||
|
||||
def forward(self, images: torch.Tensor, rules_text: list[str]) -> dict[str, Any]:
|
||||
visual_tokens, visual_mask = self.backbone.encode_image(images)
|
||||
text_tokens, text_mask = self.backbone.encode_text(rules_text)
|
||||
return {
|
||||
"visual": self.visual_extractor(visual_tokens, visual_mask),
|
||||
"text": self.text_extractor(text_tokens, text_mask),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from abc import ABC, abstractmethod
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
class FrozenBackbone(nn.Module, ABC):
|
||||
vision_dim: int
|
||||
text_dim: int
|
||||
|
||||
@abstractmethod
|
||||
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: ...
|
||||
|
||||
@abstractmethod
|
||||
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]: ...
|
||||
|
||||
def train(self, mode: bool = True) -> "FrozenBackbone":
|
||||
# The extractor trains around the backbone; pretrained backbone behavior stays fixed.
|
||||
return super().train(False)
|
||||
|
||||
|
||||
class TinyFrozenBackbone(FrozenBackbone):
|
||||
"""Offline backbone used to exercise the complete pipeline in CPU CI."""
|
||||
|
||||
def __init__(self, hidden_dim: int = 64, patch_size: int = 16):
|
||||
super().__init__()
|
||||
generator = torch.Generator().manual_seed(1234)
|
||||
self.patch = nn.Conv2d(3, hidden_dim, patch_size, stride=patch_size, bias=False)
|
||||
self.byte_embedding = nn.Embedding(257, hidden_dim, padding_idx=0)
|
||||
with torch.no_grad():
|
||||
self.patch.weight.normal_(generator=generator, std=0.04)
|
||||
self.byte_embedding.weight.normal_(generator=generator, std=0.04)
|
||||
self.byte_embedding.weight[0].zero_()
|
||||
self.requires_grad_(False)
|
||||
self.vision_dim = hidden_dim
|
||||
self.text_dim = hidden_dim
|
||||
|
||||
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
with torch.no_grad():
|
||||
tokens = self.patch(images).flatten(2).transpose(1, 2)
|
||||
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
|
||||
return tokens, mask
|
||||
|
||||
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
encoded = []
|
||||
for text in texts:
|
||||
# Hash whole whitespace-delimited tokens so the offline representation preserves words.
|
||||
ids = []
|
||||
for token in text.lower().replace(".", "").split():
|
||||
digest = hashlib.blake2b(token.encode("utf-8"), digest_size=2).digest()
|
||||
ids.append((int.from_bytes(digest, "little") % 256) + 1)
|
||||
encoded.append(ids or [1])
|
||||
length = max(len(row) for row in encoded)
|
||||
input_ids = torch.zeros(len(encoded), length, dtype=torch.long, device=self.patch.weight.device)
|
||||
mask = torch.zeros_like(input_ids, dtype=torch.bool)
|
||||
for index, row in enumerate(encoded):
|
||||
input_ids[index, : len(row)] = torch.tensor(row, device=input_ids.device)
|
||||
mask[index, : len(row)] = True
|
||||
with torch.no_grad():
|
||||
tokens = self.byte_embedding(input_ids)
|
||||
return tokens, mask
|
||||
|
||||
|
||||
class CLIPFrozenBackbone(FrozenBackbone):
|
||||
"""Frozen CLIP token encoder for the first real GPU experiment."""
|
||||
|
||||
def __init__(self, model_name: str):
|
||||
super().__init__()
|
||||
try:
|
||||
from transformers import AutoTokenizer, CLIPModel
|
||||
except ImportError as error:
|
||||
raise RuntimeError("Install the 'hf' extra to use the CLIP backbone.") from error
|
||||
self.model = CLIPModel.from_pretrained(model_name)
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
self.model.requires_grad_(False).eval()
|
||||
self.vision_dim = self.model.config.vision_config.hidden_size
|
||||
self.text_dim = self.model.config.text_config.hidden_size
|
||||
mean = torch.tensor([0.48145466, 0.4578275, 0.40821073])
|
||||
std = torch.tensor([0.26862954, 0.26130258, 0.27577711])
|
||||
self.register_buffer("image_mean", mean.view(1, 3, 1, 1), persistent=False)
|
||||
self.register_buffer("image_std", std.view(1, 3, 1, 1), persistent=False)
|
||||
|
||||
def encode_image(self, images: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
pixels = (images - self.image_mean) / self.image_std
|
||||
with torch.no_grad():
|
||||
tokens = self.model.vision_model(pixel_values=pixels).last_hidden_state
|
||||
mask = torch.ones(tokens.shape[:2], dtype=torch.bool, device=tokens.device)
|
||||
return tokens, mask
|
||||
|
||||
def encode_text(self, texts: list[str]) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
values = self.tokenizer(texts, padding=True, truncation=True, return_tensors="pt")
|
||||
values = {key: value.to(self.image_mean.device) for key, value in values.items()}
|
||||
with torch.no_grad():
|
||||
tokens = self.model.text_model(**values).last_hidden_state
|
||||
return tokens, values["attention_mask"].bool()
|
||||
|
||||
|
||||
def build_backbone(name: str, clip_model: str) -> FrozenBackbone:
|
||||
if name == "tiny":
|
||||
return TinyFrozenBackbone()
|
||||
if name == "clip":
|
||||
return CLIPFrozenBackbone(clip_model)
|
||||
raise ValueError(f"Unknown backbone: {name}")
|
||||
@@ -0,0 +1,68 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
|
||||
class CrossAttentionBlock(nn.Module):
|
||||
def __init__(self, state_dim: int, num_heads: int, dropout: float):
|
||||
super().__init__()
|
||||
self.cross_attention = nn.MultiheadAttention(
|
||||
state_dim, num_heads, dropout=dropout, batch_first=True
|
||||
)
|
||||
self.norm1 = nn.LayerNorm(state_dim)
|
||||
self.ffn = nn.Sequential(
|
||||
nn.Linear(state_dim, state_dim * 4),
|
||||
nn.GELU(),
|
||||
nn.Dropout(dropout),
|
||||
nn.Linear(state_dim * 4, state_dim),
|
||||
)
|
||||
self.norm2 = nn.LayerNorm(state_dim)
|
||||
|
||||
def forward(
|
||||
self, slots: torch.Tensor, tokens: torch.Tensor, token_mask: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
update, _ = self.cross_attention(
|
||||
slots, tokens, tokens, key_padding_mask=~token_mask, need_weights=False
|
||||
)
|
||||
slots = self.norm1(slots + update)
|
||||
return self.norm2(slots + self.ffn(slots))
|
||||
|
||||
|
||||
class SlotExtractor(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
input_dim: int,
|
||||
state_dim: int,
|
||||
num_slots: int,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
dropout: float,
|
||||
num_subjects: int,
|
||||
num_predicates: int,
|
||||
num_objects: int,
|
||||
):
|
||||
super().__init__()
|
||||
self.input_projection = nn.Linear(input_dim, state_dim)
|
||||
self.slot_queries = nn.Parameter(torch.randn(num_slots, state_dim) * 0.02)
|
||||
self.blocks = nn.ModuleList(
|
||||
[CrossAttentionBlock(state_dim, num_heads, dropout) for _ in range(num_layers)]
|
||||
)
|
||||
self.subject_head = nn.Linear(state_dim, num_subjects)
|
||||
self.predicate_head = nn.Linear(state_dim, num_predicates)
|
||||
self.object_head = nn.Linear(state_dim, num_objects)
|
||||
self.valid_head = nn.Linear(state_dim, 2)
|
||||
|
||||
def forward(self, tokens: torch.Tensor, token_mask: torch.Tensor) -> dict[str, torch.Tensor]:
|
||||
tokens = self.input_projection(tokens)
|
||||
slots = self.slot_queries.unsqueeze(0).expand(tokens.shape[0], -1, -1)
|
||||
for block in self.blocks:
|
||||
slots = block(slots, tokens, token_mask)
|
||||
return {
|
||||
"states": slots,
|
||||
"subject_logits": self.subject_head(slots),
|
||||
"predicate_logits": self.predicate_head(slots),
|
||||
"object_logits": self.object_head(slots),
|
||||
"valid_logits": self.valid_head(slots),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
[project]
|
||||
name = "composable-evidence-states-research"
|
||||
version = "0.1.0"
|
||||
description = "Research MVP for extracting composable evidence states from images and text."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10,<3.13"
|
||||
dependencies = [
|
||||
"numpy>=1.26",
|
||||
"pillow>=10.0",
|
||||
"pyyaml>=6.0",
|
||||
"scipy>=1.11",
|
||||
"torch>=2.3",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
hf = ["transformers>=4.48"]
|
||||
dev = ["pytest>=8.0", "ruff>=0.8"]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
testpaths = ["tests"]
|
||||
addopts = "-q"
|
||||
|
||||
[tool.ruff]
|
||||
line-length = 100
|
||||
target-version = "py310"
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E", "F", "I"]
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
from data.generate_rules import add_rule
|
||||
from data.generate_scenes import generate_scene
|
||||
|
||||
|
||||
def test_scene_generation_is_deterministic():
|
||||
first = generate_scene(7, seed=17)
|
||||
second = generate_scene(7, seed=17)
|
||||
assert first == second
|
||||
assert len(first["visual_facts"]) == 5
|
||||
|
||||
|
||||
def test_held_out_rule_changes_combination():
|
||||
scene = generate_scene(3, seed=17)
|
||||
train = add_rule(scene, seed=17, held_out=False)
|
||||
held_out = add_rule(scene, seed=17, held_out=True)
|
||||
assert train["combination"] != held_out["combination"]
|
||||
assert train["answer"] == held_out["answer"]
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import torch
|
||||
|
||||
from losses.matching_loss import hungarian_matching_loss
|
||||
from models.slot_extractor import SlotExtractor
|
||||
|
||||
|
||||
def test_slot_extractor_and_matching_loss_backward():
|
||||
extractor = SlotExtractor(
|
||||
input_dim=32,
|
||||
state_dim=64,
|
||||
num_slots=4,
|
||||
num_layers=2,
|
||||
num_heads=4,
|
||||
dropout=0.0,
|
||||
num_subjects=12,
|
||||
num_predicates=7,
|
||||
num_objects=12,
|
||||
)
|
||||
tokens = torch.randn(2, 10, 32)
|
||||
mask = torch.ones(2, 10, dtype=torch.bool)
|
||||
predictions = extractor(tokens, mask)
|
||||
targets = [torch.tensor([[0, 0, 4], [1, 1, 5]]), torch.tensor([[1, 0, 6]])]
|
||||
result = hungarian_matching_loss(predictions, targets)
|
||||
assert torch.isfinite(result.loss)
|
||||
result.loss.backward()
|
||||
assert extractor.slot_queries.grad is not None
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import yaml
|
||||
from torch.utils.data import DataLoader
|
||||
|
||||
from data.dataset import EvidenceDataset, collate_evidence
|
||||
from eval.evaluate_slots import evaluate
|
||||
from losses.alignment_loss import supervised_alignment_loss
|
||||
from losses.matching_loss import hungarian_matching_loss
|
||||
from models.evidence_model import EvidenceExtractorModel
|
||||
|
||||
|
||||
def set_seed(seed: int) -> None:
|
||||
random.seed(seed)
|
||||
np.random.seed(seed)
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
def train(config: dict[str, Any], backbone_name: str, epochs: int | None = None) -> Path:
|
||||
set_seed(config["seed"])
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
data_root = Path(config["data"]["root"])
|
||||
train_dataset = EvidenceDataset(data_root / "train.jsonl", config["data"]["image_size"])
|
||||
validation_dataset = EvidenceDataset(
|
||||
data_root / "validation.jsonl", config["data"]["image_size"]
|
||||
)
|
||||
loader_options = {
|
||||
"batch_size": config["training"]["batch_size"],
|
||||
"num_workers": config["training"]["num_workers"],
|
||||
"collate_fn": collate_evidence,
|
||||
}
|
||||
train_loader = DataLoader(train_dataset, shuffle=True, **loader_options)
|
||||
validation_loader = DataLoader(validation_dataset, **loader_options)
|
||||
model = EvidenceExtractorModel(config, backbone_name=backbone_name).to(device)
|
||||
trainable = [parameter for parameter in model.parameters() if parameter.requires_grad]
|
||||
optimizer = torch.optim.AdamW(
|
||||
trainable,
|
||||
lr=config["training"]["learning_rate"],
|
||||
weight_decay=config["training"]["weight_decay"],
|
||||
)
|
||||
output_dir = Path(config["training"]["output_dir"])
|
||||
if backbone_name != config["model"]["backbone"]:
|
||||
output_dir = output_dir.parent / backbone_name
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
best_path = output_dir / "best.pt"
|
||||
best_score = -1.0
|
||||
epoch_count = epochs or config["training"]["epochs"]
|
||||
|
||||
for epoch in range(1, epoch_count + 1):
|
||||
model.train()
|
||||
running_loss = 0.0
|
||||
for batch in train_loader:
|
||||
optimizer.zero_grad(set_to_none=True)
|
||||
outputs = model(batch["images"].to(device), batch["rules_text"])
|
||||
visual = hungarian_matching_loss(
|
||||
outputs["visual"], batch["visual_facts"], config["training"]["valid_weight"]
|
||||
)
|
||||
text = hungarian_matching_loss(
|
||||
outputs["text"], batch["text_facts"], config["training"]["valid_weight"]
|
||||
)
|
||||
alignment = supervised_alignment_loss(
|
||||
outputs["visual"]["states"], batch["visual_facts"], visual.assignments
|
||||
) + supervised_alignment_loss(
|
||||
outputs["text"]["states"], batch["text_facts"], text.assignments
|
||||
)
|
||||
loss = visual.loss + text.loss + config["training"]["alignment_weight"] * alignment
|
||||
loss.backward()
|
||||
torch.nn.utils.clip_grad_norm_(trainable, 1.0)
|
||||
optimizer.step()
|
||||
running_loss += float(loss.detach())
|
||||
metrics = evaluate(model, validation_loader, device)
|
||||
score = metrics["visual"]["f1"] + metrics["text"]["f1"]
|
||||
summary = {
|
||||
"epoch": epoch,
|
||||
"loss": running_loss / max(1, len(train_loader)),
|
||||
"metrics": metrics,
|
||||
}
|
||||
print(json.dumps(summary))
|
||||
if score > best_score:
|
||||
best_score = score
|
||||
torch.save(
|
||||
{"model": model.state_dict(), "epoch": epoch, "metrics": metrics}, best_path
|
||||
)
|
||||
return best_path
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--backbone", choices=("tiny", "clip"), default=None)
|
||||
parser.add_argument("--epochs", type=int, default=None)
|
||||
args = parser.parse_args()
|
||||
config = yaml.safe_load(args.config.read_text(encoding="utf-8"))
|
||||
backbone_name = args.backbone or config["model"]["backbone"]
|
||||
print(train(config, backbone_name, epochs=args.epochs))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
Reference in New Issue
Block a user