This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""Synthetic data generation and loading."""
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
from data.generate_rules import add_rule
|
||||
from data.generate_scenes import generate_scene
|
||||
from data.render_scene import render_scene
|
||||
|
||||
|
||||
def _write_split(
|
||||
name: str,
|
||||
size: int,
|
||||
start: int,
|
||||
root: Path,
|
||||
seed: int,
|
||||
image_size: int,
|
||||
held_out: bool,
|
||||
) -> None:
|
||||
path = root / f"{name}.jsonl"
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for scene_number in range(start, start + size):
|
||||
record = add_rule(generate_scene(scene_number, seed), seed, held_out=held_out)
|
||||
relative_image = Path("images") / f"{record['scene_id']}.png"
|
||||
render_scene(record, root / relative_image, image_size=image_size)
|
||||
record["image"] = str(relative_image)
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def build(config_path: Path, small: bool = False) -> Path:
|
||||
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
|
||||
root = Path(config["data"]["root"])
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
sizes = {
|
||||
"train": config["data"]["train_size"],
|
||||
"validation": config["data"]["validation_size"],
|
||||
"test_id": config["data"]["test_id_size"],
|
||||
"test_ood": config["data"]["test_ood_size"],
|
||||
}
|
||||
if small:
|
||||
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
|
||||
cursor = 0
|
||||
for split, size in sizes.items():
|
||||
_write_split(
|
||||
split,
|
||||
size,
|
||||
cursor,
|
||||
root,
|
||||
config["seed"],
|
||||
config["data"]["image_size"],
|
||||
held_out=split == "test_ood",
|
||||
)
|
||||
cursor += size
|
||||
return root
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
|
||||
parser.add_argument("--small", action="store_true")
|
||||
args = parser.parse_args()
|
||||
print(build(args.config, small=args.small))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
from torch.utils.data import Dataset
|
||||
|
||||
from data.schema import OBJECT_TO_ID, PREDICATE_TO_ID, SUBJECT_TO_ID, Fact
|
||||
|
||||
|
||||
def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int]:
|
||||
fact = Fact.from_dict(value)
|
||||
return (
|
||||
SUBJECT_TO_ID[fact.subject],
|
||||
PREDICATE_TO_ID[fact.predicate],
|
||||
OBJECT_TO_ID[fact.object],
|
||||
)
|
||||
|
||||
|
||||
class EvidenceDataset(Dataset):
|
||||
def __init__(self, manifest: Path, image_size: int = 224):
|
||||
self.root = manifest.parent
|
||||
self.image_size = image_size
|
||||
self.records = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
|
||||
|
||||
def __len__(self) -> int:
|
||||
return len(self.records)
|
||||
|
||||
def __getitem__(self, index: int) -> dict[str, Any]:
|
||||
record = self.records[index]
|
||||
image = Image.open(self.root / record["image"]).convert("RGB")
|
||||
image = image.resize((self.image_size, self.image_size))
|
||||
image_tensor = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1).float() / 255.0
|
||||
return {
|
||||
"image": image_tensor,
|
||||
"rules_text": record["rules_text"],
|
||||
"visual_facts": torch.tensor(
|
||||
[fact_to_ids(value) for value in record["visual_facts"]], dtype=torch.long
|
||||
),
|
||||
"text_facts": torch.tensor(
|
||||
[fact_to_ids(value) for value in record["text_facts"]], dtype=torch.long
|
||||
),
|
||||
"scene_id": record["scene_id"],
|
||||
}
|
||||
|
||||
|
||||
def collate_evidence(batch: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
return {
|
||||
"images": torch.stack([item["image"] for item in batch]),
|
||||
"rules_text": [item["rules_text"] for item in batch],
|
||||
"visual_facts": [item["visual_facts"] for item in batch],
|
||||
"text_facts": [item["text_facts"] for item in batch],
|
||||
"scene_id": [item["scene_id"] for item in batch],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
from data.schema import CLASSES, Fact
|
||||
|
||||
|
||||
def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[str, Any]:
|
||||
rng = random.Random((seed * 9_999_991) + int(record["scene_id"].split("-")[-1]))
|
||||
target = rng.choice(record["objects"])
|
||||
color = target["color"]
|
||||
color_index = ("red", "blue", "green", "yellow").index(color)
|
||||
class_index = (color_index + (1 if held_out else 0)) % len(CLASSES)
|
||||
class_name = CLASSES[class_index]
|
||||
rule = Fact(color, "implies_class", class_name)
|
||||
result = dict(record)
|
||||
result.update(
|
||||
{
|
||||
"rules_text": f"Every {color} object belongs to class {class_name}.",
|
||||
"text_facts": [rule.to_dict()],
|
||||
"question": f"Which object belongs to class {class_name}?",
|
||||
"answer": target["object_id"],
|
||||
"combination": f"{color}->{class_name}",
|
||||
}
|
||||
)
|
||||
return result
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from typing import Any
|
||||
|
||||
from data.schema import COLORS, OBJECT_IDS, RELATIONS, SHAPES, Fact, SceneObject
|
||||
|
||||
|
||||
def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fact:
|
||||
relation = rng.choice(RELATIONS)
|
||||
if relation == "left_of":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 55, 112)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 169, 112)
|
||||
elif relation == "right_of":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 169, 112)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 55, 112)
|
||||
elif relation == "above":
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 112, 55)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 112, 169)
|
||||
else:
|
||||
first = SceneObject(first.object_id, first.color, first.shape, 112, 169)
|
||||
second = SceneObject(second.object_id, second.color, second.shape, 112, 55)
|
||||
return Fact(first.object_id, relation, second.object_id), first, second
|
||||
|
||||
|
||||
def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, Any]:
|
||||
if num_objects != 2:
|
||||
raise ValueError("The stage-one MVP currently supports exactly two objects.")
|
||||
rng = random.Random((seed * 1_000_003) + scene_id)
|
||||
colors = rng.sample(COLORS, k=num_objects)
|
||||
shapes = rng.sample(SHAPES, k=num_objects)
|
||||
objects = [
|
||||
SceneObject(OBJECT_IDS[index], colors[index], shapes[index], 0, 0)
|
||||
for index in range(num_objects)
|
||||
]
|
||||
relation, first, second = _relation(objects[0], objects[1], rng)
|
||||
objects = [first, second]
|
||||
facts = []
|
||||
for item in objects:
|
||||
facts.extend(
|
||||
[
|
||||
Fact(item.object_id, "color", item.color),
|
||||
Fact(item.object_id, "shape", item.shape),
|
||||
]
|
||||
)
|
||||
facts.append(relation)
|
||||
return {
|
||||
"scene_id": f"scene-{scene_id:07d}",
|
||||
"objects": [item.to_dict() for item in objects],
|
||||
"visual_facts": [fact.to_dict() for fact in facts],
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from PIL import Image, ImageDraw
|
||||
|
||||
RGB = {
|
||||
"red": (220, 55, 55),
|
||||
"blue": (55, 100, 220),
|
||||
"green": (45, 165, 85),
|
||||
"yellow": (235, 195, 45),
|
||||
}
|
||||
|
||||
|
||||
def _polygon(cx: int, cy: int, radius: int, sides: int, rotation: float = -math.pi / 2):
|
||||
return [
|
||||
(
|
||||
cx + radius * math.cos(rotation + (2 * math.pi * index / sides)),
|
||||
cy + radius * math.sin(rotation + (2 * math.pi * index / sides)),
|
||||
)
|
||||
for index in range(sides)
|
||||
]
|
||||
|
||||
|
||||
def render_scene(record: dict[str, Any], output_path: Path, image_size: int = 224) -> None:
|
||||
image = Image.new("RGB", (image_size, image_size), (248, 248, 248))
|
||||
draw = ImageDraw.Draw(image)
|
||||
scale = image_size / 224
|
||||
radius = int(28 * scale)
|
||||
for item in record["objects"]:
|
||||
x, y = int(item["x"] * scale), int(item["y"] * scale)
|
||||
color = RGB[item["color"]]
|
||||
box = (x - radius, y - radius, x + radius, y + radius)
|
||||
if item["shape"] == "circle":
|
||||
draw.ellipse(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
|
||||
elif item["shape"] == "square":
|
||||
draw.rectangle(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
|
||||
elif item["shape"] == "triangle":
|
||||
draw.polygon(_polygon(x, y, radius, 3), fill=color, outline=(20, 20, 20))
|
||||
else:
|
||||
points = []
|
||||
for index in range(10):
|
||||
current_radius = radius if index % 2 == 0 else radius * 0.45
|
||||
angle = -math.pi / 2 + index * math.pi / 5
|
||||
points.append((x + current_radius * math.cos(angle), y + current_radius * math.sin(angle)))
|
||||
draw.polygon(points, fill=color, outline=(20, 20, 20))
|
||||
draw.text((x - radius, y + radius + int(4 * scale)), item["object_id"], fill=(20, 20, 20))
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.save(output_path)
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Any
|
||||
|
||||
COLORS = ("red", "blue", "green", "yellow")
|
||||
SHAPES = ("circle", "square", "triangle", "star")
|
||||
RELATIONS = ("left_of", "right_of", "above", "below")
|
||||
CLASSES = ("alpha", "beta", "gamma", "delta")
|
||||
OBJECT_IDS = ("obj1", "obj2", "obj3")
|
||||
|
||||
PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "implies_class")
|
||||
ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES
|
||||
|
||||
SUBJECT_TO_ID = {value: index for index, value in enumerate(ENTITIES)}
|
||||
PREDICATE_TO_ID = {value: index for index, value in enumerate(PREDICATES)}
|
||||
OBJECT_TO_ID = SUBJECT_TO_ID
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fact:
|
||||
subject: str
|
||||
predicate: str
|
||||
object: str
|
||||
|
||||
def to_dict(self) -> dict[str, str]:
|
||||
return asdict(self)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, value: dict[str, str]) -> "Fact":
|
||||
return cls(**value)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class SceneObject:
|
||||
object_id: str
|
||||
color: str
|
||||
shape: str
|
||||
x: int
|
||||
y: int
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
Reference in New Issue
Block a user