Build stage-one evidence extractor MVP
CI / test (push) Failing after 11m13s

This commit is contained in:
2026-10-01 22:16:29 +08:00
commit d30892168e
26 changed files with 2250 additions and 0 deletions
+2
View File
@@ -0,0 +1,2 @@
"""Synthetic data generation and loading."""
+70
View File
@@ -0,0 +1,70 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
import yaml
from data.generate_rules import add_rule
from data.generate_scenes import generate_scene
from data.render_scene import render_scene
def _write_split(
name: str,
size: int,
start: int,
root: Path,
seed: int,
image_size: int,
held_out: bool,
) -> None:
path = root / f"{name}.jsonl"
with path.open("w", encoding="utf-8") as handle:
for scene_number in range(start, start + size):
record = add_rule(generate_scene(scene_number, seed), seed, held_out=held_out)
relative_image = Path("images") / f"{record['scene_id']}.png"
render_scene(record, root / relative_image, image_size=image_size)
record["image"] = str(relative_image)
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def build(config_path: Path, small: bool = False) -> Path:
config = yaml.safe_load(config_path.read_text(encoding="utf-8"))
root = Path(config["data"]["root"])
root.mkdir(parents=True, exist_ok=True)
sizes = {
"train": config["data"]["train_size"],
"validation": config["data"]["validation_size"],
"test_id": config["data"]["test_id_size"],
"test_ood": config["data"]["test_ood_size"],
}
if small:
sizes = {"train": 64, "validation": 16, "test_id": 16, "test_ood": 16}
cursor = 0
for split, size in sizes.items():
_write_split(
split,
size,
cursor,
root,
config["seed"],
config["data"]["image_size"],
held_out=split == "test_ood",
)
cursor += size
return root
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--config", type=Path, default=Path("configs/mvp.yaml"))
parser.add_argument("--small", action="store_true")
args = parser.parse_args()
print(build(args.config, small=args.small))
if __name__ == "__main__":
main()
+59
View File
@@ -0,0 +1,59 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from data.schema import OBJECT_TO_ID, PREDICATE_TO_ID, SUBJECT_TO_ID, Fact
def fact_to_ids(value: dict[str, str]) -> tuple[int, int, int]:
fact = Fact.from_dict(value)
return (
SUBJECT_TO_ID[fact.subject],
PREDICATE_TO_ID[fact.predicate],
OBJECT_TO_ID[fact.object],
)
class EvidenceDataset(Dataset):
def __init__(self, manifest: Path, image_size: int = 224):
self.root = manifest.parent
self.image_size = image_size
self.records = [json.loads(line) for line in manifest.read_text(encoding="utf-8").splitlines()]
def __len__(self) -> int:
return len(self.records)
def __getitem__(self, index: int) -> dict[str, Any]:
record = self.records[index]
image = Image.open(self.root / record["image"]).convert("RGB")
image = image.resize((self.image_size, self.image_size))
image_tensor = torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1).float() / 255.0
return {
"image": image_tensor,
"rules_text": record["rules_text"],
"visual_facts": torch.tensor(
[fact_to_ids(value) for value in record["visual_facts"]], dtype=torch.long
),
"text_facts": torch.tensor(
[fact_to_ids(value) for value in record["text_facts"]], dtype=torch.long
),
"scene_id": record["scene_id"],
}
def collate_evidence(batch: list[dict[str, Any]]) -> dict[str, Any]:
return {
"images": torch.stack([item["image"] for item in batch]),
"rules_text": [item["rules_text"] for item in batch],
"visual_facts": [item["visual_facts"] for item in batch],
"text_facts": [item["text_facts"] for item in batch],
"scene_id": [item["scene_id"] for item in batch],
}
+28
View File
@@ -0,0 +1,28 @@
from __future__ import annotations
import random
from typing import Any
from data.schema import CLASSES, Fact
def add_rule(record: dict[str, Any], seed: int, held_out: bool = False) -> dict[str, Any]:
rng = random.Random((seed * 9_999_991) + int(record["scene_id"].split("-")[-1]))
target = rng.choice(record["objects"])
color = target["color"]
color_index = ("red", "blue", "green", "yellow").index(color)
class_index = (color_index + (1 if held_out else 0)) % len(CLASSES)
class_name = CLASSES[class_index]
rule = Fact(color, "implies_class", class_name)
result = dict(record)
result.update(
{
"rules_text": f"Every {color} object belongs to class {class_name}.",
"text_facts": [rule.to_dict()],
"question": f"Which object belongs to class {class_name}?",
"answer": target["object_id"],
"combination": f"{color}->{class_name}",
}
)
return result
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import random
from typing import Any
from data.schema import COLORS, OBJECT_IDS, RELATIONS, SHAPES, Fact, SceneObject
def _relation(first: SceneObject, second: SceneObject, rng: random.Random) -> Fact:
relation = rng.choice(RELATIONS)
if relation == "left_of":
first = SceneObject(first.object_id, first.color, first.shape, 55, 112)
second = SceneObject(second.object_id, second.color, second.shape, 169, 112)
elif relation == "right_of":
first = SceneObject(first.object_id, first.color, first.shape, 169, 112)
second = SceneObject(second.object_id, second.color, second.shape, 55, 112)
elif relation == "above":
first = SceneObject(first.object_id, first.color, first.shape, 112, 55)
second = SceneObject(second.object_id, second.color, second.shape, 112, 169)
else:
first = SceneObject(first.object_id, first.color, first.shape, 112, 169)
second = SceneObject(second.object_id, second.color, second.shape, 112, 55)
return Fact(first.object_id, relation, second.object_id), first, second
def generate_scene(scene_id: int, seed: int, num_objects: int = 2) -> dict[str, Any]:
if num_objects != 2:
raise ValueError("The stage-one MVP currently supports exactly two objects.")
rng = random.Random((seed * 1_000_003) + scene_id)
colors = rng.sample(COLORS, k=num_objects)
shapes = rng.sample(SHAPES, k=num_objects)
objects = [
SceneObject(OBJECT_IDS[index], colors[index], shapes[index], 0, 0)
for index in range(num_objects)
]
relation, first, second = _relation(objects[0], objects[1], rng)
objects = [first, second]
facts = []
for item in objects:
facts.extend(
[
Fact(item.object_id, "color", item.color),
Fact(item.object_id, "shape", item.shape),
]
)
facts.append(relation)
return {
"scene_id": f"scene-{scene_id:07d}",
"objects": [item.to_dict() for item in objects],
"visual_facts": [fact.to_dict() for fact in facts],
}
+52
View File
@@ -0,0 +1,52 @@
from __future__ import annotations
import math
from pathlib import Path
from typing import Any
from PIL import Image, ImageDraw
RGB = {
"red": (220, 55, 55),
"blue": (55, 100, 220),
"green": (45, 165, 85),
"yellow": (235, 195, 45),
}
def _polygon(cx: int, cy: int, radius: int, sides: int, rotation: float = -math.pi / 2):
return [
(
cx + radius * math.cos(rotation + (2 * math.pi * index / sides)),
cy + radius * math.sin(rotation + (2 * math.pi * index / sides)),
)
for index in range(sides)
]
def render_scene(record: dict[str, Any], output_path: Path, image_size: int = 224) -> None:
image = Image.new("RGB", (image_size, image_size), (248, 248, 248))
draw = ImageDraw.Draw(image)
scale = image_size / 224
radius = int(28 * scale)
for item in record["objects"]:
x, y = int(item["x"] * scale), int(item["y"] * scale)
color = RGB[item["color"]]
box = (x - radius, y - radius, x + radius, y + radius)
if item["shape"] == "circle":
draw.ellipse(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
elif item["shape"] == "square":
draw.rectangle(box, fill=color, outline=(20, 20, 20), width=max(1, int(2 * scale)))
elif item["shape"] == "triangle":
draw.polygon(_polygon(x, y, radius, 3), fill=color, outline=(20, 20, 20))
else:
points = []
for index in range(10):
current_radius = radius if index % 2 == 0 else radius * 0.45
angle = -math.pi / 2 + index * math.pi / 5
points.append((x + current_radius * math.cos(angle), y + current_radius * math.sin(angle)))
draw.polygon(points, fill=color, outline=(20, 20, 20))
draw.text((x - radius, y + radius + int(4 * scale)), item["object_id"], fill=(20, 20, 20))
output_path.parent.mkdir(parents=True, exist_ok=True)
image.save(output_path)
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
from dataclasses import asdict, dataclass
from typing import Any
COLORS = ("red", "blue", "green", "yellow")
SHAPES = ("circle", "square", "triangle", "star")
RELATIONS = ("left_of", "right_of", "above", "below")
CLASSES = ("alpha", "beta", "gamma", "delta")
OBJECT_IDS = ("obj1", "obj2", "obj3")
PREDICATES = ("color", "shape", "left_of", "right_of", "above", "below", "implies_class")
ENTITIES = OBJECT_IDS + COLORS + SHAPES + CLASSES
SUBJECT_TO_ID = {value: index for index, value in enumerate(ENTITIES)}
PREDICATE_TO_ID = {value: index for index, value in enumerate(PREDICATES)}
OBJECT_TO_ID = SUBJECT_TO_ID
@dataclass(frozen=True)
class Fact:
subject: str
predicate: str
object: str
def to_dict(self) -> dict[str, str]:
return asdict(self)
@classmethod
def from_dict(cls, value: dict[str, str]) -> "Fact":
return cls(**value)
@dataclass(frozen=True)
class SceneObject:
object_id: str
color: str
shape: str
x: int
y: int
def to_dict(self) -> dict[str, Any]:
return asdict(self)