Build initial logic skeleton benchmark

logicbench/models.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class Fact:
+    predicate: str
+    arguments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class Query:
+    predicate: str
+    arguments: tuple[str, ...]
+
+
+@dataclass(frozen=True)
+class Skeleton:
+    id: str
+    family: str
+    complexity: int
+    entities: dict[str, str]
+    facts: tuple[Fact, ...]
+    rules: tuple[str, ...]
+    query: Query
+    answer_options: tuple[str, ...]
+    gold_answer: str
+    proof: tuple[str, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    def renderer_payload(self) -> dict[str, Any]:
+        """Return everything needed to render the problem, but never the answer."""
+        payload = self.to_dict()
+        payload.pop("gold_answer")
+        payload.pop("proof")
+        return payload