Build initial logic skeleton benchmark

This commit is contained in:
2026-09-22 21:03:19 +08:00
commit bd5bac07de
15 changed files with 726 additions and 0 deletions
+6
View File
@@ -0,0 +1,6 @@
"""Logic skeleton benchmark."""
from .models import Fact, Query, Skeleton
from .skeletons import generate_skeletons, resolve_length
__all__ = ["Fact", "Query", "Skeleton", "generate_skeletons", "resolve_length"]
+3
View File
@@ -0,0 +1,3 @@
from .cli import main
raise SystemExit(main())
+62
View File
@@ -0,0 +1,62 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
from .client import LLMClient, LLMClientConfig
from .pipeline import run_evaluation
from .skeletons import generate_skeletons
def _load_task(path: Path) -> dict[str, Any]:
payload = json.loads(path.read_text(encoding="utf-8"))
required = {"count", "length", "seed", "output_dir"}
missing = required - payload.keys()
if missing:
raise ValueError(f"task is missing fields: {', '.join(sorted(missing))}")
return payload
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="logicbench")
subparsers = parser.add_subparsers(dest="command", required=True)
generate = subparsers.add_parser("generate", help="generate answer-bearing skeleton JSONL")
generate.add_argument("--count", type=int, default=3)
generate.add_argument("--length", default="short")
generate.add_argument("--seed", type=int, default=0)
generate.add_argument("--output", type=Path)
run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file")
run.add_argument("task", type=Path)
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
if args.command == "generate":
skeletons = generate_skeletons(args.count, args.length, args.seed)
lines = [json.dumps(item.to_dict(), ensure_ascii=False) for item in skeletons]
text = "\n".join(lines) + "\n"
if args.output:
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(text, encoding="utf-8")
else:
print(text, end="")
return 0
task = _load_task(args.task)
renderer = LLMClient(LLMClientConfig.from_env("renderer"))
solver = LLMClient(LLMClientConfig.from_env("solver"))
summary = run_evaluation(
renderer=renderer,
solver=solver,
count=int(task["count"]),
length=task["length"],
seed=int(task["seed"]),
output_dir=Path(task["output_dir"]),
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
+121
View File
@@ -0,0 +1,121 @@
from __future__ import annotations
import json
import os
import urllib.error
import urllib.request
from dataclasses import dataclass
from typing import Any
@dataclass(frozen=True)
class LLMClientConfig:
model: str
api_key: str
base_url: str = "https://api.openai.com/v1"
api_style: str = "responses"
timeout_seconds: int = 120
@classmethod
def from_env(cls, role: str) -> "LLMClientConfig":
prefix = role.upper()
model = os.getenv(f"{prefix}_MODEL", "").strip()
api_key = os.getenv(f"{prefix}_API_KEY", "").strip() or os.getenv("OPENAI_API_KEY", "").strip()
base_url = (
os.getenv(f"{prefix}_BASE_URL", "").strip()
or os.getenv("OPENAI_BASE_URL", "").strip()
or "https://api.openai.com/v1"
)
api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses"
if not model:
raise ValueError(f"{prefix}_MODEL is required")
if not api_key:
raise ValueError(f"{prefix}_API_KEY or OPENAI_API_KEY is required")
if api_style not in {"responses", "chat_completions"}:
raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions")
return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style)
class LLMClient:
"""Small dependency-free client for OpenAI Responses and compatible chat APIs."""
def __init__(self, config: LLMClientConfig):
self.config = config
def request_json(
self,
*,
system: str,
user: str,
schema_name: str,
schema: dict[str, Any],
) -> dict[str, Any]:
if self.config.api_style == "responses":
path = "/responses"
body = {
"model": self.config.model,
"input": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
"text": {
"format": {
"type": "json_schema",
"name": schema_name,
"strict": True,
"schema": schema,
}
},
}
else:
path = "/chat/completions"
body = {
"model": self.config.model,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
"response_format": {
"type": "json_schema",
"json_schema": {"name": schema_name, "strict": True, "schema": schema},
},
}
request = urllib.request.Request(
self.config.base_url + path,
data=json.dumps(body).encode("utf-8"),
headers={
"Authorization": f"Bearer {self.config.api_key}",
"Content-Type": "application/json",
},
method="POST",
)
try:
with urllib.request.urlopen(request, timeout=self.config.timeout_seconds) as response:
payload = json.loads(response.read())
except urllib.error.HTTPError as exc:
detail = exc.read().decode("utf-8", errors="replace")[:1000]
raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc
except urllib.error.URLError as exc:
raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc
text = self._extract_text(payload)
try:
result = json.loads(text)
except json.JSONDecodeError as exc:
raise RuntimeError("LLM API returned non-JSON structured output") from exc
if not isinstance(result, dict):
raise RuntimeError("LLM API structured output must be an object")
return result
def _extract_text(self, payload: dict[str, Any]) -> str:
if self.config.api_style == "chat_completions":
return payload["choices"][0]["message"]["content"]
if isinstance(payload.get("output_text"), str):
return payload["output_text"]
for item in payload.get("output", []):
if item.get("type") == "message":
for content in item.get("content", []):
if content.get("type") == "output_text":
return content["text"]
raise RuntimeError("LLM API response did not contain output text")
+40
View File
@@ -0,0 +1,40 @@
from __future__ import annotations
from dataclasses import asdict, dataclass
from typing import Any
@dataclass(frozen=True)
class Fact:
predicate: str
arguments: tuple[str, ...]
@dataclass(frozen=True)
class Query:
predicate: str
arguments: tuple[str, ...]
@dataclass(frozen=True)
class Skeleton:
id: str
family: str
complexity: int
entities: dict[str, str]
facts: tuple[Fact, ...]
rules: tuple[str, ...]
query: Query
answer_options: tuple[str, ...]
gold_answer: str
proof: tuple[str, ...]
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def renderer_payload(self) -> dict[str, Any]:
"""Return everything needed to render the problem, but never the answer."""
payload = self.to_dict()
payload.pop("gold_answer")
payload.pop("proof")
return payload
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
import json
from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Protocol
from .models import Skeleton
from .skeletons import generate_skeletons
class JsonModel(Protocol):
def request_json(
self,
*,
system: str,
user: str,
schema_name: str,
schema: dict[str, Any],
) -> dict[str, Any]: ...
@dataclass(frozen=True)
class EvaluationResult:
skeleton_id: str
family: str
complexity: int
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
model_answer: str
correct: bool
RENDER_SCHEMA = {
"type": "object",
"properties": {"problem": {"type": "string"}},
"required": ["problem"],
"additionalProperties": False,
}
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
response = client.request_json(
system=(
"You turn typed logical intermediate representations into self-contained natural-language "
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
"End by requiring exactly one of the supplied answer labels."
),
user=f"Render this intermediate representation:\n{prompt}",
schema_name="rendered_problem",
schema=RENDER_SCHEMA,
)
problem = response.get("problem")
if not isinstance(problem, str) or not problem.strip():
raise RuntimeError("renderer returned an empty problem")
return problem.strip()
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
schema = {
"type": "object",
"properties": {
"answer": {"type": "string", "enum": list(answer_options)},
},
"required": ["answer"],
"additionalProperties": False,
}
response = client.request_json(
system=(
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
"label. Do not use unstated real-world assumptions."
),
user=problem,
schema_name="solver_answer",
schema=schema,
)
answer = response.get("answer")
if answer not in answer_options:
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
return str(answer)
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
with path.open("w", encoding="utf-8") as handle:
for record in records:
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
def run_evaluation(
*,
renderer: JsonModel,
solver: JsonModel,
count: int,
length: str | int,
seed: int,
output_dir: Path,
) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
skeletons = generate_skeletons(count=count, length=length, seed=seed)
_write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
results: list[EvaluationResult] = []
for skeleton in skeletons:
problem = render_skeleton(renderer, skeleton)
answer = solve_problem(solver, problem, skeleton.answer_options)
results.append(
EvaluationResult(
skeleton_id=skeleton.id,
family=skeleton.family,
complexity=skeleton.complexity,
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
model_answer=answer,
correct=answer == skeleton.gold_answer,
)
)
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
correct = sum(item.correct for item in results)
summary = {
"count": len(results),
"correct": correct,
"accuracy": correct / len(results),
"length": length,
"complexity": skeletons[0].complexity,
"seed": seed,
}
(output_dir / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
)
return summary
+111
View File
@@ -0,0 +1,111 @@
from __future__ import annotations
import random
from collections.abc import Callable
from .models import Fact, Query, Skeleton
LENGTH_PRESETS = {"short": 2, "medium": 4, "long": 6}
def resolve_length(value: str | int) -> int:
"""Resolve a named preset or an explicit number of proof-bearing operations."""
if isinstance(value, int):
result = value
elif value in LENGTH_PRESETS:
result = LENGTH_PRESETS[value]
else:
try:
result = int(value)
except ValueError as exc:
choices = ", ".join(LENGTH_PRESETS)
raise ValueError(f"length must be an integer or one of: {choices}") from exc
if not 1 <= result <= 20:
raise ValueError("length must be between 1 and 20")
return result
def _order_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
nodes = [f"E{i}" for i in range(complexity + 1)]
facts = [Fact("LEFT_OF", (nodes[i], nodes[i + 1])) for i in range(complexity)]
rng.shuffle(facts)
return Skeleton(
id=f"order-{index:04d}",
family="transitive_order",
complexity=complexity,
entities={node: "Entity" for node in nodes},
facts=tuple(facts),
rules=("LEFT_OF is transitive", "RIGHT_OF(x,y) iff LEFT_OF(y,x)"),
query=Query("RELATION", (nodes[0], nodes[-1])),
answer_options=("LEFT_OF", "RIGHT_OF", "UNKNOWN"),
gold_answer="LEFT_OF",
proof=tuple(f"LEFT_OF({nodes[i]},{nodes[i + 1]})" for i in range(complexity)),
)
def _set_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
item = "E0"
sets = [f"S{i}" for i in range(complexity)]
facts = [Fact("MEMBER_OF", (item, sets[0]))]
facts.extend(Fact("SUBSET_OF", (sets[i], sets[i + 1])) for i in range(complexity - 1))
rng.shuffle(facts)
return Skeleton(
id=f"set-{index:04d}",
family="membership_inheritance",
complexity=complexity,
entities={item: "Entity", **{name: "Set" for name in sets}},
facts=tuple(facts),
rules=("MEMBER_OF(x,A) and SUBSET_OF(A,B) imply MEMBER_OF(x,B)",),
query=Query("MEMBER_OF", (item, sets[-1])),
answer_options=("YES", "NO", "UNKNOWN"),
gold_answer="YES",
proof=tuple(
[f"MEMBER_OF({item},{sets[0]})"]
+ [f"SUBSET_OF({sets[i]},{sets[i + 1]})" for i in range(complexity - 1)]
),
)
def _state_updates(index: int, complexity: int, rng: random.Random) -> Skeleton:
start = rng.randint(3, 9)
operations: list[tuple[str, int]] = []
value = start
for _ in range(complexity):
amount = rng.randint(1, 4)
predicate = rng.choice(("INCREMENT", "DECREMENT"))
if predicate == "DECREMENT" and value - amount < 0:
predicate = "INCREMENT"
value = value + amount if predicate == "INCREMENT" else value - amount
operations.append((predicate, amount))
facts = [Fact("INITIAL_VALUE", ("E0", str(start)))]
facts.extend(Fact(predicate, ("E0", str(amount))) for predicate, amount in operations)
lower_distractor = value - 1 if value > 0 else value + 2
return Skeleton(
id=f"state-{index:04d}",
family="sequential_state_update",
complexity=complexity,
entities={"E0": "Counter"},
facts=tuple(facts),
rules=("Apply updates in the listed order",),
query=Query("FINAL_VALUE", ("E0",)),
answer_options=(str(value), str(value + 1), str(lower_distractor)),
gold_answer=str(value),
proof=tuple(f"{predicate}({amount})" for predicate, amount in operations),
)
GENERATORS: tuple[Callable[[int, int, random.Random], Skeleton], ...] = (
_order_chain,
_set_chain,
_state_updates,
)
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[Skeleton]:
if count < 1:
raise ValueError("count must be positive")
complexity = resolve_length(length)
rng = random.Random(seed)
return [GENERATORS[i % len(GENERATORS)](i, complexity, rng) for i in range(count)]