Build initial logic skeleton benchmark
This commit is contained in:
@@ -0,0 +1,136 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .models import Skeleton
|
||||
from .skeletons import generate_skeletons
|
||||
|
||||
|
||||
class JsonModel(Protocol):
|
||||
def request_json(
|
||||
self,
|
||||
*,
|
||||
system: str,
|
||||
user: str,
|
||||
schema_name: str,
|
||||
schema: dict[str, Any],
|
||||
) -> dict[str, Any]: ...
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EvaluationResult:
|
||||
skeleton_id: str
|
||||
family: str
|
||||
complexity: int
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
model_answer: str
|
||||
correct: bool
|
||||
|
||||
|
||||
RENDER_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {"problem": {"type": "string"}},
|
||||
"required": ["problem"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"You turn typed logical intermediate representations into self-contained natural-language "
|
||||
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
|
||||
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
|
||||
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
|
||||
"End by requiring exactly one of the supplied answer labels."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
schema=RENDER_SCHEMA,
|
||||
)
|
||||
problem = response.get("problem")
|
||||
if not isinstance(problem, str) or not problem.strip():
|
||||
raise RuntimeError("renderer returned an empty problem")
|
||||
return problem.strip()
|
||||
|
||||
|
||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"answer": {"type": "string", "enum": list(answer_options)},
|
||||
},
|
||||
"required": ["answer"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"Solve the reasoning problem from the stated facts only. Return exactly one permitted answer "
|
||||
"label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
user=problem,
|
||||
schema_name="solver_answer",
|
||||
schema=schema,
|
||||
)
|
||||
answer = response.get("answer")
|
||||
if answer not in answer_options:
|
||||
raise RuntimeError(f"solver returned invalid answer label: {answer!r}")
|
||||
return str(answer)
|
||||
|
||||
|
||||
def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None:
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
for record in records:
|
||||
handle.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
|
||||
def run_evaluation(
|
||||
*,
|
||||
renderer: JsonModel,
|
||||
solver: JsonModel,
|
||||
count: int,
|
||||
length: str | int,
|
||||
seed: int,
|
||||
output_dir: Path,
|
||||
) -> dict[str, Any]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
skeletons = generate_skeletons(count=count, length=length, seed=seed)
|
||||
_write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
|
||||
|
||||
results: list[EvaluationResult] = []
|
||||
for skeleton in skeletons:
|
||||
problem = render_skeleton(renderer, skeleton)
|
||||
answer = solve_problem(solver, problem, skeleton.answer_options)
|
||||
results.append(
|
||||
EvaluationResult(
|
||||
skeleton_id=skeleton.id,
|
||||
family=skeleton.family,
|
||||
complexity=skeleton.complexity,
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
model_answer=answer,
|
||||
correct=answer == skeleton.gold_answer,
|
||||
)
|
||||
)
|
||||
|
||||
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
|
||||
correct = sum(item.correct for item in results)
|
||||
summary = {
|
||||
"count": len(results),
|
||||
"correct": correct,
|
||||
"accuracy": correct / len(results),
|
||||
"length": length,
|
||||
"complexity": skeletons[0].complexity,
|
||||
"seed": seed,
|
||||
}
|
||||
(output_dir / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
)
|
||||
return summary
|
||||
Reference in New Issue
Block a user