Build initial logic skeleton benchmark

logicbench/cli.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from .client import LLMClient, LLMClientConfig
+from .pipeline import run_evaluation
+from .skeletons import generate_skeletons
+
+
+def _load_task(path: Path) -> dict[str, Any]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    required = {"count", "length", "seed", "output_dir"}
+    missing = required - payload.keys()
+    if missing:
+        raise ValueError(f"task is missing fields: {', '.join(sorted(missing))}")
+    return payload
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="logicbench")
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    generate = subparsers.add_parser("generate", help="generate answer-bearing skeleton JSONL")
+    generate.add_argument("--count", type=int, default=3)
+    generate.add_argument("--length", default="short")
+    generate.add_argument("--seed", type=int, default=0)
+    generate.add_argument("--output", type=Path)
+
+    run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file")
+    run.add_argument("task", type=Path)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    args = build_parser().parse_args(argv)
+    if args.command == "generate":
+        skeletons = generate_skeletons(args.count, args.length, args.seed)
+        lines = [json.dumps(item.to_dict(), ensure_ascii=False) for item in skeletons]
+        text = "\n".join(lines) + "\n"
+        if args.output:
+            args.output.parent.mkdir(parents=True, exist_ok=True)
+            args.output.write_text(text, encoding="utf-8")
+        else:
+            print(text, end="")
+        return 0
+
+    task = _load_task(args.task)
+    renderer = LLMClient(LLMClientConfig.from_env("renderer"))
+    solver = LLMClient(LLMClientConfig.from_env("solver"))
+    summary = run_evaluation(
+        renderer=renderer,
+        solver=solver,
+        count=int(task["count"]),
+        length=task["length"],
+        seed=int(task["seed"]),
+        output_dir=Path(task["output_dir"]),
+    )
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+    return 0