From bd5bac07de49c9b4dc7dbbdb3d1ce30943c2603f Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Tue, 22 Sep 2026 21:03:19 +0800 Subject: [PATCH] Build initial logic skeleton benchmark --- .env.example | 14 ++++ .gitea/workflows/short-eval.yml | 32 ++++++++ .gitignore | 8 ++ README.md | 83 +++++++++++++++++++ logicbench/__init__.py | 6 ++ logicbench/__main__.py | 3 + logicbench/cli.py | 62 +++++++++++++++ logicbench/client.py | 121 ++++++++++++++++++++++++++++ logicbench/models.py | 40 ++++++++++ logicbench/pipeline.py | 136 ++++++++++++++++++++++++++++++++ logicbench/skeletons.py | 111 ++++++++++++++++++++++++++ pyproject.toml | 19 +++++ tasks/short-smoke.json | 8 ++ tests/test_pipeline.py | 46 +++++++++++ tests/test_skeletons.py | 37 +++++++++ 15 files changed, 726 insertions(+) create mode 100644 .env.example create mode 100644 .gitea/workflows/short-eval.yml create mode 100644 .gitignore create mode 100644 README.md create mode 100644 logicbench/__init__.py create mode 100644 logicbench/__main__.py create mode 100644 logicbench/cli.py create mode 100644 logicbench/client.py create mode 100644 logicbench/models.py create mode 100644 logicbench/pipeline.py create mode 100644 logicbench/skeletons.py create mode 100644 pyproject.toml create mode 100644 tasks/short-smoke.json create mode 100644 tests/test_pipeline.py create mode 100644 tests/test_skeletons.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a7a4be2 --- /dev/null +++ b/.env.example @@ -0,0 +1,14 @@ +# Shared fallback. Store the real value as a Gitea Actions secret, never here. +OPENAI_API_KEY= +OPENAI_BASE_URL=https://api.openai.com/v1 + +# Renderer and solver can use different OpenAI-compatible providers/models. +RENDERER_API_KEY= +RENDERER_BASE_URL= +RENDERER_API_STYLE=responses +RENDERER_MODEL= + +SOLVER_API_KEY= +SOLVER_BASE_URL= +SOLVER_API_STYLE=responses +SOLVER_MODEL= diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml new file mode 100644 index 0000000..a03f5dc --- /dev/null +++ b/.gitea/workflows/short-eval.yml @@ -0,0 +1,32 @@ +name: Short abstraction evaluation + +on: + workflow_dispatch: + +jobs: + short-eval: + runs-on: logic-bench + timeout-minutes: 15 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + RENDERER_API_KEY: ${{ secrets.RENDERER_API_KEY }} + RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} + RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} + RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} + SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} + SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} + SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} + SOLVER_MODEL: ${{ vars.SOLVER_MODEL }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Run unit tests + run: python3 -m unittest discover -s tests -v + + - name: Run three short live cases + run: python3 -m logicbench run tasks/short-smoke.json + + - name: Show evaluation summary + if: always() + run: test -f artifacts/short-smoke/summary.json && cat artifacts/short-smoke/summary.json || true diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..371103c --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +__pycache__/ +*.py[cod] +*.egg-info/ +.venv/ +.env +artifacts/ +dist/ +build/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..c0a7d5f --- /dev/null +++ b/README.md @@ -0,0 +1,83 @@ +# Logic Skeleton Bench + +Logic Skeleton Bench tests whether a solver transfers the same abstract structure across different surface stories. It first creates a typed intermediate representation with a deterministic gold answer, asks one LLM to render that representation as a natural-language problem, asks another LLM to solve only the rendered problem, and then scores the answer. + +The renderer and solver are intentionally separate clients. Each can use a different model, API endpoint, key, and OpenAI-compatible API style, so model-pair heatmaps can be added without changing the skeleton generator. + +## What the first version contains + +Three finite skeleton families are included: + +- `transitive_order`: a chain of directional relations; +- `membership_inheritance`: membership propagated through subset relations; +- `sequential_state_update`: ordered arithmetic state changes. + +Every generated skeleton contains its facts, rules, query, permitted answer labels, deterministic `gold_answer`, and proof trace. The renderer receives a copy with `gold_answer` and `proof` removed. + +`length` means logical complexity, not natural-language tokens. It is the number of proof-bearing facts or state updates. Presets are `short=2`, `medium=4`, and `long=6`; an integer from 1 through 20 is also accepted. + +## Local use + +Python 3.11 or newer is sufficient; the package has no runtime dependencies. + +Generate three short skeletons without calling an LLM: + +```bash +python3 -m logicbench generate --count 3 --length short --seed 20260922 +``` + +Run the checked-in live task: + +```bash +export OPENAI_API_KEY='...' +export RENDERER_MODEL='your-renderer-model' +export SOLVER_MODEL='your-solver-model' +python3 -m logicbench run tasks/short-smoke.json +``` + +Role-specific values override the shared fallback: + +| Variable | Meaning | +| --- | --- | +| `RENDERER_API_KEY`, `SOLVER_API_KEY` | Optional separate keys; each falls back to `OPENAI_API_KEY` | +| `RENDERER_BASE_URL`, `SOLVER_BASE_URL` | Optional endpoints; each falls back to `OPENAI_BASE_URL`, then `https://api.openai.com/v1` | +| `RENDERER_API_STYLE`, `SOLVER_API_STYLE` | `responses` (default) or `chat_completions` | +| `RENDERER_MODEL`, `SOLVER_MODEL` | Required, independently selected model IDs | + +Outputs are written under the task's `output_dir`: + +- `skeletons.jsonl` is written before any LLM call and includes gold answers; +- `results.jsonl` contains rendered problems, predictions, and per-case scores; +- `summary.json` contains aggregate accuracy and complexity metadata. + +## Gitea Actions one-shot test + +The workflow `.gitea/workflows/short-eval.yml` has only `workflow_dispatch`, so it consumes model quota only when someone manually clicks **Run workflow**. It runs three cases with `length: short` and has a 15-minute timeout. + +Create these repository Actions settings: + +1. Secret `OPENAI_API_KEY`, or separate secrets `RENDERER_API_KEY` and `SOLVER_API_KEY`. +2. Variables `RENDERER_MODEL` and `SOLVER_MODEL`. +3. Optional variables `RENDERER_BASE_URL`, `SOLVER_BASE_URL`, `RENDERER_API_STYLE`, and `SOLVER_API_STYLE`. Leave base URLs empty for OpenAI; use `responses` as the API style unless a compatible provider only supports chat completions. + +The workflow targets `runs-on: logic-bench`. Add this label to the existing `act_runner` using one of these methods: + +- In the Gitea UI, open the runner settings, edit its labels, and add `logic-bench:docker://docker.gitea.com/runner-images:ubuntu-latest`. +- Or add the same string to `runner.labels` in the runner's `config.yaml`, preserving existing labels, then restart `act_runner` with that config. +- Or start the daemon with `--labels` / `GITEA_RUNNER_LABELS`; this has higher precedence than the config file and must include every label the runner should continue advertising. + +Example config fragment: + +```yaml +runner: + labels: + - "logic-bench:docker://docker.gitea.com/runner-images:ubuntu-latest" +``` + +After restarting the runner, verify that `logic-bench` appears in its Gitea settings, then manually dispatch **Short abstraction evaluation** from the repository's Actions page. + +## Tests + +```bash +python3 -m unittest discover -s tests -v +``` diff --git a/logicbench/__init__.py b/logicbench/__init__.py new file mode 100644 index 0000000..f091257 --- /dev/null +++ b/logicbench/__init__.py @@ -0,0 +1,6 @@ +"""Logic skeleton benchmark.""" + +from .models import Fact, Query, Skeleton +from .skeletons import generate_skeletons, resolve_length + +__all__ = ["Fact", "Query", "Skeleton", "generate_skeletons", "resolve_length"] diff --git a/logicbench/__main__.py b/logicbench/__main__.py new file mode 100644 index 0000000..eb53e2f --- /dev/null +++ b/logicbench/__main__.py @@ -0,0 +1,3 @@ +from .cli import main + +raise SystemExit(main()) diff --git a/logicbench/cli.py b/logicbench/cli.py new file mode 100644 index 0000000..4145877 --- /dev/null +++ b/logicbench/cli.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from .client import LLMClient, LLMClientConfig +from .pipeline import run_evaluation +from .skeletons import generate_skeletons + + +def _load_task(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + required = {"count", "length", "seed", "output_dir"} + missing = required - payload.keys() + if missing: + raise ValueError(f"task is missing fields: {', '.join(sorted(missing))}") + return payload + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="logicbench") + subparsers = parser.add_subparsers(dest="command", required=True) + + generate = subparsers.add_parser("generate", help="generate answer-bearing skeleton JSONL") + generate.add_argument("--count", type=int, default=3) + generate.add_argument("--length", default="short") + generate.add_argument("--seed", type=int, default=0) + generate.add_argument("--output", type=Path) + + run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file") + run.add_argument("task", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = build_parser().parse_args(argv) + if args.command == "generate": + skeletons = generate_skeletons(args.count, args.length, args.seed) + lines = [json.dumps(item.to_dict(), ensure_ascii=False) for item in skeletons] + text = "\n".join(lines) + "\n" + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text, encoding="utf-8") + else: + print(text, end="") + return 0 + + task = _load_task(args.task) + renderer = LLMClient(LLMClientConfig.from_env("renderer")) + solver = LLMClient(LLMClientConfig.from_env("solver")) + summary = run_evaluation( + renderer=renderer, + solver=solver, + count=int(task["count"]), + length=task["length"], + seed=int(task["seed"]), + output_dir=Path(task["output_dir"]), + ) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 diff --git a/logicbench/client.py b/logicbench/client.py new file mode 100644 index 0000000..1e6fd67 --- /dev/null +++ b/logicbench/client.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class LLMClientConfig: + model: str + api_key: str + base_url: str = "https://api.openai.com/v1" + api_style: str = "responses" + timeout_seconds: int = 120 + + @classmethod + def from_env(cls, role: str) -> "LLMClientConfig": + prefix = role.upper() + model = os.getenv(f"{prefix}_MODEL", "").strip() + api_key = os.getenv(f"{prefix}_API_KEY", "").strip() or os.getenv("OPENAI_API_KEY", "").strip() + base_url = ( + os.getenv(f"{prefix}_BASE_URL", "").strip() + or os.getenv("OPENAI_BASE_URL", "").strip() + or "https://api.openai.com/v1" + ) + api_style = os.getenv(f"{prefix}_API_STYLE", "").strip() or "responses" + if not model: + raise ValueError(f"{prefix}_MODEL is required") + if not api_key: + raise ValueError(f"{prefix}_API_KEY or OPENAI_API_KEY is required") + if api_style not in {"responses", "chat_completions"}: + raise ValueError(f"{prefix}_API_STYLE must be responses or chat_completions") + return cls(model=model, api_key=api_key, base_url=base_url.rstrip("/"), api_style=api_style) + + +class LLMClient: + """Small dependency-free client for OpenAI Responses and compatible chat APIs.""" + + def __init__(self, config: LLMClientConfig): + self.config = config + + def request_json( + self, + *, + system: str, + user: str, + schema_name: str, + schema: dict[str, Any], + ) -> dict[str, Any]: + if self.config.api_style == "responses": + path = "/responses" + body = { + "model": self.config.model, + "input": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "text": { + "format": { + "type": "json_schema", + "name": schema_name, + "strict": True, + "schema": schema, + } + }, + } + else: + path = "/chat/completions" + body = { + "model": self.config.model, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + "response_format": { + "type": "json_schema", + "json_schema": {"name": schema_name, "strict": True, "schema": schema}, + }, + } + + request = urllib.request.Request( + self.config.base_url + path, + data=json.dumps(body).encode("utf-8"), + headers={ + "Authorization": f"Bearer {self.config.api_key}", + "Content-Type": "application/json", + }, + method="POST", + ) + try: + with urllib.request.urlopen(request, timeout=self.config.timeout_seconds) as response: + payload = json.loads(response.read()) + except urllib.error.HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace")[:1000] + raise RuntimeError(f"LLM API returned HTTP {exc.code}: {detail}") from exc + except urllib.error.URLError as exc: + raise RuntimeError(f"LLM API request failed: {exc.reason}") from exc + + text = self._extract_text(payload) + try: + result = json.loads(text) + except json.JSONDecodeError as exc: + raise RuntimeError("LLM API returned non-JSON structured output") from exc + if not isinstance(result, dict): + raise RuntimeError("LLM API structured output must be an object") + return result + + def _extract_text(self, payload: dict[str, Any]) -> str: + if self.config.api_style == "chat_completions": + return payload["choices"][0]["message"]["content"] + if isinstance(payload.get("output_text"), str): + return payload["output_text"] + for item in payload.get("output", []): + if item.get("type") == "message": + for content in item.get("content", []): + if content.get("type") == "output_text": + return content["text"] + raise RuntimeError("LLM API response did not contain output text") diff --git a/logicbench/models.py b/logicbench/models.py new file mode 100644 index 0000000..b606e2b --- /dev/null +++ b/logicbench/models.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +from dataclasses import asdict, dataclass +from typing import Any + + +@dataclass(frozen=True) +class Fact: + predicate: str + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class Query: + predicate: str + arguments: tuple[str, ...] + + +@dataclass(frozen=True) +class Skeleton: + id: str + family: str + complexity: int + entities: dict[str, str] + facts: tuple[Fact, ...] + rules: tuple[str, ...] + query: Query + answer_options: tuple[str, ...] + gold_answer: str + proof: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return asdict(self) + + def renderer_payload(self) -> dict[str, Any]: + """Return everything needed to render the problem, but never the answer.""" + payload = self.to_dict() + payload.pop("gold_answer") + payload.pop("proof") + return payload diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py new file mode 100644 index 0000000..d135e1b --- /dev/null +++ b/logicbench/pipeline.py @@ -0,0 +1,136 @@ +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Protocol + +from .models import Skeleton +from .skeletons import generate_skeletons + + +class JsonModel(Protocol): + def request_json( + self, + *, + system: str, + user: str, + schema_name: str, + schema: dict[str, Any], + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True) +class EvaluationResult: + skeleton_id: str + family: str + complexity: int + rendered_problem: str + answer_options: tuple[str, ...] + gold_answer: str + model_answer: str + correct: bool + + +RENDER_SCHEMA = { + "type": "object", + "properties": {"problem": {"type": "string"}}, + "required": ["problem"], + "additionalProperties": False, +} + + +def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str: + prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2) + response = client.request_json( + system=( + "You turn typed logical intermediate representations into self-contained natural-language " + "reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but " + "preserve every fact, its order when order matters, the rules, the query, and the exact answer " + "labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. " + "End by requiring exactly one of the supplied answer labels." + ), + user=f"Render this intermediate representation:\n{prompt}", + schema_name="rendered_problem", + schema=RENDER_SCHEMA, + ) + problem = response.get("problem") + if not isinstance(problem, str) or not problem.strip(): + raise RuntimeError("renderer returned an empty problem") + return problem.strip() + + +def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str: + schema = { + "type": "object", + "properties": { + "answer": {"type": "string", "enum": list(answer_options)}, + }, + "required": ["answer"], + "additionalProperties": False, + } + response = client.request_json( + system=( + "Solve the reasoning problem from the stated facts only. Return exactly one permitted answer " + "label. Do not use unstated real-world assumptions." + ), + user=problem, + schema_name="solver_answer", + schema=schema, + ) + answer = response.get("answer") + if answer not in answer_options: + raise RuntimeError(f"solver returned invalid answer label: {answer!r}") + return str(answer) + + +def _write_jsonl(path: Path, records: list[dict[str, Any]]) -> None: + with path.open("w", encoding="utf-8") as handle: + for record in records: + handle.write(json.dumps(record, ensure_ascii=False) + "\n") + + +def run_evaluation( + *, + renderer: JsonModel, + solver: JsonModel, + count: int, + length: str | int, + seed: int, + output_dir: Path, +) -> dict[str, Any]: + output_dir.mkdir(parents=True, exist_ok=True) + skeletons = generate_skeletons(count=count, length=length, seed=seed) + _write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons]) + + results: list[EvaluationResult] = [] + for skeleton in skeletons: + problem = render_skeleton(renderer, skeleton) + answer = solve_problem(solver, problem, skeleton.answer_options) + results.append( + EvaluationResult( + skeleton_id=skeleton.id, + family=skeleton.family, + complexity=skeleton.complexity, + rendered_problem=problem, + answer_options=skeleton.answer_options, + gold_answer=skeleton.gold_answer, + model_answer=answer, + correct=answer == skeleton.gold_answer, + ) + ) + + _write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results]) + correct = sum(item.correct for item in results) + summary = { + "count": len(results), + "correct": correct, + "accuracy": correct / len(results), + "length": length, + "complexity": skeletons[0].complexity, + "seed": seed, + } + (output_dir / "summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return summary diff --git a/logicbench/skeletons.py b/logicbench/skeletons.py new file mode 100644 index 0000000..b8ffaa0 --- /dev/null +++ b/logicbench/skeletons.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import random +from collections.abc import Callable + +from .models import Fact, Query, Skeleton + + +LENGTH_PRESETS = {"short": 2, "medium": 4, "long": 6} + + +def resolve_length(value: str | int) -> int: + """Resolve a named preset or an explicit number of proof-bearing operations.""" + if isinstance(value, int): + result = value + elif value in LENGTH_PRESETS: + result = LENGTH_PRESETS[value] + else: + try: + result = int(value) + except ValueError as exc: + choices = ", ".join(LENGTH_PRESETS) + raise ValueError(f"length must be an integer or one of: {choices}") from exc + if not 1 <= result <= 20: + raise ValueError("length must be between 1 and 20") + return result + + +def _order_chain(index: int, complexity: int, rng: random.Random) -> Skeleton: + nodes = [f"E{i}" for i in range(complexity + 1)] + facts = [Fact("LEFT_OF", (nodes[i], nodes[i + 1])) for i in range(complexity)] + rng.shuffle(facts) + return Skeleton( + id=f"order-{index:04d}", + family="transitive_order", + complexity=complexity, + entities={node: "Entity" for node in nodes}, + facts=tuple(facts), + rules=("LEFT_OF is transitive", "RIGHT_OF(x,y) iff LEFT_OF(y,x)"), + query=Query("RELATION", (nodes[0], nodes[-1])), + answer_options=("LEFT_OF", "RIGHT_OF", "UNKNOWN"), + gold_answer="LEFT_OF", + proof=tuple(f"LEFT_OF({nodes[i]},{nodes[i + 1]})" for i in range(complexity)), + ) + + +def _set_chain(index: int, complexity: int, rng: random.Random) -> Skeleton: + item = "E0" + sets = [f"S{i}" for i in range(complexity)] + facts = [Fact("MEMBER_OF", (item, sets[0]))] + facts.extend(Fact("SUBSET_OF", (sets[i], sets[i + 1])) for i in range(complexity - 1)) + rng.shuffle(facts) + return Skeleton( + id=f"set-{index:04d}", + family="membership_inheritance", + complexity=complexity, + entities={item: "Entity", **{name: "Set" for name in sets}}, + facts=tuple(facts), + rules=("MEMBER_OF(x,A) and SUBSET_OF(A,B) imply MEMBER_OF(x,B)",), + query=Query("MEMBER_OF", (item, sets[-1])), + answer_options=("YES", "NO", "UNKNOWN"), + gold_answer="YES", + proof=tuple( + [f"MEMBER_OF({item},{sets[0]})"] + + [f"SUBSET_OF({sets[i]},{sets[i + 1]})" for i in range(complexity - 1)] + ), + ) + + +def _state_updates(index: int, complexity: int, rng: random.Random) -> Skeleton: + start = rng.randint(3, 9) + operations: list[tuple[str, int]] = [] + value = start + for _ in range(complexity): + amount = rng.randint(1, 4) + predicate = rng.choice(("INCREMENT", "DECREMENT")) + if predicate == "DECREMENT" and value - amount < 0: + predicate = "INCREMENT" + value = value + amount if predicate == "INCREMENT" else value - amount + operations.append((predicate, amount)) + + facts = [Fact("INITIAL_VALUE", ("E0", str(start)))] + facts.extend(Fact(predicate, ("E0", str(amount))) for predicate, amount in operations) + lower_distractor = value - 1 if value > 0 else value + 2 + return Skeleton( + id=f"state-{index:04d}", + family="sequential_state_update", + complexity=complexity, + entities={"E0": "Counter"}, + facts=tuple(facts), + rules=("Apply updates in the listed order",), + query=Query("FINAL_VALUE", ("E0",)), + answer_options=(str(value), str(value + 1), str(lower_distractor)), + gold_answer=str(value), + proof=tuple(f"{predicate}({amount})" for predicate, amount in operations), + ) + + +GENERATORS: tuple[Callable[[int, int, random.Random], Skeleton], ...] = ( + _order_chain, + _set_chain, + _state_updates, +) + + +def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[Skeleton]: + if count < 1: + raise ValueError("count must be positive") + complexity = resolve_length(length) + rng = random.Random(seed) + return [GENERATORS[i % len(GENERATORS)](i, complexity, rng) for i in range(count)] diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..28781d0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "logic-skeleton-bench" +version = "0.1.0" +description = "Generate logical skeletons, render them as stories, and evaluate LLM solvers." +readme = "README.md" +requires-python = ">=3.11" +license = { text = "MIT" } +authors = [{ name = "Codex Bot", email = "codex-bot@gitea.entroia.local" }] +dependencies = [] + +[project.scripts] +logicbench = "logicbench.cli:main" + +[tool.setuptools.packages.find] +include = ["logicbench*"] diff --git a/tasks/short-smoke.json b/tasks/short-smoke.json new file mode 100644 index 0000000..8a5ff48 --- /dev/null +++ b/tasks/short-smoke.json @@ -0,0 +1,8 @@ +{ + "name": "short-smoke", + "description": "Render and solve one short example from each initial skeleton family.", + "count": 3, + "length": "short", + "seed": 20260922, + "output_dir": "artifacts/short-smoke" +} diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py new file mode 100644 index 0000000..ea04616 --- /dev/null +++ b/tests/test_pipeline.py @@ -0,0 +1,46 @@ +import tempfile +import unittest +from pathlib import Path + +from logicbench.pipeline import run_evaluation + + +class QueueClient: + def __init__(self, responses): + self.responses = iter(responses) + + def request_json(self, **_kwargs): + return next(self.responses) + + +class PipelineTests(unittest.TestCase): + def test_end_to_end_writes_skeletons_before_scored_results(self): + renderer = QueueClient([ + {"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."}, + {"problem": "Choose YES, NO, or UNKNOWN."}, + {"problem": "Choose 9, 10, or 8."}, + ]) + # Seed 20260922 makes the third gold answer 9. + solver = QueueClient([ + {"answer": "LEFT_OF"}, + {"answer": "YES"}, + {"answer": "9"}, + ]) + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) + summary = run_evaluation( + renderer=renderer, + solver=solver, + count=3, + length="short", + seed=20260922, + output_dir=output, + ) + self.assertTrue((output / "skeletons.jsonl").is_file()) + self.assertTrue((output / "results.jsonl").is_file()) + self.assertEqual(summary["count"], 3) + self.assertEqual(summary["correct"], 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_skeletons.py b/tests/test_skeletons.py new file mode 100644 index 0000000..73f638c --- /dev/null +++ b/tests/test_skeletons.py @@ -0,0 +1,37 @@ +import unittest + +from logicbench.skeletons import generate_skeletons, resolve_length + + +class SkeletonTests(unittest.TestCase): + def test_short_means_two_proof_bearing_operations(self): + skeletons = generate_skeletons(count=3, length="short", seed=42) + self.assertEqual([item.complexity for item in skeletons], [2, 2, 2]) + self.assertEqual([item.family for item in skeletons], [ + "transitive_order", + "membership_inheritance", + "sequential_state_update", + ]) + self.assertTrue(all(len(item.proof) == 2 for item in skeletons)) + + def test_generation_is_deterministic(self): + first = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)] + second = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)] + self.assertEqual(first, second) + + def test_renderer_never_receives_gold_or_proof(self): + skeleton = generate_skeletons(1, "short", seed=1)[0] + payload = skeleton.renderer_payload() + self.assertNotIn("gold_answer", payload) + self.assertNotIn("proof", payload) + self.assertIn("answer_options", payload) + + def test_length_validation(self): + self.assertEqual(resolve_length("medium"), 4) + self.assertEqual(resolve_length("5"), 5) + with self.assertRaises(ValueError): + resolve_length("huge") + + +if __name__ == "__main__": + unittest.main()