diff --git a/.gitea/workflows/short-eval.yml b/.gitea/workflows/short-eval.yml index a650639..aa035b8 100644 --- a/.gitea/workflows/short-eval.yml +++ b/.gitea/workflows/short-eval.yml @@ -12,6 +12,7 @@ jobs: RENDERER_BASE_URL: ${{ vars.RENDERER_BASE_URL }} RENDERER_API_STYLE: ${{ vars.RENDERER_API_STYLE }} RENDERER_MODEL: ${{ vars.RENDERER_MODEL }} + RENDERER_ENABLE_THINKING: "false" SOLVER_API_KEY: ${{ secrets.SOLVER_API_KEY }} SOLVER_BASE_URL: ${{ vars.SOLVER_BASE_URL }} SOLVER_API_STYLE: ${{ vars.SOLVER_API_STYLE }} diff --git a/README.md b/README.md index fda7953..425dfee 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,12 @@ The full grammar and execution contract are defined in [the dynamic-world IR spe `entity_count` and `simulation_steps` are separate task fields. Explicit proof lengths from 3 through 10 are accepted. +`render_chunk_size` controls how many initial observations or realized events are sent in one renderer request. It defaults to `60`; the renderer first creates one shared setting plan, and every chunk receives the same deterministic symbol-to-name map. It can be set in the task JSON or overridden at run time: + +```bash +python3 -m logicbench run tasks/short-smoke.json --render-chunk-size 40 +``` + ## Local use Python 3.11 or newer is sufficient; there are no runtime dependencies. diff --git a/docs/skeleton-language.md b/docs/skeleton-language.md index 4f909d9..845dd51 100644 --- a/docs/skeleton-language.md +++ b/docs/skeleton-language.md @@ -216,7 +216,9 @@ proof event IDs or proof facts generator checks ``` -The renderer must map every symbol to one stable surface name, preserve all events and their direction and ordering, and produce one chronological fictional history. Every rendered initial fact and event line retains its `I####` or `V#####` audit ID. The evaluator rejects a rendering when any expected ID is absent. The renderer may compress formatting, but it may not summarize away distractors, state hidden rules, expose the proof, or directly assert the queried proposition. +Rendering uses one planning request followed by bounded history chunks. The plan fixes one setting and style, while the evaluator itself creates a deterministic symbol-to-name map and supplies that exact complete map to every chunk. `render_chunk_size` is the maximum records per chunk and defaults to `60`. It is configurable in a task file or through `--render-chunk-size`. + +The renderer must use the supplied stable surface names, preserve all events and their direction and ordering, and produce one chronological fictional history. Every rendered initial fact and event line retains its `I####` or `V#####` audit ID. The evaluator concatenates chunks in source order and rejects a rendering when any expected ID is absent. The renderer may compress formatting, but it may not summarize away distractors, state hidden rules, expose the proof, or directly assert the queried proposition. ## Complexity dimensions @@ -227,6 +229,7 @@ length minimum gold-proof events entity_count total typed entities simulation_steps candidate evolution steps event_count realized events; measured after simulation +render_chunk_size maximum records in one rendering request; default 60 ``` Named proof presets are `short=4`, `medium=6`, and `long=8`. The task file separately controls world size and simulation duration. diff --git a/logicbench/cli.py b/logicbench/cli.py index d72f3fa..8cfe9f9 100644 --- a/logicbench/cli.py +++ b/logicbench/cli.py @@ -35,6 +35,11 @@ def build_parser() -> argparse.ArgumentParser: run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file") run.add_argument("task", type=Path) + run.add_argument( + "--render-chunk-size", + type=int, + help="override the task's records per renderer request (default: 60)", + ) return parser @@ -60,6 +65,11 @@ def main(argv: list[str] | None = None) -> int: task = _load_task(args.task) renderer = LLMClient(LLMClientConfig.from_env("renderer")) solver = LLMClient(LLMClientConfig.from_env("solver")) + render_chunk_size = ( + args.render_chunk_size + if args.render_chunk_size is not None + else int(task.get("render_chunk_size", 60)) + ) summary = run_evaluation( renderer=renderer, solver=solver, @@ -69,6 +79,7 @@ def main(argv: list[str] | None = None) -> int: output_dir=Path(task["output_dir"]), entity_count=int(task.get("entity_count", 128)), simulation_steps=int(task.get("simulation_steps", 1000)), + render_chunk_size=render_chunk_size, ) print(json.dumps(summary, ensure_ascii=False, indent=2)) return 0 diff --git a/logicbench/pipeline.py b/logicbench/pipeline.py index dfe7d3b..f1d9b32 100644 --- a/logicbench/pipeline.py +++ b/logicbench/pipeline.py @@ -38,39 +38,129 @@ class EvaluationResult: correct: bool -RENDER_SCHEMA = { +RENDER_PLAN_SCHEMA = { "type": "object", - "properties": {"problem": {"type": "string"}}, - "required": ["problem"], + "properties": { + "setting_title": {"type": "string"}, + "setting_premise": {"type": "string"}, + "style_guide": {"type": "string"}, + }, + "required": ["setting_title", "setting_premise", "style_guide"], + "additionalProperties": False, +} + +RENDER_CHUNK_SCHEMA = { + "type": "object", + "properties": {"text": {"type": "string"}}, + "required": ["text"], "additionalProperties": False, } -def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str: - prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2) - response = client.request_json( +def _surface_names(skeleton: WorldSkeleton) -> dict[str, str]: + first = ("Zeru", "Parn", "Moki", "Tavi", "Luma", "Neri", "Kado", "Vela", "Sori", "Denu") + second = ("an", "or", "is", "um", "el", "ai", "on", "ar", "en", "ul") + names: dict[str, str] = {} + counters: dict[str, int] = {} + for entity in skeleton.entities: + index = counters.get(entity.type, 0) + counters[entity.type] = index + 1 + if entity.type == "Entity": + names[entity.id] = first[index % len(first)] + second[(index // len(first)) % len(second)] + elif entity.type == "Set": + names[entity.id] = f"Collection-{index + 1:02d}" + elif entity.type == "Region": + names[entity.id] = f"Chamber-{index + 1:02d}" + else: + names[entity.id] = f"Gauge-{index + 1:02d}" + return names + + +def render_skeleton( + client: JsonModel, + skeleton: WorldSkeleton, + *, + render_chunk_size: int = 60, +) -> str: + if render_chunk_size < 1: + raise ValueError("render_chunk_size must be positive") + payload = skeleton.renderer_payload() + names = _surface_names(skeleton) + type_counts: dict[str, int] = {} + for entity_type in payload["entity_types"].values(): + type_counts[entity_type] = type_counts.get(entity_type, 0) + 1 + plan = client.request_json( system=( - "Turn a large formal history from one simulated world into a self-contained natural-language " - "reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to " - "every symbol. Render every initial observation and every realized event exactly once, preserving " - "time order, participants, relation direction, numeric values, and state changes. Render the query " - "at the end. The events are facts that actually occurred, not repeated probability experiments. " - "Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring " - "instructions. Do not directly state the queried proposition. Begin each rendered fact or event line " - "with its original I#### or V##### audit ID. Use compact chronological log lines rather than " - "explanatory prose, but do not omit apparent distractors. State the answer labels; the " - "evaluator separately supplies the solver response protocol." + "Design a concise fictional setting for a large symbolic event history. Return only a title, a short " + "premise, and a style guide that maps the supplied relation and event vocabulary to consistent natural " + "language. Do not invent world facts, rules, probabilities, a query answer, or a proof." ), - user=f"Render this intermediate representation:\n{prompt}", - schema_name="rendered_problem", - schema=RENDER_SCHEMA, + user=json.dumps( + { + "entity_type_counts": type_counts, + "relations": [ + "LOCATED_IN", + "ABOVE", + "MEMBER", + "SUBSET", + "DISJOINT", + "CONNECTED", + ], + "event_kinds": ["MOVE", "UPDATE_NUMBER", "SET_STATE"], + }, + ensure_ascii=False, + ), + schema_name="rendering_plan", + schema=RENDER_PLAN_SCHEMA, ) - problem = response.get("problem") - if not isinstance(problem, str) or not problem.strip(): - raise RuntimeError("renderer returned an empty problem") - problem = problem.strip() - expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]] - expected_ids.extend(event.id for event in skeleton.events) + if not all(isinstance(plan.get(key), str) and plan[key].strip() for key in RENDER_PLAN_SCHEMA["required"]): + raise RuntimeError("renderer returned an incomplete rendering plan") + + records = [ + {"section": "initial_observation", **item} + for item in skeleton.initial_state["public_observations"] + ] + records.extend({"section": "event", **asdict(event)} for event in skeleton.events) + rendered_chunks: list[str] = [] + for start in range(0, len(records), render_chunk_size): + chunk = records[start : start + render_chunk_size] + response = client.request_json( + system=( + "Render the supplied slice of one world history as compact natural-language log lines. Use the " + "provided setting, style guide, and exact surface-name map. Render every record exactly once and " + "begin its line with the record's I#### or V##### audit ID. Preserve all participants, relation " + "directions, values, before/after states, and event order. Do not summarize, infer rules, answer a " + "query, or add facts. Return only the finished lines for this slice." + ), + user=json.dumps( + { + "rendering_plan": plan, + "surface_names": names, + "records": chunk, + }, + ensure_ascii=False, + ), + schema_name="rendered_history_chunk", + schema=RENDER_CHUNK_SCHEMA, + ) + text = response.get("text") + if not isinstance(text, str) or not text.strip(): + raise RuntimeError(f"renderer returned an empty history chunk beginning at record {start}") + rendered_chunks.append(text.strip()) + + query_names = [names.get(argument, argument) for argument in skeleton.query.arguments] + if skeleton.query.predicate == "ABOVE": + question = f"At the end of the history, is {query_names[0]} above {query_names[1]}?" + elif skeleton.query.predicate == "MEMBER": + question = f"At the end of the history, is {query_names[0]} a member of {query_names[1]}?" + else: + raise RuntimeError(f"unsupported rendered query predicate: {skeleton.query.predicate}") + problem = ( + f"# {plan['setting_title'].strip()}\n\n{plan['setting_premise'].strip()}\n\n" + + "\n".join(rendered_chunks) + + f"\n\nQuestion: {question}\nAnswer labels: {', '.join(skeleton.answer_options)}" + ) + expected_ids = [item["id"] for item in records] missing = [identifier for identifier in expected_ids if identifier not in problem] if missing: preview = ", ".join(missing[:10]) @@ -145,7 +235,10 @@ def run_evaluation( output_dir: Path, entity_count: int = 128, simulation_steps: int = 1000, + render_chunk_size: int = 60, ) -> dict[str, Any]: + if render_chunk_size < 1: + raise ValueError("render_chunk_size must be positive") output_dir.mkdir(parents=True, exist_ok=True) skeletons = generate_skeletons( count=count, @@ -160,7 +253,11 @@ def run_evaluation( rendered_records: list[dict[str, str]] = [] solver_records: list[dict[str, str]] = [] for skeleton in skeletons: - problem = render_skeleton(renderer, skeleton) + problem = render_skeleton( + renderer, + skeleton, + render_chunk_size=render_chunk_size, + ) rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem}) _write_jsonl(output_dir / "rendered.jsonl", rendered_records) solver_output = request_solver_output(solver, problem, skeleton.answer_options) @@ -206,6 +303,7 @@ def run_evaluation( "seed": seed, "entity_count": entity_count, "simulation_steps": simulation_steps, + "render_chunk_size": render_chunk_size, "event_counts": [len(item.events) for item in skeletons], } (output_dir / "summary.json").write_text( diff --git a/tasks/short-smoke.json b/tasks/short-smoke.json index 4c4ab40..6443950 100644 --- a/tasks/short-smoke.json +++ b/tasks/short-smoke.json @@ -5,6 +5,7 @@ "length": "short", "entity_count": 128, "simulation_steps": 1000, + "render_chunk_size": 60, "seed": 20260922, "output_dir": "artifacts/short-smoke" } diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 165d726..bfbc57c 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -23,11 +23,33 @@ class FailingSolver: class AuditRenderer: + def __init__(self): + self.chunk_sizes = [] + self.surface_name_maps = [] + def request_json(self, **kwargs): - payload = json.loads(kwargs["user"][kwargs["user"].find("{") :]) - identifiers = [item["id"] for item in payload["initial_observations"]] - identifiers.extend(item["id"] for item in payload["events"]) - return {"problem": "A compact behavioral problem.\n" + "\n".join(identifiers)} + if kwargs["schema_name"] == "rendering_plan": + return { + "setting_title": "Crystal Archive", + "setting_premise": "A compact behavioral problem.", + "style_guide": "Describe every record as an archive log entry.", + } + payload = json.loads(kwargs["user"]) + records = payload["records"] + self.chunk_sizes.append(len(records)) + self.surface_name_maps.append(payload["surface_names"]) + return {"text": "\n".join(f"{item['id']} rendered" for item in records)} + + +class OmittingRenderer: + def request_json(self, **kwargs): + if kwargs["schema_name"] == "rendering_plan": + return { + "setting_title": "Incomplete Archive", + "setting_premise": "Some records are missing.", + "style_guide": "Use log lines.", + } + return {"text": "This rendering omitted its audit IDs."} class PipelineTests(unittest.TestCase): @@ -49,6 +71,7 @@ class PipelineTests(unittest.TestCase): length="short", seed=20260922, output_dir=output, + render_chunk_size=17, ) self.assertTrue((output / "skeletons.jsonl").is_file()) self.assertTrue((output / "rendered.jsonl").is_file()) @@ -57,6 +80,12 @@ class PipelineTests(unittest.TestCase): self.assertEqual(summary["count"], 5) self.assertEqual(summary["correct"], 5) self.assertEqual(summary["format_compliant"], 5) + self.assertEqual(summary["render_chunk_size"], 17) + self.assertTrue(renderer.chunk_sizes) + self.assertLessEqual(max(renderer.chunk_sizes), 17) + self.assertTrue( + all(mapping == renderer.surface_name_maps[0] for mapping in renderer.surface_name_maps) + ) def test_answer_parser_requires_final_marker_and_permitted_label(self): self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES") @@ -105,7 +134,7 @@ class PipelineTests(unittest.TestCase): self.assertIn('"parse_error":', (output / "results.jsonl").read_text()) def test_renderer_omission_is_rejected_before_solving(self): - renderer = QueueClient([{"problem": "This rendering omitted its audit IDs."}]) + renderer = OmittingRenderer() with tempfile.TemporaryDirectory() as directory: with self.assertRaisesRegex(RuntimeError, "renderer omitted"): run_evaluation(