Render dynamic histories in configurable chunks

This commit is contained in:
2026-09-23 10:35:29 +08:00
parent 43aff1e2ee
commit a06ca45f30
7 changed files with 181 additions and 32 deletions
+11
View File
@@ -35,6 +35,11 @@ def build_parser() -> argparse.ArgumentParser:
run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file")
run.add_argument("task", type=Path)
run.add_argument(
"--render-chunk-size",
type=int,
help="override the task's records per renderer request (default: 60)",
)
return parser
@@ -60,6 +65,11 @@ def main(argv: list[str] | None = None) -> int:
task = _load_task(args.task)
renderer = LLMClient(LLMClientConfig.from_env("renderer"))
solver = LLMClient(LLMClientConfig.from_env("solver"))
render_chunk_size = (
args.render_chunk_size
if args.render_chunk_size is not None
else int(task.get("render_chunk_size", 60))
)
summary = run_evaluation(
renderer=renderer,
solver=solver,
@@ -69,6 +79,7 @@ def main(argv: list[str] | None = None) -> int:
output_dir=Path(task["output_dir"]),
entity_count=int(task.get("entity_count", 128)),
simulation_steps=int(task.get("simulation_steps", 1000)),
render_chunk_size=render_chunk_size,
)
print(json.dumps(summary, ensure_ascii=False, indent=2))
return 0
+124 -26
View File
@@ -38,39 +38,129 @@ class EvaluationResult:
correct: bool
RENDER_SCHEMA = {
RENDER_PLAN_SCHEMA = {
"type": "object",
"properties": {"problem": {"type": "string"}},
"required": ["problem"],
"properties": {
"setting_title": {"type": "string"},
"setting_premise": {"type": "string"},
"style_guide": {"type": "string"},
},
"required": ["setting_title", "setting_premise", "style_guide"],
"additionalProperties": False,
}
RENDER_CHUNK_SCHEMA = {
"type": "object",
"properties": {"text": {"type": "string"}},
"required": ["text"],
"additionalProperties": False,
}
def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str:
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
response = client.request_json(
def _surface_names(skeleton: WorldSkeleton) -> dict[str, str]:
first = ("Zeru", "Parn", "Moki", "Tavi", "Luma", "Neri", "Kado", "Vela", "Sori", "Denu")
second = ("an", "or", "is", "um", "el", "ai", "on", "ar", "en", "ul")
names: dict[str, str] = {}
counters: dict[str, int] = {}
for entity in skeleton.entities:
index = counters.get(entity.type, 0)
counters[entity.type] = index + 1
if entity.type == "Entity":
names[entity.id] = first[index % len(first)] + second[(index // len(first)) % len(second)]
elif entity.type == "Set":
names[entity.id] = f"Collection-{index + 1:02d}"
elif entity.type == "Region":
names[entity.id] = f"Chamber-{index + 1:02d}"
else:
names[entity.id] = f"Gauge-{index + 1:02d}"
return names
def render_skeleton(
client: JsonModel,
skeleton: WorldSkeleton,
*,
render_chunk_size: int = 60,
) -> str:
if render_chunk_size < 1:
raise ValueError("render_chunk_size must be positive")
payload = skeleton.renderer_payload()
names = _surface_names(skeleton)
type_counts: dict[str, int] = {}
for entity_type in payload["entity_types"].values():
type_counts[entity_type] = type_counts.get(entity_type, 0) + 1
plan = client.request_json(
system=(
"Turn a large formal history from one simulated world into a self-contained natural-language "
"reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to "
"every symbol. Render every initial observation and every realized event exactly once, preserving "
"time order, participants, relation direction, numeric values, and state changes. Render the query "
"at the end. The events are facts that actually occurred, not repeated probability experiments. "
"Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring "
"instructions. Do not directly state the queried proposition. Begin each rendered fact or event line "
"with its original I#### or V##### audit ID. Use compact chronological log lines rather than "
"explanatory prose, but do not omit apparent distractors. State the answer labels; the "
"evaluator separately supplies the solver response protocol."
"Design a concise fictional setting for a large symbolic event history. Return only a title, a short "
"premise, and a style guide that maps the supplied relation and event vocabulary to consistent natural "
"language. Do not invent world facts, rules, probabilities, a query answer, or a proof."
),
user=f"Render this intermediate representation:\n{prompt}",
schema_name="rendered_problem",
schema=RENDER_SCHEMA,
user=json.dumps(
{
"entity_type_counts": type_counts,
"relations": [
"LOCATED_IN",
"ABOVE",
"MEMBER",
"SUBSET",
"DISJOINT",
"CONNECTED",
],
"event_kinds": ["MOVE", "UPDATE_NUMBER", "SET_STATE"],
},
ensure_ascii=False,
),
schema_name="rendering_plan",
schema=RENDER_PLAN_SCHEMA,
)
problem = response.get("problem")
if not isinstance(problem, str) or not problem.strip():
raise RuntimeError("renderer returned an empty problem")
problem = problem.strip()
expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]]
expected_ids.extend(event.id for event in skeleton.events)
if not all(isinstance(plan.get(key), str) and plan[key].strip() for key in RENDER_PLAN_SCHEMA["required"]):
raise RuntimeError("renderer returned an incomplete rendering plan")
records = [
{"section": "initial_observation", **item}
for item in skeleton.initial_state["public_observations"]
]
records.extend({"section": "event", **asdict(event)} for event in skeleton.events)
rendered_chunks: list[str] = []
for start in range(0, len(records), render_chunk_size):
chunk = records[start : start + render_chunk_size]
response = client.request_json(
system=(
"Render the supplied slice of one world history as compact natural-language log lines. Use the "
"provided setting, style guide, and exact surface-name map. Render every record exactly once and "
"begin its line with the record's I#### or V##### audit ID. Preserve all participants, relation "
"directions, values, before/after states, and event order. Do not summarize, infer rules, answer a "
"query, or add facts. Return only the finished lines for this slice."
),
user=json.dumps(
{
"rendering_plan": plan,
"surface_names": names,
"records": chunk,
},
ensure_ascii=False,
),
schema_name="rendered_history_chunk",
schema=RENDER_CHUNK_SCHEMA,
)
text = response.get("text")
if not isinstance(text, str) or not text.strip():
raise RuntimeError(f"renderer returned an empty history chunk beginning at record {start}")
rendered_chunks.append(text.strip())
query_names = [names.get(argument, argument) for argument in skeleton.query.arguments]
if skeleton.query.predicate == "ABOVE":
question = f"At the end of the history, is {query_names[0]} above {query_names[1]}?"
elif skeleton.query.predicate == "MEMBER":
question = f"At the end of the history, is {query_names[0]} a member of {query_names[1]}?"
else:
raise RuntimeError(f"unsupported rendered query predicate: {skeleton.query.predicate}")
problem = (
f"# {plan['setting_title'].strip()}\n\n{plan['setting_premise'].strip()}\n\n"
+ "\n".join(rendered_chunks)
+ f"\n\nQuestion: {question}\nAnswer labels: {', '.join(skeleton.answer_options)}"
)
expected_ids = [item["id"] for item in records]
missing = [identifier for identifier in expected_ids if identifier not in problem]
if missing:
preview = ", ".join(missing[:10])
@@ -145,7 +235,10 @@ def run_evaluation(
output_dir: Path,
entity_count: int = 128,
simulation_steps: int = 1000,
render_chunk_size: int = 60,
) -> dict[str, Any]:
if render_chunk_size < 1:
raise ValueError("render_chunk_size must be positive")
output_dir.mkdir(parents=True, exist_ok=True)
skeletons = generate_skeletons(
count=count,
@@ -160,7 +253,11 @@ def run_evaluation(
rendered_records: list[dict[str, str]] = []
solver_records: list[dict[str, str]] = []
for skeleton in skeletons:
problem = render_skeleton(renderer, skeleton)
problem = render_skeleton(
renderer,
skeleton,
render_chunk_size=render_chunk_size,
)
rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem})
_write_jsonl(output_dir / "rendered.jsonl", rendered_records)
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
@@ -206,6 +303,7 @@ def run_evaluation(
"seed": seed,
"entity_count": entity_count,
"simulation_steps": simulation_steps,
"render_chunk_size": render_chunk_size,
"event_counts": [len(item.events) for item in skeletons],
}
(output_dir / "summary.json").write_text(