Render dynamic histories in configurable chunks
This commit is contained in:
@@ -35,6 +35,11 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
|
||||
run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file")
|
||||
run.add_argument("task", type=Path)
|
||||
run.add_argument(
|
||||
"--render-chunk-size",
|
||||
type=int,
|
||||
help="override the task's records per renderer request (default: 60)",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
@@ -60,6 +65,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
task = _load_task(args.task)
|
||||
renderer = LLMClient(LLMClientConfig.from_env("renderer"))
|
||||
solver = LLMClient(LLMClientConfig.from_env("solver"))
|
||||
render_chunk_size = (
|
||||
args.render_chunk_size
|
||||
if args.render_chunk_size is not None
|
||||
else int(task.get("render_chunk_size", 60))
|
||||
)
|
||||
summary = run_evaluation(
|
||||
renderer=renderer,
|
||||
solver=solver,
|
||||
@@ -69,6 +79,7 @@ def main(argv: list[str] | None = None) -> int:
|
||||
output_dir=Path(task["output_dir"]),
|
||||
entity_count=int(task.get("entity_count", 128)),
|
||||
simulation_steps=int(task.get("simulation_steps", 1000)),
|
||||
render_chunk_size=render_chunk_size,
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
+124
-26
@@ -38,39 +38,129 @@ class EvaluationResult:
|
||||
correct: bool
|
||||
|
||||
|
||||
RENDER_SCHEMA = {
|
||||
RENDER_PLAN_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {"problem": {"type": "string"}},
|
||||
"required": ["problem"],
|
||||
"properties": {
|
||||
"setting_title": {"type": "string"},
|
||||
"setting_premise": {"type": "string"},
|
||||
"style_guide": {"type": "string"},
|
||||
},
|
||||
"required": ["setting_title", "setting_premise", "style_guide"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
RENDER_CHUNK_SCHEMA = {
|
||||
"type": "object",
|
||||
"properties": {"text": {"type": "string"}},
|
||||
"required": ["text"],
|
||||
"additionalProperties": False,
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
def _surface_names(skeleton: WorldSkeleton) -> dict[str, str]:
|
||||
first = ("Zeru", "Parn", "Moki", "Tavi", "Luma", "Neri", "Kado", "Vela", "Sori", "Denu")
|
||||
second = ("an", "or", "is", "um", "el", "ai", "on", "ar", "en", "ul")
|
||||
names: dict[str, str] = {}
|
||||
counters: dict[str, int] = {}
|
||||
for entity in skeleton.entities:
|
||||
index = counters.get(entity.type, 0)
|
||||
counters[entity.type] = index + 1
|
||||
if entity.type == "Entity":
|
||||
names[entity.id] = first[index % len(first)] + second[(index // len(first)) % len(second)]
|
||||
elif entity.type == "Set":
|
||||
names[entity.id] = f"Collection-{index + 1:02d}"
|
||||
elif entity.type == "Region":
|
||||
names[entity.id] = f"Chamber-{index + 1:02d}"
|
||||
else:
|
||||
names[entity.id] = f"Gauge-{index + 1:02d}"
|
||||
return names
|
||||
|
||||
|
||||
def render_skeleton(
|
||||
client: JsonModel,
|
||||
skeleton: WorldSkeleton,
|
||||
*,
|
||||
render_chunk_size: int = 60,
|
||||
) -> str:
|
||||
if render_chunk_size < 1:
|
||||
raise ValueError("render_chunk_size must be positive")
|
||||
payload = skeleton.renderer_payload()
|
||||
names = _surface_names(skeleton)
|
||||
type_counts: dict[str, int] = {}
|
||||
for entity_type in payload["entity_types"].values():
|
||||
type_counts[entity_type] = type_counts.get(entity_type, 0) + 1
|
||||
plan = client.request_json(
|
||||
system=(
|
||||
"Turn a large formal history from one simulated world into a self-contained natural-language "
|
||||
"reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to "
|
||||
"every symbol. Render every initial observation and every realized event exactly once, preserving "
|
||||
"time order, participants, relation direction, numeric values, and state changes. Render the query "
|
||||
"at the end. The events are facts that actually occurred, not repeated probability experiments. "
|
||||
"Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring "
|
||||
"instructions. Do not directly state the queried proposition. Begin each rendered fact or event line "
|
||||
"with its original I#### or V##### audit ID. Use compact chronological log lines rather than "
|
||||
"explanatory prose, but do not omit apparent distractors. State the answer labels; the "
|
||||
"evaluator separately supplies the solver response protocol."
|
||||
"Design a concise fictional setting for a large symbolic event history. Return only a title, a short "
|
||||
"premise, and a style guide that maps the supplied relation and event vocabulary to consistent natural "
|
||||
"language. Do not invent world facts, rules, probabilities, a query answer, or a proof."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
schema=RENDER_SCHEMA,
|
||||
user=json.dumps(
|
||||
{
|
||||
"entity_type_counts": type_counts,
|
||||
"relations": [
|
||||
"LOCATED_IN",
|
||||
"ABOVE",
|
||||
"MEMBER",
|
||||
"SUBSET",
|
||||
"DISJOINT",
|
||||
"CONNECTED",
|
||||
],
|
||||
"event_kinds": ["MOVE", "UPDATE_NUMBER", "SET_STATE"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
schema_name="rendering_plan",
|
||||
schema=RENDER_PLAN_SCHEMA,
|
||||
)
|
||||
problem = response.get("problem")
|
||||
if not isinstance(problem, str) or not problem.strip():
|
||||
raise RuntimeError("renderer returned an empty problem")
|
||||
problem = problem.strip()
|
||||
expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]]
|
||||
expected_ids.extend(event.id for event in skeleton.events)
|
||||
if not all(isinstance(plan.get(key), str) and plan[key].strip() for key in RENDER_PLAN_SCHEMA["required"]):
|
||||
raise RuntimeError("renderer returned an incomplete rendering plan")
|
||||
|
||||
records = [
|
||||
{"section": "initial_observation", **item}
|
||||
for item in skeleton.initial_state["public_observations"]
|
||||
]
|
||||
records.extend({"section": "event", **asdict(event)} for event in skeleton.events)
|
||||
rendered_chunks: list[str] = []
|
||||
for start in range(0, len(records), render_chunk_size):
|
||||
chunk = records[start : start + render_chunk_size]
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"Render the supplied slice of one world history as compact natural-language log lines. Use the "
|
||||
"provided setting, style guide, and exact surface-name map. Render every record exactly once and "
|
||||
"begin its line with the record's I#### or V##### audit ID. Preserve all participants, relation "
|
||||
"directions, values, before/after states, and event order. Do not summarize, infer rules, answer a "
|
||||
"query, or add facts. Return only the finished lines for this slice."
|
||||
),
|
||||
user=json.dumps(
|
||||
{
|
||||
"rendering_plan": plan,
|
||||
"surface_names": names,
|
||||
"records": chunk,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
schema_name="rendered_history_chunk",
|
||||
schema=RENDER_CHUNK_SCHEMA,
|
||||
)
|
||||
text = response.get("text")
|
||||
if not isinstance(text, str) or not text.strip():
|
||||
raise RuntimeError(f"renderer returned an empty history chunk beginning at record {start}")
|
||||
rendered_chunks.append(text.strip())
|
||||
|
||||
query_names = [names.get(argument, argument) for argument in skeleton.query.arguments]
|
||||
if skeleton.query.predicate == "ABOVE":
|
||||
question = f"At the end of the history, is {query_names[0]} above {query_names[1]}?"
|
||||
elif skeleton.query.predicate == "MEMBER":
|
||||
question = f"At the end of the history, is {query_names[0]} a member of {query_names[1]}?"
|
||||
else:
|
||||
raise RuntimeError(f"unsupported rendered query predicate: {skeleton.query.predicate}")
|
||||
problem = (
|
||||
f"# {plan['setting_title'].strip()}\n\n{plan['setting_premise'].strip()}\n\n"
|
||||
+ "\n".join(rendered_chunks)
|
||||
+ f"\n\nQuestion: {question}\nAnswer labels: {', '.join(skeleton.answer_options)}"
|
||||
)
|
||||
expected_ids = [item["id"] for item in records]
|
||||
missing = [identifier for identifier in expected_ids if identifier not in problem]
|
||||
if missing:
|
||||
preview = ", ".join(missing[:10])
|
||||
@@ -145,7 +235,10 @@ def run_evaluation(
|
||||
output_dir: Path,
|
||||
entity_count: int = 128,
|
||||
simulation_steps: int = 1000,
|
||||
render_chunk_size: int = 60,
|
||||
) -> dict[str, Any]:
|
||||
if render_chunk_size < 1:
|
||||
raise ValueError("render_chunk_size must be positive")
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
skeletons = generate_skeletons(
|
||||
count=count,
|
||||
@@ -160,7 +253,11 @@ def run_evaluation(
|
||||
rendered_records: list[dict[str, str]] = []
|
||||
solver_records: list[dict[str, str]] = []
|
||||
for skeleton in skeletons:
|
||||
problem = render_skeleton(renderer, skeleton)
|
||||
problem = render_skeleton(
|
||||
renderer,
|
||||
skeleton,
|
||||
render_chunk_size=render_chunk_size,
|
||||
)
|
||||
rendered_records.append({"skeleton_id": skeleton.id, "rendered_problem": problem})
|
||||
_write_jsonl(output_dir / "rendered.jsonl", rendered_records)
|
||||
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
|
||||
@@ -206,6 +303,7 @@ def run_evaluation(
|
||||
"seed": seed,
|
||||
"entity_count": entity_count,
|
||||
"simulation_steps": simulation_steps,
|
||||
"render_chunk_size": render_chunk_size,
|
||||
"event_counts": [len(item.events) for item in skeletons],
|
||||
}
|
||||
(output_dir / "summary.json").write_text(
|
||||
|
||||
Reference in New Issue
Block a user