Generate proof-bearing dynamic world histories

This commit is contained in:
2026-09-23 10:23:09 +08:00
parent 6f929437a2
commit 43aff1e2ee
13 changed files with 1023 additions and 550 deletions
+38 -20
View File
@@ -6,7 +6,7 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Protocol
from .models import BehaviorSkeleton
from .models import WorldSkeleton
from .skeletons import generate_skeletons
@@ -31,7 +31,6 @@ class EvaluationResult:
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
gold_present_probability: float
solver_output: str
model_answer: str | None
format_compliant: bool
@@ -47,20 +46,20 @@ RENDER_SCHEMA = {
}
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str:
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
response = client.request_json(
system=(
"You turn behavioral observations from an unknown stochastic system into one self-contained "
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
"not for an exact probability. Present repeated histories in a compact table rather than verbose "
"paragraphs. State the supplied answer labels, but do not impose a response format or forbid an "
"explanation; the evaluator supplies its own output protocol to the solver."
"Turn a large formal history from one simulated world into a self-contained natural-language "
"reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to "
"every symbol. Render every initial observation and every realized event exactly once, preserving "
"time order, participants, relation direction, numeric values, and state changes. Render the query "
"at the end. The events are facts that actually occurred, not repeated probability experiments. "
"Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring "
"instructions. Do not directly state the queried proposition. Begin each rendered fact or event line "
"with its original I#### or V##### audit ID. Use compact chronological log lines rather than "
"explanatory prose, but do not omit apparent distractors. State the answer labels; the "
"evaluator separately supplies the solver response protocol."
),
user=f"Render this intermediate representation:\n{prompt}",
schema_name="rendered_problem",
@@ -69,7 +68,16 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
problem = response.get("problem")
if not isinstance(problem, str) or not problem.strip():
raise RuntimeError("renderer returned an empty problem")
return problem.strip()
problem = problem.strip()
expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]]
expected_ids.extend(event.id for event in skeleton.events)
missing = [identifier for identifier in expected_ids if identifier not in problem]
if missing:
preview = ", ".join(missing[:10])
raise RuntimeError(
f"renderer omitted {len(missing)} initial observations or events; first missing IDs: {preview}"
)
return problem
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
@@ -110,10 +118,10 @@ def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ..
def request_solver_output(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
return client.request_text(
system=(
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
"200 words, then put "
"Solve the question from the described world history. Track entity identity, relation direction, "
"temporal updates, set inheritance, ordering, and relevant compositions while ignoring distractors. "
"Use only the stated observations and ordinary meanings explicitly established by the story. Keep "
"the explanation under 200 words, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
@@ -135,9 +143,17 @@ def run_evaluation(
length: str | int,
seed: int,
output_dir: Path,
entity_count: int = 128,
simulation_steps: int = 1000,
) -> dict[str, Any]:
output_dir.mkdir(parents=True, exist_ok=True)
skeletons = generate_skeletons(count=count, length=length, seed=seed)
skeletons = generate_skeletons(
count=count,
length=length,
seed=seed,
entity_count=entity_count,
simulation_steps=simulation_steps,
)
_write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
results: list[EvaluationResult] = []
@@ -165,7 +181,6 @@ def run_evaluation(
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
gold_present_probability=skeleton.gold_present_probability,
solver_output=solver_output,
model_answer=answer,
format_compliant=format_compliant,
@@ -189,6 +204,9 @@ def run_evaluation(
"length": length,
"complexity": skeletons[0].complexity,
"seed": seed,
"entity_count": entity_count,
"simulation_steps": simulation_steps,
"event_counts": [len(item.events) for item in skeletons],
}
(output_dir / "summary.json").write_text(
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"