Generate proof-bearing dynamic world histories
This commit is contained in:
+38
-20
@@ -6,7 +6,7 @@ from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .models import BehaviorSkeleton
|
||||
from .models import WorldSkeleton
|
||||
from .skeletons import generate_skeletons
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ class EvaluationResult:
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
solver_output: str
|
||||
model_answer: str | None
|
||||
format_compliant: bool
|
||||
@@ -47,20 +46,20 @@ RENDER_SCHEMA = {
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"You turn behavioral observations from an unknown stochastic system into one self-contained "
|
||||
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
|
||||
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
|
||||
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
|
||||
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
|
||||
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
|
||||
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
|
||||
"not for an exact probability. Present repeated histories in a compact table rather than verbose "
|
||||
"paragraphs. State the supplied answer labels, but do not impose a response format or forbid an "
|
||||
"explanation; the evaluator supplies its own output protocol to the solver."
|
||||
"Turn a large formal history from one simulated world into a self-contained natural-language "
|
||||
"reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to "
|
||||
"every symbol. Render every initial observation and every realized event exactly once, preserving "
|
||||
"time order, participants, relation direction, numeric values, and state changes. Render the query "
|
||||
"at the end. The events are facts that actually occurred, not repeated probability experiments. "
|
||||
"Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring "
|
||||
"instructions. Do not directly state the queried proposition. Begin each rendered fact or event line "
|
||||
"with its original I#### or V##### audit ID. Use compact chronological log lines rather than "
|
||||
"explanatory prose, but do not omit apparent distractors. State the answer labels; the "
|
||||
"evaluator separately supplies the solver response protocol."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
@@ -69,7 +68,16 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
problem = response.get("problem")
|
||||
if not isinstance(problem, str) or not problem.strip():
|
||||
raise RuntimeError("renderer returned an empty problem")
|
||||
return problem.strip()
|
||||
problem = problem.strip()
|
||||
expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]]
|
||||
expected_ids.extend(event.id for event in skeleton.events)
|
||||
missing = [identifier for identifier in expected_ids if identifier not in problem]
|
||||
if missing:
|
||||
preview = ", ".join(missing[:10])
|
||||
raise RuntimeError(
|
||||
f"renderer omitted {len(missing)} initial observations or events; first missing IDs: {preview}"
|
||||
)
|
||||
return problem
|
||||
|
||||
|
||||
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||
@@ -110,10 +118,10 @@ def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ..
|
||||
def request_solver_output(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
|
||||
return client.request_text(
|
||||
system=(
|
||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
|
||||
"200 words, then put "
|
||||
"Solve the question from the described world history. Track entity identity, relation direction, "
|
||||
"temporal updates, set inheritance, ordering, and relevant compositions while ignoring distractors. "
|
||||
"Use only the stated observations and ordinary meanings explicitly established by the story. Keep "
|
||||
"the explanation under 200 words, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
@@ -135,9 +143,17 @@ def run_evaluation(
|
||||
length: str | int,
|
||||
seed: int,
|
||||
output_dir: Path,
|
||||
entity_count: int = 128,
|
||||
simulation_steps: int = 1000,
|
||||
) -> dict[str, Any]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
skeletons = generate_skeletons(count=count, length=length, seed=seed)
|
||||
skeletons = generate_skeletons(
|
||||
count=count,
|
||||
length=length,
|
||||
seed=seed,
|
||||
entity_count=entity_count,
|
||||
simulation_steps=simulation_steps,
|
||||
)
|
||||
_write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
|
||||
|
||||
results: list[EvaluationResult] = []
|
||||
@@ -165,7 +181,6 @@ def run_evaluation(
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
gold_present_probability=skeleton.gold_present_probability,
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
format_compliant=format_compliant,
|
||||
@@ -189,6 +204,9 @@ def run_evaluation(
|
||||
"length": length,
|
||||
"complexity": skeletons[0].complexity,
|
||||
"seed": seed,
|
||||
"entity_count": entity_count,
|
||||
"simulation_steps": simulation_steps,
|
||||
"event_counts": [len(item.events) for item in skeletons],
|
||||
}
|
||||
(output_dir / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
|
||||
Reference in New Issue
Block a user