Generate proof-bearing dynamic world histories
This commit is contained in:
@@ -1,14 +1,15 @@
|
||||
"""Behavioral hidden-system benchmark."""
|
||||
|
||||
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
|
||||
from .models import Entity, HiddenProgram, Query, TransitionRule, WorldEvent, WorldSkeleton
|
||||
from .skeletons import generate_skeletons, resolve_length
|
||||
|
||||
__all__ = [
|
||||
"BehaviorCase",
|
||||
"BehaviorSkeleton",
|
||||
"Condition",
|
||||
"HiddenRule",
|
||||
"HiddenSystem",
|
||||
"Entity",
|
||||
"HiddenProgram",
|
||||
"Query",
|
||||
"TransitionRule",
|
||||
"WorldEvent",
|
||||
"WorldSkeleton",
|
||||
"generate_skeletons",
|
||||
"resolve_length",
|
||||
]
|
||||
|
||||
+11
-1
@@ -29,6 +29,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
generate.add_argument("--count", type=int, default=3)
|
||||
generate.add_argument("--length", default="short")
|
||||
generate.add_argument("--seed", type=int, default=0)
|
||||
generate.add_argument("--entity-count", type=int, default=128)
|
||||
generate.add_argument("--simulation-steps", type=int, default=1000)
|
||||
generate.add_argument("--output", type=Path)
|
||||
|
||||
run = subparsers.add_parser("run", help="run renderer and solver from a task JSON file")
|
||||
@@ -39,7 +41,13 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.command == "generate":
|
||||
skeletons = generate_skeletons(args.count, args.length, args.seed)
|
||||
skeletons = generate_skeletons(
|
||||
args.count,
|
||||
args.length,
|
||||
args.seed,
|
||||
entity_count=args.entity_count,
|
||||
simulation_steps=args.simulation_steps,
|
||||
)
|
||||
lines = [json.dumps(item.to_dict(), ensure_ascii=False) for item in skeletons]
|
||||
text = "\n".join(lines) + "\n"
|
||||
if args.output:
|
||||
@@ -59,6 +67,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
length=task["length"],
|
||||
seed=int(task["seed"]),
|
||||
output_dir=Path(task["output_dir"]),
|
||||
entity_count=int(task.get("entity_count", 128)),
|
||||
simulation_steps=int(task.get("simulation_steps", 1000)),
|
||||
)
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
@@ -81,7 +81,7 @@ class LLMClient:
|
||||
"schema": schema,
|
||||
}
|
||||
},
|
||||
"max_output_tokens": 2500,
|
||||
"max_output_tokens": 12000,
|
||||
}
|
||||
else:
|
||||
path = "/chat/completions"
|
||||
@@ -95,7 +95,7 @@ class LLMClient:
|
||||
"type": "json_schema",
|
||||
"json_schema": {"name": schema_name, "strict": True, "schema": schema},
|
||||
},
|
||||
"max_tokens": 2500,
|
||||
"max_tokens": 12000,
|
||||
}
|
||||
|
||||
if self.config.enable_thinking is not None:
|
||||
|
||||
+43
-44
@@ -5,78 +5,77 @@ from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Condition:
|
||||
"""A predicate over immutable inputs or mutable simulator state."""
|
||||
|
||||
source: str
|
||||
key: str
|
||||
operator: str
|
||||
value: Any
|
||||
class Entity:
|
||||
id: str
|
||||
type: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HiddenRule:
|
||||
"""One executable rule in the hidden system language."""
|
||||
|
||||
class TransitionRule:
|
||||
id: str
|
||||
on_event: str
|
||||
conditions: tuple[Condition, ...]
|
||||
trigger: str
|
||||
preconditions: tuple[str, ...]
|
||||
probability: float
|
||||
action: str
|
||||
target: str
|
||||
effects: tuple[str, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HiddenSystem:
|
||||
"""The latent program. This object must never be sent to the renderer."""
|
||||
|
||||
initial_record: str
|
||||
conflict_policy: str
|
||||
rules: tuple[HiddenRule, ...]
|
||||
class HiddenProgram:
|
||||
types: tuple[str, ...]
|
||||
relations: tuple[str, ...]
|
||||
operators: tuple[str, ...]
|
||||
quantifiers: tuple[str, ...]
|
||||
structures: tuple[str, ...]
|
||||
inference_rules: tuple[str, ...]
|
||||
transition_rules: tuple[TransitionRule, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BehaviorCase:
|
||||
"""Observable inputs and aggregate output from repeated independent trials."""
|
||||
|
||||
class WorldEvent:
|
||||
id: str
|
||||
attributes: dict[str, Any]
|
||||
events: tuple[str, ...]
|
||||
trials: int
|
||||
observed_present: int | None
|
||||
observed_absent: int | None
|
||||
time: int
|
||||
kind: str
|
||||
participants: tuple[str, ...]
|
||||
arguments: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BehaviorSkeleton:
|
||||
"""A hidden program plus its public behavioral evidence and answer-bearing query."""
|
||||
class Query:
|
||||
predicate: str
|
||||
arguments: tuple[str, ...]
|
||||
time: int
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class WorldSkeleton:
|
||||
id: str
|
||||
family: str
|
||||
complexity: int
|
||||
event_vocabulary: tuple[str, ...]
|
||||
attribute_vocabulary: dict[str, str]
|
||||
observation_protocol: tuple[str, ...]
|
||||
hidden_system: HiddenSystem
|
||||
support_cases: tuple[BehaviorCase, ...]
|
||||
query_case: BehaviorCase
|
||||
entity_count: int
|
||||
simulation_steps: int
|
||||
hidden_program: HiddenProgram
|
||||
entities: tuple[Entity, ...]
|
||||
initial_state: dict[str, Any]
|
||||
events: tuple[WorldEvent, ...]
|
||||
final_state: dict[str, Any]
|
||||
query: Query
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
latent_trace: tuple[str, ...]
|
||||
proof_event_ids: tuple[str, ...]
|
||||
proof_facts: tuple[str, ...]
|
||||
generator_checks: dict[str, Any]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
def renderer_payload(self) -> dict[str, Any]:
|
||||
"""Expose behavior only: no program, probability, trace, or answer."""
|
||||
"""Expose realized history, never the source program, proof, or answer."""
|
||||
return {
|
||||
"family": self.family,
|
||||
"complexity": self.complexity,
|
||||
"event_vocabulary": self.event_vocabulary,
|
||||
"attribute_vocabulary": self.attribute_vocabulary,
|
||||
"observation_protocol": self.observation_protocol,
|
||||
"support_cases": [asdict(case) for case in self.support_cases],
|
||||
"query_case": asdict(self.query_case),
|
||||
"entity_types": {entity.id: entity.type for entity in self.entities},
|
||||
"initial_observations": self.initial_state["public_observations"],
|
||||
"events": [asdict(event) for event in self.events],
|
||||
"query": asdict(self.query),
|
||||
"answer_options": self.answer_options,
|
||||
}
|
||||
|
||||
+38
-20
@@ -6,7 +6,7 @@ from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .models import BehaviorSkeleton
|
||||
from .models import WorldSkeleton
|
||||
from .skeletons import generate_skeletons
|
||||
|
||||
|
||||
@@ -31,7 +31,6 @@ class EvaluationResult:
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
solver_output: str
|
||||
model_answer: str | None
|
||||
format_compliant: bool
|
||||
@@ -47,20 +46,20 @@ RENDER_SCHEMA = {
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
def render_skeleton(client: JsonModel, skeleton: WorldSkeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"You turn behavioral observations from an unknown stochastic system into one self-contained "
|
||||
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
|
||||
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
|
||||
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
|
||||
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
|
||||
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
|
||||
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
|
||||
"not for an exact probability. Present repeated histories in a compact table rather than verbose "
|
||||
"paragraphs. State the supplied answer labels, but do not impose a response format or forbid an "
|
||||
"explanation; the evaluator supplies its own output protocol to the solver."
|
||||
"Turn a large formal history from one simulated world into a self-contained natural-language "
|
||||
"reasoning problem. Invent one coherent concrete or fictional setting and assign stable names to "
|
||||
"every symbol. Render every initial observation and every realized event exactly once, preserving "
|
||||
"time order, participants, relation direction, numeric values, and state changes. Render the query "
|
||||
"at the end. The events are facts that actually occurred, not repeated probability experiments. "
|
||||
"Never mention the generator, hidden program, probabilities, proof path, gold answer, or authoring "
|
||||
"instructions. Do not directly state the queried proposition. Begin each rendered fact or event line "
|
||||
"with its original I#### or V##### audit ID. Use compact chronological log lines rather than "
|
||||
"explanatory prose, but do not omit apparent distractors. State the answer labels; the "
|
||||
"evaluator separately supplies the solver response protocol."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
@@ -69,7 +68,16 @@ def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
problem = response.get("problem")
|
||||
if not isinstance(problem, str) or not problem.strip():
|
||||
raise RuntimeError("renderer returned an empty problem")
|
||||
return problem.strip()
|
||||
problem = problem.strip()
|
||||
expected_ids = [item["id"] for item in skeleton.initial_state["public_observations"]]
|
||||
expected_ids.extend(event.id for event in skeleton.events)
|
||||
missing = [identifier for identifier in expected_ids if identifier not in problem]
|
||||
if missing:
|
||||
preview = ", ".join(missing[:10])
|
||||
raise RuntimeError(
|
||||
f"renderer omitted {len(missing)} initial observations or events; first missing IDs: {preview}"
|
||||
)
|
||||
return problem
|
||||
|
||||
|
||||
def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||
@@ -110,10 +118,10 @@ def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ..
|
||||
def request_solver_output(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> str:
|
||||
return client.request_text(
|
||||
system=(
|
||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
|
||||
"200 words, then put "
|
||||
"Solve the question from the described world history. Track entity identity, relation direction, "
|
||||
"temporal updates, set inheritance, ordering, and relevant compositions while ignoring distractors. "
|
||||
"Use only the stated observations and ordinary meanings explicitly established by the story. Keep "
|
||||
"the explanation under 200 words, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
@@ -135,9 +143,17 @@ def run_evaluation(
|
||||
length: str | int,
|
||||
seed: int,
|
||||
output_dir: Path,
|
||||
entity_count: int = 128,
|
||||
simulation_steps: int = 1000,
|
||||
) -> dict[str, Any]:
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
skeletons = generate_skeletons(count=count, length=length, seed=seed)
|
||||
skeletons = generate_skeletons(
|
||||
count=count,
|
||||
length=length,
|
||||
seed=seed,
|
||||
entity_count=entity_count,
|
||||
simulation_steps=simulation_steps,
|
||||
)
|
||||
_write_jsonl(output_dir / "skeletons.jsonl", [item.to_dict() for item in skeletons])
|
||||
|
||||
results: list[EvaluationResult] = []
|
||||
@@ -165,7 +181,6 @@ def run_evaluation(
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
gold_present_probability=skeleton.gold_present_probability,
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
format_compliant=format_compliant,
|
||||
@@ -189,6 +204,9 @@ def run_evaluation(
|
||||
"length": length,
|
||||
"complexity": skeletons[0].complexity,
|
||||
"seed": seed,
|
||||
"entity_count": entity_count,
|
||||
"simulation_steps": simulation_steps,
|
||||
"event_counts": [len(item.events) for item in skeletons],
|
||||
}
|
||||
(output_dir / "summary.json").write_text(
|
||||
json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
|
||||
|
||||
+579
-216
@@ -1,26 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from itertools import combinations
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
|
||||
from .models import Entity, HiddenProgram, Query, TransitionRule, WorldEvent, WorldSkeleton
|
||||
|
||||
|
||||
LENGTH_PRESETS = {"short": 4, "medium": 6, "long": 8}
|
||||
ANSWER_OPTIONS = ("PRESENT", "ABSENT")
|
||||
ANSWER_OPTIONS = ("YES", "NO")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _State:
|
||||
present: bool = False
|
||||
invalidated: bool = False
|
||||
flags: frozenset[str] = frozenset()
|
||||
@dataclass
|
||||
class _World:
|
||||
locations: dict[str, str]
|
||||
order_level: dict[str, int]
|
||||
set_level: dict[str, int]
|
||||
above: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
subsets: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
members: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
disjoint: dict[frozenset[str], str] = field(default_factory=dict)
|
||||
connected: set[frozenset[str]] = field(default_factory=set)
|
||||
numbers: dict[str, int] = field(default_factory=dict)
|
||||
flags: dict[tuple[str, str], bool] = field(default_factory=dict)
|
||||
|
||||
|
||||
def resolve_length(value: str | int) -> int:
|
||||
"""Resolve the number of distinct events in the held-out query."""
|
||||
"""Resolve required proof length, independently of event-log or text length."""
|
||||
if isinstance(value, int):
|
||||
result = value
|
||||
elif value in LENGTH_PRESETS:
|
||||
@@ -29,246 +36,602 @@ def resolve_length(value: str | int) -> int:
|
||||
try:
|
||||
result = int(value)
|
||||
except ValueError as exc:
|
||||
choices = ", ".join(LENGTH_PRESETS)
|
||||
raise ValueError(f"length must be an integer or one of: {choices}") from exc
|
||||
if not 4 <= result <= 10:
|
||||
raise ValueError("length must be between 4 and 10")
|
||||
raise ValueError(f"length must be an integer or one of: {', '.join(LENGTH_PRESETS)}") from exc
|
||||
if not 3 <= result <= 10:
|
||||
raise ValueError("length must be between 3 and 10")
|
||||
return result
|
||||
|
||||
|
||||
def _condition_holds(condition: Condition, attributes: dict[str, Any], state: _State) -> bool:
|
||||
if condition.source == "attribute":
|
||||
actual = attributes.get(condition.key)
|
||||
elif condition.source == "state":
|
||||
actual = condition.key in state.flags
|
||||
else:
|
||||
raise ValueError(f"unknown condition source: {condition.source}")
|
||||
def _program() -> HiddenProgram:
|
||||
return HiddenProgram(
|
||||
types=("Entity", "Set", "Region", "Number", "Boolean", "State"),
|
||||
relations=(
|
||||
"EQUAL",
|
||||
"NOT_EQUAL",
|
||||
"ABOVE",
|
||||
"BELOW",
|
||||
"MEMBER",
|
||||
"SUBSET",
|
||||
"DISJOINT",
|
||||
"ADJACENT",
|
||||
"CONNECTED",
|
||||
"CAUSES",
|
||||
"LOCATED_IN",
|
||||
),
|
||||
operators=(
|
||||
"NOT",
|
||||
"AND",
|
||||
"OR",
|
||||
"IMPLIES",
|
||||
"ADD",
|
||||
"SUBTRACT",
|
||||
"COMPARE",
|
||||
"UNION",
|
||||
"INTERSECTION",
|
||||
"MOVE",
|
||||
"SWAP",
|
||||
"SET",
|
||||
"COMPOSE",
|
||||
),
|
||||
quantifiers=("ALL", "EXISTS"),
|
||||
structures=("CHAIN", "BRANCH", "MERGE", "CYCLE", "NEST", "CONDITION", "SEQUENCE"),
|
||||
inference_rules=(
|
||||
"ABOVE(x,y) AND ABOVE(y,z) IMPLIES ABOVE(x,z)",
|
||||
"ABOVE(x,y) IMPLIES BELOW(y,x)",
|
||||
"MEMBER(x,A) AND SUBSET(A,B) IMPLIES MEMBER(x,B)",
|
||||
"MEMBER(x,A) AND DISJOINT(A,B) IMPLIES NOT MEMBER(x,B)",
|
||||
"CONNECTED is symmetric",
|
||||
"Numeric updates compose in temporal order",
|
||||
),
|
||||
transition_rules=(
|
||||
TransitionRule(
|
||||
"move",
|
||||
"SELECT_ENTITY",
|
||||
("target region differs from current region",),
|
||||
0.72,
|
||||
("MOVE(entity, region)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"encounter-order",
|
||||
"COLOCATED_PAIR",
|
||||
("entities share a region", "latent order levels are adjacent"),
|
||||
0.78,
|
||||
("SET ABOVE(higher, lower)", "SET BELOW(lower, higher)"),
|
||||
),
|
||||
TransitionRule(
|
||||
"membership",
|
||||
"ENTITY_SET_CONTACT",
|
||||
("set is at base level",),
|
||||
0.68,
|
||||
("SET MEMBER(entity, set)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"subset",
|
||||
"SET_PAIR_CONTACT",
|
||||
("set levels are adjacent",),
|
||||
0.76,
|
||||
("SET SUBSET(lower, higher)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"disjoint",
|
||||
"SET_PAIR_CONTACT",
|
||||
("sets have equal level", "sets are distinct"),
|
||||
0.64,
|
||||
("SET DISJOINT(left, right)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"numeric-update",
|
||||
"COUNTER_PULSE",
|
||||
(),
|
||||
0.82,
|
||||
("ADD or SUBTRACT(counter, amount)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"connect",
|
||||
"ENTITY_PAIR_CONTACT",
|
||||
(),
|
||||
0.55,
|
||||
("SET CONNECTED(left, right)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"state-change",
|
||||
"ENTITY_SIGNAL",
|
||||
(),
|
||||
0.58,
|
||||
("SET Boolean state",),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
operations = {
|
||||
"EQ": lambda left, right: left == right,
|
||||
"LE": lambda left, right: left <= right,
|
||||
"GT": lambda left, right: left > right,
|
||||
"IS_SET": lambda left, right: left is bool(right),
|
||||
|
||||
def _entities(entity_count: int) -> tuple[list[Entity], dict[str, list[str]]]:
|
||||
if entity_count < 48:
|
||||
raise ValueError("entity_count must be at least 48")
|
||||
region_count = max(4, entity_count // 32)
|
||||
set_count = max(24, entity_count // 5)
|
||||
counter_count = max(8, entity_count // 16)
|
||||
agent_count = entity_count - region_count - set_count - counter_count
|
||||
groups = {
|
||||
"Entity": [f"E{i:03d}" for i in range(agent_count)],
|
||||
"Set": [f"S{i:03d}" for i in range(set_count)],
|
||||
"Region": [f"R{i:03d}" for i in range(region_count)],
|
||||
"Number": [f"N{i:03d}" for i in range(counter_count)],
|
||||
}
|
||||
try:
|
||||
return operations[condition.operator](actual, condition.value)
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unknown condition operator: {condition.operator}") from exc
|
||||
entities = [Entity(identifier, kind) for kind, values in groups.items() for identifier in values]
|
||||
return entities, groups
|
||||
|
||||
|
||||
def _apply_action(state: _State, rule: HiddenRule) -> _State:
|
||||
if rule.action == "SET_FLAG":
|
||||
return _State(state.present, state.invalidated, state.flags | {rule.target})
|
||||
if rule.action == "ADD_RECORD":
|
||||
if state.invalidated:
|
||||
return state
|
||||
return _State(True, False, state.flags)
|
||||
if rule.action == "INVALIDATE_RECORD":
|
||||
return _State(False, True, state.flags)
|
||||
raise ValueError(f"unknown action: {rule.action}")
|
||||
|
||||
|
||||
def exact_present_probability(
|
||||
system: HiddenSystem, attributes: dict[str, Any], events: tuple[str, ...]
|
||||
) -> float:
|
||||
"""Enumerate all stochastic branches while respecting event and state order."""
|
||||
initial = _State(present=system.initial_record == "PRESENT")
|
||||
distribution: dict[_State, float] = {initial: 1.0}
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
next_distribution: dict[_State, float] = {}
|
||||
for state, mass in distribution.items():
|
||||
if not all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + mass
|
||||
continue
|
||||
inactive_mass = mass * (1.0 - rule.probability)
|
||||
active_state = _apply_action(state, rule)
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + inactive_mass
|
||||
next_distribution[active_state] = (
|
||||
next_distribution.get(active_state, 0.0) + mass * rule.probability
|
||||
)
|
||||
distribution = next_distribution
|
||||
return sum(mass for state, mass in distribution.items() if state.present)
|
||||
|
||||
|
||||
def _simulate_once(
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
rng: random.Random,
|
||||
) -> bool:
|
||||
state = _State(present=system.initial_record == "PRESENT")
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
if all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
if rng.random() < rule.probability:
|
||||
state = _apply_action(state, rule)
|
||||
return state.present
|
||||
|
||||
|
||||
def _observe_case(
|
||||
case_id: str,
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
trials: int,
|
||||
rng: random.Random,
|
||||
) -> BehaviorCase:
|
||||
present = sum(_simulate_once(system, attributes, events, rng) for _ in range(trials))
|
||||
return BehaviorCase(
|
||||
id=case_id,
|
||||
attributes=attributes,
|
||||
events=events,
|
||||
trials=trials,
|
||||
observed_present=present,
|
||||
observed_absent=trials - present,
|
||||
def _initial_world(
|
||||
groups: dict[str, list[str]], rng: random.Random, proof_length: int
|
||||
) -> tuple[_World, dict[str, Any]]:
|
||||
agents, sets, regions, counters = (
|
||||
groups["Entity"],
|
||||
groups["Set"],
|
||||
groups["Region"],
|
||||
groups["Number"],
|
||||
)
|
||||
|
||||
|
||||
def _build_system(index: int, complexity: int, rng: random.Random) -> HiddenSystem:
|
||||
"""Build a finite stochastic program with a gated rule and veto conflicts."""
|
||||
present_favored = index % 2 == 0
|
||||
add_probability = rng.choice((0.85, 0.9, 0.95)) if present_favored else rng.choice((0.65, 0.75))
|
||||
veto_probability = rng.choice((0.15, 0.25)) if present_favored else rng.choice((0.7, 0.8, 0.9))
|
||||
marker_probability = rng.choice((0.6, 0.75, 0.9))
|
||||
gated_action = "ADD_RECORD" if present_favored else "INVALIDATE_RECORD"
|
||||
gated_probability = rng.choice((0.6, 0.75, 0.85))
|
||||
|
||||
rules = [
|
||||
HiddenRule("R0", "E0", (), add_probability, "ADD_RECORD", "record"),
|
||||
HiddenRule(
|
||||
"R1",
|
||||
"E1",
|
||||
(Condition("attribute", "load", "LE", 4),),
|
||||
veto_probability,
|
||||
"INVALIDATE_RECORD",
|
||||
"record",
|
||||
),
|
||||
HiddenRule("R2", "E2", (), marker_probability, "SET_FLAG", "marked"),
|
||||
HiddenRule(
|
||||
"R3",
|
||||
"E3",
|
||||
(Condition("state", "marked", "IS_SET", True),),
|
||||
gated_probability,
|
||||
gated_action,
|
||||
"record",
|
||||
),
|
||||
order_levels = max(proof_length + 2, 8)
|
||||
set_levels = max(proof_length + 1, 8)
|
||||
locations = {agent: rng.choice(regions) for agent in agents}
|
||||
world = _World(
|
||||
locations=locations,
|
||||
order_level={agent: index % order_levels for index, agent in enumerate(agents)},
|
||||
set_level={item: index % set_levels for index, item in enumerate(sets)},
|
||||
numbers={counter: rng.randint(10, 90) for counter in counters},
|
||||
)
|
||||
public = [
|
||||
{"id": f"I{index:04d}", "predicate": "LOCATED_IN", "arguments": [agent, region]}
|
||||
for index, (agent, region) in enumerate(locations.items())
|
||||
]
|
||||
for number in range(4, complexity):
|
||||
action = "ADD_RECORD" if (number + index) % 3 else "INVALIDATE_RECORD"
|
||||
probability = rng.choice((0.2, 0.35, 0.5))
|
||||
condition = Condition("attribute", "class", "EQ", "priority")
|
||||
rules.append(
|
||||
HiddenRule(f"R{number}", f"E{number}", (condition,), probability, action, "record")
|
||||
)
|
||||
return HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=tuple(rules),
|
||||
offset = len(public)
|
||||
public.extend(
|
||||
{
|
||||
"id": f"I{offset + index:04d}",
|
||||
"predicate": "INITIAL_VALUE",
|
||||
"arguments": [counter, value],
|
||||
}
|
||||
for index, (counter, value) in enumerate(world.numbers.items())
|
||||
)
|
||||
hidden = {
|
||||
"order_level": world.order_level,
|
||||
"set_level": world.set_level,
|
||||
"initial_locations": dict(locations),
|
||||
"initial_numbers": dict(world.numbers),
|
||||
}
|
||||
return world, {"public_observations": public, "hidden_state": hidden}
|
||||
|
||||
|
||||
def _support_event_sets(complexity: int) -> list[tuple[str, ...]]:
|
||||
events = [f"E{number}" for number in range(complexity)]
|
||||
cases: list[tuple[str, ...]] = [(event,) for event in events]
|
||||
cases.extend((("E2", "E3"), ("E3", "E2"), ("E0", "E1")))
|
||||
cases.extend(tuple(pair) for pair in combinations(events, 2) if tuple(pair) not in cases)
|
||||
return cases
|
||||
def _event(
|
||||
events: list[WorldEvent],
|
||||
time: int,
|
||||
kind: str,
|
||||
participants: tuple[str, ...],
|
||||
**arguments: Any,
|
||||
) -> WorldEvent:
|
||||
item = WorldEvent(f"V{len(events):05d}", time, kind, participants, arguments)
|
||||
events.append(item)
|
||||
return item
|
||||
|
||||
|
||||
def _generate_one(index: int, complexity: int, rng: random.Random) -> BehaviorSkeleton:
|
||||
system = _build_system(index, complexity, rng)
|
||||
low = {"load": 3, "class": "priority"}
|
||||
high = {"load": 8, "class": "standard"}
|
||||
query_attributes = dict(low)
|
||||
query_events = tuple(f"E{number}" for number in range(complexity))
|
||||
probability = exact_present_probability(system, query_attributes, query_events)
|
||||
if abs(probability - 0.5) < 0.08:
|
||||
raise RuntimeError("generated query has an insufficient probability margin")
|
||||
gold = "PRESENT" if probability > 0.5 else "ABSENT"
|
||||
def _simulate(
|
||||
world: _World,
|
||||
groups: dict[str, list[str]],
|
||||
steps: int,
|
||||
rng: random.Random,
|
||||
) -> list[WorldEvent]:
|
||||
agents, sets, regions, counters = (
|
||||
groups["Entity"],
|
||||
groups["Set"],
|
||||
groups["Region"],
|
||||
groups["Number"],
|
||||
)
|
||||
events: list[WorldEvent] = []
|
||||
kinds = ("ENCOUNTER", "MOVE", "MEMBER", "SUBSET", "DISJOINT", "NUMBER", "CONNECT", "STATE")
|
||||
weights = (30, 12, 11, 15, 7, 11, 8, 6)
|
||||
probabilities = {
|
||||
"ENCOUNTER": 0.78,
|
||||
"MOVE": 0.72,
|
||||
"MEMBER": 0.68,
|
||||
"SUBSET": 0.76,
|
||||
"DISJOINT": 0.64,
|
||||
"NUMBER": 0.82,
|
||||
"CONNECT": 0.55,
|
||||
"STATE": 0.58,
|
||||
}
|
||||
|
||||
trials = 48 if complexity <= 4 else 64
|
||||
support: list[BehaviorCase] = []
|
||||
for case_number, events in enumerate(_support_event_sets(complexity)):
|
||||
attributes = low if case_number % 3 else high
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{case_number:03d}", system, dict(attributes), events, trials, rng
|
||||
for time in range(1, steps + 1):
|
||||
kind = rng.choices(kinds, weights=weights, k=1)[0]
|
||||
if rng.random() >= probabilities[kind]:
|
||||
continue
|
||||
|
||||
if kind == "MOVE":
|
||||
agent = rng.choice(agents)
|
||||
choices = [region for region in regions if region != world.locations[agent]]
|
||||
destination = rng.choice(choices)
|
||||
origin = world.locations[agent]
|
||||
world.locations[agent] = destination
|
||||
_event(events, time, "MOVE", (agent, origin, destination), origin=origin, destination=destination)
|
||||
|
||||
elif kind == "ENCOUNTER":
|
||||
lower_level = rng.randrange(max(world.order_level.values()))
|
||||
lower = [item for item in agents if world.order_level[item] == lower_level]
|
||||
higher = [item for item in agents if world.order_level[item] == lower_level + 1]
|
||||
if not lower or not higher:
|
||||
continue
|
||||
low, high = rng.choice(lower), rng.choice(higher)
|
||||
if world.locations[low] != world.locations[high] or (high, low) in world.above:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(high, low),
|
||||
predicate="ABOVE",
|
||||
subject=high,
|
||||
object=low,
|
||||
region=world.locations[high],
|
||||
)
|
||||
world.above[(high, low)] = item.id
|
||||
|
||||
elif kind == "MEMBER":
|
||||
agent = rng.choice(agents)
|
||||
base_sets = [item for item in sets if world.set_level[item] == 0]
|
||||
container = rng.choice(base_sets)
|
||||
if (agent, container) in world.members:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(agent, container),
|
||||
predicate="MEMBER",
|
||||
subject=agent,
|
||||
object=container,
|
||||
)
|
||||
world.members[(agent, container)] = item.id
|
||||
|
||||
elif kind == "SUBSET":
|
||||
lower_level = rng.randrange(max(world.set_level.values()))
|
||||
lower = [item for item in sets if world.set_level[item] == lower_level]
|
||||
higher = [item for item in sets if world.set_level[item] == lower_level + 1]
|
||||
child, parent = rng.choice(lower), rng.choice(higher)
|
||||
if (child, parent) in world.subsets or any(left == child for left, _right in world.subsets):
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(child, parent),
|
||||
predicate="SUBSET",
|
||||
subject=child,
|
||||
object=parent,
|
||||
)
|
||||
world.subsets[(child, parent)] = item.id
|
||||
|
||||
elif kind == "DISJOINT":
|
||||
level = rng.randrange(max(world.set_level.values()) + 1)
|
||||
same_level = [item for item in sets if world.set_level[item] == level]
|
||||
if len(same_level) < 2:
|
||||
continue
|
||||
left, right = rng.sample(same_level, 2)
|
||||
key = frozenset((left, right))
|
||||
if key in world.disjoint:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(left, right),
|
||||
predicate="DISJOINT",
|
||||
subject=left,
|
||||
object=right,
|
||||
)
|
||||
world.disjoint[key] = item.id
|
||||
|
||||
elif kind == "NUMBER":
|
||||
counter = rng.choice(counters)
|
||||
delta = rng.choice((-7, -5, -3, 2, 4, 6))
|
||||
before = world.numbers[counter]
|
||||
after = max(0, before + delta)
|
||||
world.numbers[counter] = after
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"UPDATE_NUMBER",
|
||||
(counter,),
|
||||
operation="ADD" if delta >= 0 else "SUBTRACT",
|
||||
amount=abs(delta),
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
|
||||
elif kind == "CONNECT":
|
||||
left, right = rng.sample(agents, 2)
|
||||
key = frozenset((left, right))
|
||||
if key in world.connected:
|
||||
continue
|
||||
world.connected.add(key)
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(left, right),
|
||||
predicate="CONNECTED",
|
||||
subject=left,
|
||||
object=right,
|
||||
)
|
||||
|
||||
else:
|
||||
agent = rng.choice(agents)
|
||||
flag = rng.choice(("ACTIVE", "SEALED", "VERIFIED"))
|
||||
before = world.flags.get((agent, flag), False)
|
||||
after = not before
|
||||
world.flags[(agent, flag)] = after
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"SET_STATE",
|
||||
(agent,),
|
||||
state=flag,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
def _shortest_path(
|
||||
edges: dict[tuple[str, str], str], minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in edges.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
candidates: list[tuple[str, str, tuple[str, ...], tuple[str, ...]]] = []
|
||||
for source in adjacency:
|
||||
queue = deque([(source, (source,), ())])
|
||||
seen = {source}
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
if len(event_ids) >= minimum:
|
||||
candidates.append((source, node, nodes, event_ids))
|
||||
continue
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda item: (len(item[3]), item[0], item[1]))
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _membership_path(
|
||||
world: _World, minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
subset_path = _shortest_path(world.subsets, minimum - 1)
|
||||
if subset_path is None:
|
||||
return None
|
||||
candidates = []
|
||||
for (entity, base_set), member_event in world.members.items():
|
||||
path = _shortest_path_from(world.subsets, base_set, minimum - 1)
|
||||
if path:
|
||||
target, nodes, event_ids = path
|
||||
candidates.append((entity, target, (member_event,) + event_ids, (entity,) + nodes))
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda item: (len(item[2]), item[0], item[1]))
|
||||
entity, target, event_ids, nodes = candidates[0]
|
||||
return entity, target, nodes, event_ids
|
||||
|
||||
|
||||
def _membership_negative(
|
||||
world: _World, minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in world.subsets.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
for (entity, base_set), member_event in sorted(world.members.items()):
|
||||
reachable = {base_set}
|
||||
queue = deque([(base_set, (base_set,), ())])
|
||||
paths: list[tuple[str, tuple[str, ...], tuple[str, ...]]] = []
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
paths.append((node, nodes, event_ids))
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in reachable:
|
||||
reachable.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
for known_set, nodes, subset_events in paths:
|
||||
if 1 + len(subset_events) < minimum - 1:
|
||||
continue
|
||||
for pair, disjoint_event in world.disjoint.items():
|
||||
if known_set not in pair:
|
||||
continue
|
||||
other = next(item for item in pair if item != known_set)
|
||||
if other in reachable:
|
||||
continue
|
||||
proof_ids = (member_event,) + subset_events + (disjoint_event,)
|
||||
proof_nodes = (entity,) + nodes + (other,)
|
||||
return entity, other, proof_nodes, proof_ids
|
||||
return None
|
||||
|
||||
|
||||
def _shortest_path_from(
|
||||
edges: dict[tuple[str, str], str], source: str, minimum: int
|
||||
) -> tuple[str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in edges.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
queue = deque([(source, (source,), ())])
|
||||
seen = {source}
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
if len(event_ids) >= minimum:
|
||||
return node, nodes, event_ids
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
return None
|
||||
|
||||
|
||||
def _choose_query(
|
||||
index: int, complexity: int, world: _World, events: list[WorldEvent]
|
||||
) -> tuple[Query, str, tuple[str, ...], tuple[str, ...], str] | None:
|
||||
if index % 2 == 0:
|
||||
path = _shortest_path(world.above, complexity)
|
||||
if path is None:
|
||||
return None
|
||||
source, target, nodes, proof_ids = path
|
||||
positive = index % 4 == 0
|
||||
arguments = (source, target) if positive else (target, source)
|
||||
gold = "YES" if positive else "NO"
|
||||
facts = tuple(
|
||||
f"ABOVE({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(len(nodes) - 1)
|
||||
)
|
||||
if not positive:
|
||||
facts += (f"ABOVE({source},{target}) therefore NOT ABOVE({target},{source})",)
|
||||
return Query("ABOVE", arguments, events[-1].time), gold, proof_ids, facts, "spatial_order"
|
||||
|
||||
if index % 4 == 1:
|
||||
negative = _membership_negative(world, complexity)
|
||||
if negative is None:
|
||||
return None
|
||||
entity, target, nodes, proof_ids = negative
|
||||
known_set, disjoint_set = nodes[-2], nodes[-1]
|
||||
facts = [f"MEMBER({entity},{nodes[1]})"]
|
||||
facts.extend(
|
||||
f"SUBSET({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(1, len(nodes) - 2)
|
||||
)
|
||||
facts.extend(
|
||||
(
|
||||
f"DISJOINT({known_set},{disjoint_set})",
|
||||
f"therefore NOT MEMBER({entity},{target})",
|
||||
)
|
||||
)
|
||||
# Keeping events fixed while changing load makes the veto guard observable.
|
||||
next_id = len(support)
|
||||
support.append(
|
||||
_observe_case(f"history-{next_id:03d}", system, dict(low), ("E0", "E1"), trials, rng)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 1:03d}", system, dict(high), ("E0", "E1"), trials, rng
|
||||
return (
|
||||
Query("MEMBER", (entity, target), events[-1].time),
|
||||
"NO",
|
||||
proof_ids,
|
||||
tuple(facts),
|
||||
"membership_disjointness",
|
||||
)
|
||||
|
||||
membership = _membership_path(world, complexity)
|
||||
if membership is None:
|
||||
return None
|
||||
entity, target, nodes, proof_ids = membership
|
||||
facts = [f"MEMBER({entity},{nodes[1]})"]
|
||||
facts.extend(
|
||||
f"SUBSET({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(1, len(nodes) - 1)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 2:03d}",
|
||||
system,
|
||||
dict(low),
|
||||
("E0", "E2", "E3"),
|
||||
trials,
|
||||
rng,
|
||||
)
|
||||
return (
|
||||
Query("MEMBER", (entity, target), events[-1].time),
|
||||
"YES",
|
||||
proof_ids,
|
||||
tuple(facts),
|
||||
"membership_inheritance",
|
||||
)
|
||||
|
||||
query = BehaviorCase(
|
||||
id="query",
|
||||
attributes=query_attributes,
|
||||
events=query_events,
|
||||
trials=1,
|
||||
observed_present=None,
|
||||
observed_absent=None,
|
||||
|
||||
def _final_state(world: _World) -> dict[str, Any]:
|
||||
return {
|
||||
"locations": dict(world.locations),
|
||||
"above": [[left, right] for left, right in sorted(world.above)],
|
||||
"subsets": [[left, right] for left, right in sorted(world.subsets)],
|
||||
"members": [[entity, container] for entity, container in sorted(world.members)],
|
||||
"disjoint": [sorted(pair) for pair in sorted(world.disjoint, key=lambda item: sorted(item))],
|
||||
"connected": [sorted(pair) for pair in sorted(world.connected, key=lambda item: sorted(item))],
|
||||
"numbers": dict(world.numbers),
|
||||
"flags": [
|
||||
{"entity": entity, "state": state, "value": value}
|
||||
for (entity, state), value in sorted(world.flags.items())
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _generate_one(
|
||||
index: int,
|
||||
complexity: int,
|
||||
entity_count: int,
|
||||
simulation_steps: int,
|
||||
rng: random.Random,
|
||||
) -> WorldSkeleton:
|
||||
entities, groups = _entities(entity_count)
|
||||
world, initial_state = _initial_world(groups, rng, complexity)
|
||||
events = _simulate(world, groups, simulation_steps, rng)
|
||||
if not events:
|
||||
raise RuntimeError("simulation produced no events")
|
||||
selected = _choose_query(index, complexity, world, events)
|
||||
if selected is None:
|
||||
raise RuntimeError("simulation did not produce a sufficiently deep proof")
|
||||
query, gold, proof_ids, proof_facts, query_family = selected
|
||||
direct = any(
|
||||
event.kind == "RELATE"
|
||||
and event.arguments.get("predicate") == query.predicate
|
||||
and (event.arguments.get("subject"), event.arguments.get("object")) == query.arguments
|
||||
for event in events
|
||||
)
|
||||
trace = tuple(
|
||||
[f"query events: {' -> '.join(query_events)}"]
|
||||
+ [
|
||||
f"{rule.id}: on {rule.on_event}, p={rule.probability}, {rule.action}({rule.target})"
|
||||
for rule in system.rules
|
||||
]
|
||||
+ [f"exact P(PRESENT)={probability:.8f}", f"gold={gold}"]
|
||||
)
|
||||
return BehaviorSkeleton(
|
||||
id=f"dynamic-{index:04d}",
|
||||
family="stochastic_behavioral_system",
|
||||
if direct:
|
||||
raise RuntimeError("query was directly stated in the event history")
|
||||
checks = {
|
||||
"direct_query_event_absent": True,
|
||||
"proof_length": len(proof_ids),
|
||||
"minimum_proof_length": complexity,
|
||||
"event_count": len(events),
|
||||
"proof_event_ratio": len(proof_ids) / len(events),
|
||||
"query_family": query_family,
|
||||
"weak_baselines": {
|
||||
"direct_fact": False,
|
||||
"proof_shorter_than_requested": len(proof_ids) < complexity,
|
||||
"proof_is_large_fraction_of_history": len(proof_ids) / len(events) >= 0.05,
|
||||
},
|
||||
}
|
||||
return WorldSkeleton(
|
||||
id=f"world-{index:04d}",
|
||||
family="dynamic_multi_logic_world",
|
||||
complexity=complexity,
|
||||
event_vocabulary=tuple(f"E{number}" for number in range(complexity)),
|
||||
attribute_vocabulary={"load": "integer", "class": "category"},
|
||||
observation_protocol=(
|
||||
"Each trial begins with an absent record and no internal flags.",
|
||||
"Events occur from left to right; the same hidden system is used in every trial.",
|
||||
"PRESENT and ABSENT counts are final observations over independent repeated trials.",
|
||||
"For the query, choose which final outcome is more likely; an exact probability is not required.",
|
||||
),
|
||||
hidden_system=system,
|
||||
support_cases=tuple(support),
|
||||
query_case=query,
|
||||
entity_count=entity_count,
|
||||
simulation_steps=simulation_steps,
|
||||
hidden_program=_program(),
|
||||
entities=tuple(entities),
|
||||
initial_state=initial_state,
|
||||
events=tuple(events),
|
||||
final_state=_final_state(world),
|
||||
query=query,
|
||||
answer_options=ANSWER_OPTIONS,
|
||||
gold_answer=gold,
|
||||
gold_present_probability=probability,
|
||||
latent_trace=trace,
|
||||
proof_event_ids=proof_ids,
|
||||
proof_facts=proof_facts,
|
||||
generator_checks=checks,
|
||||
)
|
||||
|
||||
|
||||
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[BehaviorSkeleton]:
|
||||
def generate_skeletons(
|
||||
count: int,
|
||||
length: str | int,
|
||||
seed: int = 0,
|
||||
entity_count: int = 128,
|
||||
simulation_steps: int = 1000,
|
||||
) -> list[WorldSkeleton]:
|
||||
if count < 1:
|
||||
raise ValueError("count must be positive")
|
||||
if simulation_steps < 100:
|
||||
raise ValueError("simulation_steps must be at least 100")
|
||||
complexity = resolve_length(length)
|
||||
rng = random.Random(seed)
|
||||
generated: list[BehaviorSkeleton] = []
|
||||
generated: list[WorldSkeleton] = []
|
||||
for index in range(count):
|
||||
for _attempt in range(100):
|
||||
for _attempt in range(60):
|
||||
try:
|
||||
generated.append(_generate_one(index, complexity, rng))
|
||||
generated.append(
|
||||
_generate_one(index, complexity, entity_count, simulation_steps, rng)
|
||||
)
|
||||
break
|
||||
except RuntimeError as exc:
|
||||
if "insufficient probability margin" not in str(exc):
|
||||
raise
|
||||
except RuntimeError:
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError("could not generate a query with a stable probability margin")
|
||||
raise RuntimeError("could not generate a world with the requested proof complexity")
|
||||
return generated
|
||||
|
||||
Reference in New Issue
Block a user