Generate proof-bearing dynamic world histories
This commit is contained in:
+579
-216
@@ -1,26 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from dataclasses import dataclass
|
||||
from itertools import combinations
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
|
||||
from .models import Entity, HiddenProgram, Query, TransitionRule, WorldEvent, WorldSkeleton
|
||||
|
||||
|
||||
LENGTH_PRESETS = {"short": 4, "medium": 6, "long": 8}
|
||||
ANSWER_OPTIONS = ("PRESENT", "ABSENT")
|
||||
ANSWER_OPTIONS = ("YES", "NO")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _State:
|
||||
present: bool = False
|
||||
invalidated: bool = False
|
||||
flags: frozenset[str] = frozenset()
|
||||
@dataclass
|
||||
class _World:
|
||||
locations: dict[str, str]
|
||||
order_level: dict[str, int]
|
||||
set_level: dict[str, int]
|
||||
above: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
subsets: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
members: dict[tuple[str, str], str] = field(default_factory=dict)
|
||||
disjoint: dict[frozenset[str], str] = field(default_factory=dict)
|
||||
connected: set[frozenset[str]] = field(default_factory=set)
|
||||
numbers: dict[str, int] = field(default_factory=dict)
|
||||
flags: dict[tuple[str, str], bool] = field(default_factory=dict)
|
||||
|
||||
|
||||
def resolve_length(value: str | int) -> int:
|
||||
"""Resolve the number of distinct events in the held-out query."""
|
||||
"""Resolve required proof length, independently of event-log or text length."""
|
||||
if isinstance(value, int):
|
||||
result = value
|
||||
elif value in LENGTH_PRESETS:
|
||||
@@ -29,246 +36,602 @@ def resolve_length(value: str | int) -> int:
|
||||
try:
|
||||
result = int(value)
|
||||
except ValueError as exc:
|
||||
choices = ", ".join(LENGTH_PRESETS)
|
||||
raise ValueError(f"length must be an integer or one of: {choices}") from exc
|
||||
if not 4 <= result <= 10:
|
||||
raise ValueError("length must be between 4 and 10")
|
||||
raise ValueError(f"length must be an integer or one of: {', '.join(LENGTH_PRESETS)}") from exc
|
||||
if not 3 <= result <= 10:
|
||||
raise ValueError("length must be between 3 and 10")
|
||||
return result
|
||||
|
||||
|
||||
def _condition_holds(condition: Condition, attributes: dict[str, Any], state: _State) -> bool:
|
||||
if condition.source == "attribute":
|
||||
actual = attributes.get(condition.key)
|
||||
elif condition.source == "state":
|
||||
actual = condition.key in state.flags
|
||||
else:
|
||||
raise ValueError(f"unknown condition source: {condition.source}")
|
||||
def _program() -> HiddenProgram:
|
||||
return HiddenProgram(
|
||||
types=("Entity", "Set", "Region", "Number", "Boolean", "State"),
|
||||
relations=(
|
||||
"EQUAL",
|
||||
"NOT_EQUAL",
|
||||
"ABOVE",
|
||||
"BELOW",
|
||||
"MEMBER",
|
||||
"SUBSET",
|
||||
"DISJOINT",
|
||||
"ADJACENT",
|
||||
"CONNECTED",
|
||||
"CAUSES",
|
||||
"LOCATED_IN",
|
||||
),
|
||||
operators=(
|
||||
"NOT",
|
||||
"AND",
|
||||
"OR",
|
||||
"IMPLIES",
|
||||
"ADD",
|
||||
"SUBTRACT",
|
||||
"COMPARE",
|
||||
"UNION",
|
||||
"INTERSECTION",
|
||||
"MOVE",
|
||||
"SWAP",
|
||||
"SET",
|
||||
"COMPOSE",
|
||||
),
|
||||
quantifiers=("ALL", "EXISTS"),
|
||||
structures=("CHAIN", "BRANCH", "MERGE", "CYCLE", "NEST", "CONDITION", "SEQUENCE"),
|
||||
inference_rules=(
|
||||
"ABOVE(x,y) AND ABOVE(y,z) IMPLIES ABOVE(x,z)",
|
||||
"ABOVE(x,y) IMPLIES BELOW(y,x)",
|
||||
"MEMBER(x,A) AND SUBSET(A,B) IMPLIES MEMBER(x,B)",
|
||||
"MEMBER(x,A) AND DISJOINT(A,B) IMPLIES NOT MEMBER(x,B)",
|
||||
"CONNECTED is symmetric",
|
||||
"Numeric updates compose in temporal order",
|
||||
),
|
||||
transition_rules=(
|
||||
TransitionRule(
|
||||
"move",
|
||||
"SELECT_ENTITY",
|
||||
("target region differs from current region",),
|
||||
0.72,
|
||||
("MOVE(entity, region)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"encounter-order",
|
||||
"COLOCATED_PAIR",
|
||||
("entities share a region", "latent order levels are adjacent"),
|
||||
0.78,
|
||||
("SET ABOVE(higher, lower)", "SET BELOW(lower, higher)"),
|
||||
),
|
||||
TransitionRule(
|
||||
"membership",
|
||||
"ENTITY_SET_CONTACT",
|
||||
("set is at base level",),
|
||||
0.68,
|
||||
("SET MEMBER(entity, set)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"subset",
|
||||
"SET_PAIR_CONTACT",
|
||||
("set levels are adjacent",),
|
||||
0.76,
|
||||
("SET SUBSET(lower, higher)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"disjoint",
|
||||
"SET_PAIR_CONTACT",
|
||||
("sets have equal level", "sets are distinct"),
|
||||
0.64,
|
||||
("SET DISJOINT(left, right)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"numeric-update",
|
||||
"COUNTER_PULSE",
|
||||
(),
|
||||
0.82,
|
||||
("ADD or SUBTRACT(counter, amount)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"connect",
|
||||
"ENTITY_PAIR_CONTACT",
|
||||
(),
|
||||
0.55,
|
||||
("SET CONNECTED(left, right)",),
|
||||
),
|
||||
TransitionRule(
|
||||
"state-change",
|
||||
"ENTITY_SIGNAL",
|
||||
(),
|
||||
0.58,
|
||||
("SET Boolean state",),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
operations = {
|
||||
"EQ": lambda left, right: left == right,
|
||||
"LE": lambda left, right: left <= right,
|
||||
"GT": lambda left, right: left > right,
|
||||
"IS_SET": lambda left, right: left is bool(right),
|
||||
|
||||
def _entities(entity_count: int) -> tuple[list[Entity], dict[str, list[str]]]:
|
||||
if entity_count < 48:
|
||||
raise ValueError("entity_count must be at least 48")
|
||||
region_count = max(4, entity_count // 32)
|
||||
set_count = max(24, entity_count // 5)
|
||||
counter_count = max(8, entity_count // 16)
|
||||
agent_count = entity_count - region_count - set_count - counter_count
|
||||
groups = {
|
||||
"Entity": [f"E{i:03d}" for i in range(agent_count)],
|
||||
"Set": [f"S{i:03d}" for i in range(set_count)],
|
||||
"Region": [f"R{i:03d}" for i in range(region_count)],
|
||||
"Number": [f"N{i:03d}" for i in range(counter_count)],
|
||||
}
|
||||
try:
|
||||
return operations[condition.operator](actual, condition.value)
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unknown condition operator: {condition.operator}") from exc
|
||||
entities = [Entity(identifier, kind) for kind, values in groups.items() for identifier in values]
|
||||
return entities, groups
|
||||
|
||||
|
||||
def _apply_action(state: _State, rule: HiddenRule) -> _State:
|
||||
if rule.action == "SET_FLAG":
|
||||
return _State(state.present, state.invalidated, state.flags | {rule.target})
|
||||
if rule.action == "ADD_RECORD":
|
||||
if state.invalidated:
|
||||
return state
|
||||
return _State(True, False, state.flags)
|
||||
if rule.action == "INVALIDATE_RECORD":
|
||||
return _State(False, True, state.flags)
|
||||
raise ValueError(f"unknown action: {rule.action}")
|
||||
|
||||
|
||||
def exact_present_probability(
|
||||
system: HiddenSystem, attributes: dict[str, Any], events: tuple[str, ...]
|
||||
) -> float:
|
||||
"""Enumerate all stochastic branches while respecting event and state order."""
|
||||
initial = _State(present=system.initial_record == "PRESENT")
|
||||
distribution: dict[_State, float] = {initial: 1.0}
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
next_distribution: dict[_State, float] = {}
|
||||
for state, mass in distribution.items():
|
||||
if not all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + mass
|
||||
continue
|
||||
inactive_mass = mass * (1.0 - rule.probability)
|
||||
active_state = _apply_action(state, rule)
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + inactive_mass
|
||||
next_distribution[active_state] = (
|
||||
next_distribution.get(active_state, 0.0) + mass * rule.probability
|
||||
)
|
||||
distribution = next_distribution
|
||||
return sum(mass for state, mass in distribution.items() if state.present)
|
||||
|
||||
|
||||
def _simulate_once(
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
rng: random.Random,
|
||||
) -> bool:
|
||||
state = _State(present=system.initial_record == "PRESENT")
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
if all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
if rng.random() < rule.probability:
|
||||
state = _apply_action(state, rule)
|
||||
return state.present
|
||||
|
||||
|
||||
def _observe_case(
|
||||
case_id: str,
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
trials: int,
|
||||
rng: random.Random,
|
||||
) -> BehaviorCase:
|
||||
present = sum(_simulate_once(system, attributes, events, rng) for _ in range(trials))
|
||||
return BehaviorCase(
|
||||
id=case_id,
|
||||
attributes=attributes,
|
||||
events=events,
|
||||
trials=trials,
|
||||
observed_present=present,
|
||||
observed_absent=trials - present,
|
||||
def _initial_world(
|
||||
groups: dict[str, list[str]], rng: random.Random, proof_length: int
|
||||
) -> tuple[_World, dict[str, Any]]:
|
||||
agents, sets, regions, counters = (
|
||||
groups["Entity"],
|
||||
groups["Set"],
|
||||
groups["Region"],
|
||||
groups["Number"],
|
||||
)
|
||||
|
||||
|
||||
def _build_system(index: int, complexity: int, rng: random.Random) -> HiddenSystem:
|
||||
"""Build a finite stochastic program with a gated rule and veto conflicts."""
|
||||
present_favored = index % 2 == 0
|
||||
add_probability = rng.choice((0.85, 0.9, 0.95)) if present_favored else rng.choice((0.65, 0.75))
|
||||
veto_probability = rng.choice((0.15, 0.25)) if present_favored else rng.choice((0.7, 0.8, 0.9))
|
||||
marker_probability = rng.choice((0.6, 0.75, 0.9))
|
||||
gated_action = "ADD_RECORD" if present_favored else "INVALIDATE_RECORD"
|
||||
gated_probability = rng.choice((0.6, 0.75, 0.85))
|
||||
|
||||
rules = [
|
||||
HiddenRule("R0", "E0", (), add_probability, "ADD_RECORD", "record"),
|
||||
HiddenRule(
|
||||
"R1",
|
||||
"E1",
|
||||
(Condition("attribute", "load", "LE", 4),),
|
||||
veto_probability,
|
||||
"INVALIDATE_RECORD",
|
||||
"record",
|
||||
),
|
||||
HiddenRule("R2", "E2", (), marker_probability, "SET_FLAG", "marked"),
|
||||
HiddenRule(
|
||||
"R3",
|
||||
"E3",
|
||||
(Condition("state", "marked", "IS_SET", True),),
|
||||
gated_probability,
|
||||
gated_action,
|
||||
"record",
|
||||
),
|
||||
order_levels = max(proof_length + 2, 8)
|
||||
set_levels = max(proof_length + 1, 8)
|
||||
locations = {agent: rng.choice(regions) for agent in agents}
|
||||
world = _World(
|
||||
locations=locations,
|
||||
order_level={agent: index % order_levels for index, agent in enumerate(agents)},
|
||||
set_level={item: index % set_levels for index, item in enumerate(sets)},
|
||||
numbers={counter: rng.randint(10, 90) for counter in counters},
|
||||
)
|
||||
public = [
|
||||
{"id": f"I{index:04d}", "predicate": "LOCATED_IN", "arguments": [agent, region]}
|
||||
for index, (agent, region) in enumerate(locations.items())
|
||||
]
|
||||
for number in range(4, complexity):
|
||||
action = "ADD_RECORD" if (number + index) % 3 else "INVALIDATE_RECORD"
|
||||
probability = rng.choice((0.2, 0.35, 0.5))
|
||||
condition = Condition("attribute", "class", "EQ", "priority")
|
||||
rules.append(
|
||||
HiddenRule(f"R{number}", f"E{number}", (condition,), probability, action, "record")
|
||||
)
|
||||
return HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=tuple(rules),
|
||||
offset = len(public)
|
||||
public.extend(
|
||||
{
|
||||
"id": f"I{offset + index:04d}",
|
||||
"predicate": "INITIAL_VALUE",
|
||||
"arguments": [counter, value],
|
||||
}
|
||||
for index, (counter, value) in enumerate(world.numbers.items())
|
||||
)
|
||||
hidden = {
|
||||
"order_level": world.order_level,
|
||||
"set_level": world.set_level,
|
||||
"initial_locations": dict(locations),
|
||||
"initial_numbers": dict(world.numbers),
|
||||
}
|
||||
return world, {"public_observations": public, "hidden_state": hidden}
|
||||
|
||||
|
||||
def _support_event_sets(complexity: int) -> list[tuple[str, ...]]:
|
||||
events = [f"E{number}" for number in range(complexity)]
|
||||
cases: list[tuple[str, ...]] = [(event,) for event in events]
|
||||
cases.extend((("E2", "E3"), ("E3", "E2"), ("E0", "E1")))
|
||||
cases.extend(tuple(pair) for pair in combinations(events, 2) if tuple(pair) not in cases)
|
||||
return cases
|
||||
def _event(
|
||||
events: list[WorldEvent],
|
||||
time: int,
|
||||
kind: str,
|
||||
participants: tuple[str, ...],
|
||||
**arguments: Any,
|
||||
) -> WorldEvent:
|
||||
item = WorldEvent(f"V{len(events):05d}", time, kind, participants, arguments)
|
||||
events.append(item)
|
||||
return item
|
||||
|
||||
|
||||
def _generate_one(index: int, complexity: int, rng: random.Random) -> BehaviorSkeleton:
|
||||
system = _build_system(index, complexity, rng)
|
||||
low = {"load": 3, "class": "priority"}
|
||||
high = {"load": 8, "class": "standard"}
|
||||
query_attributes = dict(low)
|
||||
query_events = tuple(f"E{number}" for number in range(complexity))
|
||||
probability = exact_present_probability(system, query_attributes, query_events)
|
||||
if abs(probability - 0.5) < 0.08:
|
||||
raise RuntimeError("generated query has an insufficient probability margin")
|
||||
gold = "PRESENT" if probability > 0.5 else "ABSENT"
|
||||
def _simulate(
|
||||
world: _World,
|
||||
groups: dict[str, list[str]],
|
||||
steps: int,
|
||||
rng: random.Random,
|
||||
) -> list[WorldEvent]:
|
||||
agents, sets, regions, counters = (
|
||||
groups["Entity"],
|
||||
groups["Set"],
|
||||
groups["Region"],
|
||||
groups["Number"],
|
||||
)
|
||||
events: list[WorldEvent] = []
|
||||
kinds = ("ENCOUNTER", "MOVE", "MEMBER", "SUBSET", "DISJOINT", "NUMBER", "CONNECT", "STATE")
|
||||
weights = (30, 12, 11, 15, 7, 11, 8, 6)
|
||||
probabilities = {
|
||||
"ENCOUNTER": 0.78,
|
||||
"MOVE": 0.72,
|
||||
"MEMBER": 0.68,
|
||||
"SUBSET": 0.76,
|
||||
"DISJOINT": 0.64,
|
||||
"NUMBER": 0.82,
|
||||
"CONNECT": 0.55,
|
||||
"STATE": 0.58,
|
||||
}
|
||||
|
||||
trials = 48 if complexity <= 4 else 64
|
||||
support: list[BehaviorCase] = []
|
||||
for case_number, events in enumerate(_support_event_sets(complexity)):
|
||||
attributes = low if case_number % 3 else high
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{case_number:03d}", system, dict(attributes), events, trials, rng
|
||||
for time in range(1, steps + 1):
|
||||
kind = rng.choices(kinds, weights=weights, k=1)[0]
|
||||
if rng.random() >= probabilities[kind]:
|
||||
continue
|
||||
|
||||
if kind == "MOVE":
|
||||
agent = rng.choice(agents)
|
||||
choices = [region for region in regions if region != world.locations[agent]]
|
||||
destination = rng.choice(choices)
|
||||
origin = world.locations[agent]
|
||||
world.locations[agent] = destination
|
||||
_event(events, time, "MOVE", (agent, origin, destination), origin=origin, destination=destination)
|
||||
|
||||
elif kind == "ENCOUNTER":
|
||||
lower_level = rng.randrange(max(world.order_level.values()))
|
||||
lower = [item for item in agents if world.order_level[item] == lower_level]
|
||||
higher = [item for item in agents if world.order_level[item] == lower_level + 1]
|
||||
if not lower or not higher:
|
||||
continue
|
||||
low, high = rng.choice(lower), rng.choice(higher)
|
||||
if world.locations[low] != world.locations[high] or (high, low) in world.above:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(high, low),
|
||||
predicate="ABOVE",
|
||||
subject=high,
|
||||
object=low,
|
||||
region=world.locations[high],
|
||||
)
|
||||
world.above[(high, low)] = item.id
|
||||
|
||||
elif kind == "MEMBER":
|
||||
agent = rng.choice(agents)
|
||||
base_sets = [item for item in sets if world.set_level[item] == 0]
|
||||
container = rng.choice(base_sets)
|
||||
if (agent, container) in world.members:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(agent, container),
|
||||
predicate="MEMBER",
|
||||
subject=agent,
|
||||
object=container,
|
||||
)
|
||||
world.members[(agent, container)] = item.id
|
||||
|
||||
elif kind == "SUBSET":
|
||||
lower_level = rng.randrange(max(world.set_level.values()))
|
||||
lower = [item for item in sets if world.set_level[item] == lower_level]
|
||||
higher = [item for item in sets if world.set_level[item] == lower_level + 1]
|
||||
child, parent = rng.choice(lower), rng.choice(higher)
|
||||
if (child, parent) in world.subsets or any(left == child for left, _right in world.subsets):
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(child, parent),
|
||||
predicate="SUBSET",
|
||||
subject=child,
|
||||
object=parent,
|
||||
)
|
||||
world.subsets[(child, parent)] = item.id
|
||||
|
||||
elif kind == "DISJOINT":
|
||||
level = rng.randrange(max(world.set_level.values()) + 1)
|
||||
same_level = [item for item in sets if world.set_level[item] == level]
|
||||
if len(same_level) < 2:
|
||||
continue
|
||||
left, right = rng.sample(same_level, 2)
|
||||
key = frozenset((left, right))
|
||||
if key in world.disjoint:
|
||||
continue
|
||||
item = _event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(left, right),
|
||||
predicate="DISJOINT",
|
||||
subject=left,
|
||||
object=right,
|
||||
)
|
||||
world.disjoint[key] = item.id
|
||||
|
||||
elif kind == "NUMBER":
|
||||
counter = rng.choice(counters)
|
||||
delta = rng.choice((-7, -5, -3, 2, 4, 6))
|
||||
before = world.numbers[counter]
|
||||
after = max(0, before + delta)
|
||||
world.numbers[counter] = after
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"UPDATE_NUMBER",
|
||||
(counter,),
|
||||
operation="ADD" if delta >= 0 else "SUBTRACT",
|
||||
amount=abs(delta),
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
|
||||
elif kind == "CONNECT":
|
||||
left, right = rng.sample(agents, 2)
|
||||
key = frozenset((left, right))
|
||||
if key in world.connected:
|
||||
continue
|
||||
world.connected.add(key)
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"RELATE",
|
||||
(left, right),
|
||||
predicate="CONNECTED",
|
||||
subject=left,
|
||||
object=right,
|
||||
)
|
||||
|
||||
else:
|
||||
agent = rng.choice(agents)
|
||||
flag = rng.choice(("ACTIVE", "SEALED", "VERIFIED"))
|
||||
before = world.flags.get((agent, flag), False)
|
||||
after = not before
|
||||
world.flags[(agent, flag)] = after
|
||||
_event(
|
||||
events,
|
||||
time,
|
||||
"SET_STATE",
|
||||
(agent,),
|
||||
state=flag,
|
||||
before=before,
|
||||
after=after,
|
||||
)
|
||||
return events
|
||||
|
||||
|
||||
def _shortest_path(
|
||||
edges: dict[tuple[str, str], str], minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in edges.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
candidates: list[tuple[str, str, tuple[str, ...], tuple[str, ...]]] = []
|
||||
for source in adjacency:
|
||||
queue = deque([(source, (source,), ())])
|
||||
seen = {source}
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
if len(event_ids) >= minimum:
|
||||
candidates.append((source, node, nodes, event_ids))
|
||||
continue
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda item: (len(item[3]), item[0], item[1]))
|
||||
return candidates[0]
|
||||
|
||||
|
||||
def _membership_path(
|
||||
world: _World, minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
subset_path = _shortest_path(world.subsets, minimum - 1)
|
||||
if subset_path is None:
|
||||
return None
|
||||
candidates = []
|
||||
for (entity, base_set), member_event in world.members.items():
|
||||
path = _shortest_path_from(world.subsets, base_set, minimum - 1)
|
||||
if path:
|
||||
target, nodes, event_ids = path
|
||||
candidates.append((entity, target, (member_event,) + event_ids, (entity,) + nodes))
|
||||
if not candidates:
|
||||
return None
|
||||
candidates.sort(key=lambda item: (len(item[2]), item[0], item[1]))
|
||||
entity, target, event_ids, nodes = candidates[0]
|
||||
return entity, target, nodes, event_ids
|
||||
|
||||
|
||||
def _membership_negative(
|
||||
world: _World, minimum: int
|
||||
) -> tuple[str, str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in world.subsets.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
for (entity, base_set), member_event in sorted(world.members.items()):
|
||||
reachable = {base_set}
|
||||
queue = deque([(base_set, (base_set,), ())])
|
||||
paths: list[tuple[str, tuple[str, ...], tuple[str, ...]]] = []
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
paths.append((node, nodes, event_ids))
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in reachable:
|
||||
reachable.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
for known_set, nodes, subset_events in paths:
|
||||
if 1 + len(subset_events) < minimum - 1:
|
||||
continue
|
||||
for pair, disjoint_event in world.disjoint.items():
|
||||
if known_set not in pair:
|
||||
continue
|
||||
other = next(item for item in pair if item != known_set)
|
||||
if other in reachable:
|
||||
continue
|
||||
proof_ids = (member_event,) + subset_events + (disjoint_event,)
|
||||
proof_nodes = (entity,) + nodes + (other,)
|
||||
return entity, other, proof_nodes, proof_ids
|
||||
return None
|
||||
|
||||
|
||||
def _shortest_path_from(
|
||||
edges: dict[tuple[str, str], str], source: str, minimum: int
|
||||
) -> tuple[str, tuple[str, ...], tuple[str, ...]] | None:
|
||||
adjacency: dict[str, list[tuple[str, str]]] = {}
|
||||
for (left, right), event_id in edges.items():
|
||||
adjacency.setdefault(left, []).append((right, event_id))
|
||||
queue = deque([(source, (source,), ())])
|
||||
seen = {source}
|
||||
while queue:
|
||||
node, nodes, event_ids = queue.popleft()
|
||||
if len(event_ids) >= minimum:
|
||||
return node, nodes, event_ids
|
||||
for target, event_id in adjacency.get(node, []):
|
||||
if target not in seen:
|
||||
seen.add(target)
|
||||
queue.append((target, nodes + (target,), event_ids + (event_id,)))
|
||||
return None
|
||||
|
||||
|
||||
def _choose_query(
|
||||
index: int, complexity: int, world: _World, events: list[WorldEvent]
|
||||
) -> tuple[Query, str, tuple[str, ...], tuple[str, ...], str] | None:
|
||||
if index % 2 == 0:
|
||||
path = _shortest_path(world.above, complexity)
|
||||
if path is None:
|
||||
return None
|
||||
source, target, nodes, proof_ids = path
|
||||
positive = index % 4 == 0
|
||||
arguments = (source, target) if positive else (target, source)
|
||||
gold = "YES" if positive else "NO"
|
||||
facts = tuple(
|
||||
f"ABOVE({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(len(nodes) - 1)
|
||||
)
|
||||
if not positive:
|
||||
facts += (f"ABOVE({source},{target}) therefore NOT ABOVE({target},{source})",)
|
||||
return Query("ABOVE", arguments, events[-1].time), gold, proof_ids, facts, "spatial_order"
|
||||
|
||||
if index % 4 == 1:
|
||||
negative = _membership_negative(world, complexity)
|
||||
if negative is None:
|
||||
return None
|
||||
entity, target, nodes, proof_ids = negative
|
||||
known_set, disjoint_set = nodes[-2], nodes[-1]
|
||||
facts = [f"MEMBER({entity},{nodes[1]})"]
|
||||
facts.extend(
|
||||
f"SUBSET({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(1, len(nodes) - 2)
|
||||
)
|
||||
facts.extend(
|
||||
(
|
||||
f"DISJOINT({known_set},{disjoint_set})",
|
||||
f"therefore NOT MEMBER({entity},{target})",
|
||||
)
|
||||
)
|
||||
# Keeping events fixed while changing load makes the veto guard observable.
|
||||
next_id = len(support)
|
||||
support.append(
|
||||
_observe_case(f"history-{next_id:03d}", system, dict(low), ("E0", "E1"), trials, rng)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 1:03d}", system, dict(high), ("E0", "E1"), trials, rng
|
||||
return (
|
||||
Query("MEMBER", (entity, target), events[-1].time),
|
||||
"NO",
|
||||
proof_ids,
|
||||
tuple(facts),
|
||||
"membership_disjointness",
|
||||
)
|
||||
|
||||
membership = _membership_path(world, complexity)
|
||||
if membership is None:
|
||||
return None
|
||||
entity, target, nodes, proof_ids = membership
|
||||
facts = [f"MEMBER({entity},{nodes[1]})"]
|
||||
facts.extend(
|
||||
f"SUBSET({nodes[position]},{nodes[position + 1]})"
|
||||
for position in range(1, len(nodes) - 1)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 2:03d}",
|
||||
system,
|
||||
dict(low),
|
||||
("E0", "E2", "E3"),
|
||||
trials,
|
||||
rng,
|
||||
)
|
||||
return (
|
||||
Query("MEMBER", (entity, target), events[-1].time),
|
||||
"YES",
|
||||
proof_ids,
|
||||
tuple(facts),
|
||||
"membership_inheritance",
|
||||
)
|
||||
|
||||
query = BehaviorCase(
|
||||
id="query",
|
||||
attributes=query_attributes,
|
||||
events=query_events,
|
||||
trials=1,
|
||||
observed_present=None,
|
||||
observed_absent=None,
|
||||
|
||||
def _final_state(world: _World) -> dict[str, Any]:
|
||||
return {
|
||||
"locations": dict(world.locations),
|
||||
"above": [[left, right] for left, right in sorted(world.above)],
|
||||
"subsets": [[left, right] for left, right in sorted(world.subsets)],
|
||||
"members": [[entity, container] for entity, container in sorted(world.members)],
|
||||
"disjoint": [sorted(pair) for pair in sorted(world.disjoint, key=lambda item: sorted(item))],
|
||||
"connected": [sorted(pair) for pair in sorted(world.connected, key=lambda item: sorted(item))],
|
||||
"numbers": dict(world.numbers),
|
||||
"flags": [
|
||||
{"entity": entity, "state": state, "value": value}
|
||||
for (entity, state), value in sorted(world.flags.items())
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _generate_one(
|
||||
index: int,
|
||||
complexity: int,
|
||||
entity_count: int,
|
||||
simulation_steps: int,
|
||||
rng: random.Random,
|
||||
) -> WorldSkeleton:
|
||||
entities, groups = _entities(entity_count)
|
||||
world, initial_state = _initial_world(groups, rng, complexity)
|
||||
events = _simulate(world, groups, simulation_steps, rng)
|
||||
if not events:
|
||||
raise RuntimeError("simulation produced no events")
|
||||
selected = _choose_query(index, complexity, world, events)
|
||||
if selected is None:
|
||||
raise RuntimeError("simulation did not produce a sufficiently deep proof")
|
||||
query, gold, proof_ids, proof_facts, query_family = selected
|
||||
direct = any(
|
||||
event.kind == "RELATE"
|
||||
and event.arguments.get("predicate") == query.predicate
|
||||
and (event.arguments.get("subject"), event.arguments.get("object")) == query.arguments
|
||||
for event in events
|
||||
)
|
||||
trace = tuple(
|
||||
[f"query events: {' -> '.join(query_events)}"]
|
||||
+ [
|
||||
f"{rule.id}: on {rule.on_event}, p={rule.probability}, {rule.action}({rule.target})"
|
||||
for rule in system.rules
|
||||
]
|
||||
+ [f"exact P(PRESENT)={probability:.8f}", f"gold={gold}"]
|
||||
)
|
||||
return BehaviorSkeleton(
|
||||
id=f"dynamic-{index:04d}",
|
||||
family="stochastic_behavioral_system",
|
||||
if direct:
|
||||
raise RuntimeError("query was directly stated in the event history")
|
||||
checks = {
|
||||
"direct_query_event_absent": True,
|
||||
"proof_length": len(proof_ids),
|
||||
"minimum_proof_length": complexity,
|
||||
"event_count": len(events),
|
||||
"proof_event_ratio": len(proof_ids) / len(events),
|
||||
"query_family": query_family,
|
||||
"weak_baselines": {
|
||||
"direct_fact": False,
|
||||
"proof_shorter_than_requested": len(proof_ids) < complexity,
|
||||
"proof_is_large_fraction_of_history": len(proof_ids) / len(events) >= 0.05,
|
||||
},
|
||||
}
|
||||
return WorldSkeleton(
|
||||
id=f"world-{index:04d}",
|
||||
family="dynamic_multi_logic_world",
|
||||
complexity=complexity,
|
||||
event_vocabulary=tuple(f"E{number}" for number in range(complexity)),
|
||||
attribute_vocabulary={"load": "integer", "class": "category"},
|
||||
observation_protocol=(
|
||||
"Each trial begins with an absent record and no internal flags.",
|
||||
"Events occur from left to right; the same hidden system is used in every trial.",
|
||||
"PRESENT and ABSENT counts are final observations over independent repeated trials.",
|
||||
"For the query, choose which final outcome is more likely; an exact probability is not required.",
|
||||
),
|
||||
hidden_system=system,
|
||||
support_cases=tuple(support),
|
||||
query_case=query,
|
||||
entity_count=entity_count,
|
||||
simulation_steps=simulation_steps,
|
||||
hidden_program=_program(),
|
||||
entities=tuple(entities),
|
||||
initial_state=initial_state,
|
||||
events=tuple(events),
|
||||
final_state=_final_state(world),
|
||||
query=query,
|
||||
answer_options=ANSWER_OPTIONS,
|
||||
gold_answer=gold,
|
||||
gold_present_probability=probability,
|
||||
latent_trace=trace,
|
||||
proof_event_ids=proof_ids,
|
||||
proof_facts=proof_facts,
|
||||
generator_checks=checks,
|
||||
)
|
||||
|
||||
|
||||
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[BehaviorSkeleton]:
|
||||
def generate_skeletons(
|
||||
count: int,
|
||||
length: str | int,
|
||||
seed: int = 0,
|
||||
entity_count: int = 128,
|
||||
simulation_steps: int = 1000,
|
||||
) -> list[WorldSkeleton]:
|
||||
if count < 1:
|
||||
raise ValueError("count must be positive")
|
||||
if simulation_steps < 100:
|
||||
raise ValueError("simulation_steps must be at least 100")
|
||||
complexity = resolve_length(length)
|
||||
rng = random.Random(seed)
|
||||
generated: list[BehaviorSkeleton] = []
|
||||
generated: list[WorldSkeleton] = []
|
||||
for index in range(count):
|
||||
for _attempt in range(100):
|
||||
for _attempt in range(60):
|
||||
try:
|
||||
generated.append(_generate_one(index, complexity, rng))
|
||||
generated.append(
|
||||
_generate_one(index, complexity, entity_count, simulation_steps, rng)
|
||||
)
|
||||
break
|
||||
except RuntimeError as exc:
|
||||
if "insufficient probability margin" not in str(exc):
|
||||
raise
|
||||
except RuntimeError:
|
||||
continue
|
||||
else:
|
||||
raise RuntimeError("could not generate a query with a stable probability margin")
|
||||
raise RuntimeError("could not generate a world with the requested proof complexity")
|
||||
return generated
|
||||
|
||||
Reference in New Issue
Block a user