Replace explicit rules with behavioral hidden systems

This commit is contained in:
2026-09-23 00:10:37 +08:00
parent 3cf3db7b12
commit 9f54f8f5a3
12 changed files with 598 additions and 380 deletions
+11 -3
View File
@@ -1,6 +1,14 @@
"""Logic skeleton benchmark."""
"""Behavioral hidden-system benchmark."""
from .models import Fact, Query, Skeleton
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
from .skeletons import generate_skeletons, resolve_length
__all__ = ["Fact", "Query", "Skeleton", "generate_skeletons", "resolve_length"]
__all__ = [
"BehaviorCase",
"BehaviorSkeleton",
"Condition",
"HiddenRule",
"HiddenSystem",
"generate_skeletons",
"resolve_length",
]
+3 -1
View File
@@ -23,7 +23,9 @@ def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="logicbench")
subparsers = parser.add_subparsers(dest="command", required=True)
generate = subparsers.add_parser("generate", help="generate answer-bearing skeleton JSONL")
generate = subparsers.add_parser(
"generate", help="generate hidden systems and answer-bearing behavior skeletons"
)
generate.add_argument("--count", type=int, default=3)
generate.add_argument("--length", default="short")
generate.add_argument("--seed", type=int, default=0)
+59 -17
View File
@@ -5,36 +5,78 @@ from typing import Any
@dataclass(frozen=True)
class Fact:
predicate: str
arguments: tuple[str, ...]
class Condition:
"""A predicate over immutable inputs or mutable simulator state."""
source: str
key: str
operator: str
value: Any
@dataclass(frozen=True)
class Query:
predicate: str
arguments: tuple[str, ...]
class HiddenRule:
"""One executable rule in the hidden system language."""
id: str
on_event: str
conditions: tuple[Condition, ...]
probability: float
action: str
target: str
@dataclass(frozen=True)
class Skeleton:
class HiddenSystem:
"""The latent program. This object must never be sent to the renderer."""
initial_record: str
conflict_policy: str
rules: tuple[HiddenRule, ...]
@dataclass(frozen=True)
class BehaviorCase:
"""Observable inputs and aggregate output from repeated independent trials."""
id: str
attributes: dict[str, Any]
events: tuple[str, ...]
trials: int
observed_present: int | None
observed_absent: int | None
@dataclass(frozen=True)
class BehaviorSkeleton:
"""A hidden program plus its public behavioral evidence and answer-bearing query."""
id: str
family: str
complexity: int
entities: dict[str, str]
facts: tuple[Fact, ...]
rules: tuple[str, ...]
query: Query
event_vocabulary: tuple[str, ...]
attribute_vocabulary: dict[str, str]
observation_protocol: tuple[str, ...]
hidden_system: HiddenSystem
support_cases: tuple[BehaviorCase, ...]
query_case: BehaviorCase
answer_options: tuple[str, ...]
gold_answer: str
proof: tuple[str, ...]
gold_present_probability: float
latent_trace: tuple[str, ...]
def to_dict(self) -> dict[str, Any]:
return asdict(self)
def renderer_payload(self) -> dict[str, Any]:
"""Return everything needed to render the problem, but never the answer."""
payload = self.to_dict()
payload.pop("gold_answer")
payload.pop("proof")
return payload
"""Expose behavior only: no program, probability, trace, or answer."""
return {
"family": self.family,
"complexity": self.complexity,
"event_vocabulary": self.event_vocabulary,
"attribute_vocabulary": self.attribute_vocabulary,
"observation_protocol": self.observation_protocol,
"support_cases": [asdict(case) for case in self.support_cases],
"query_case": asdict(self.query_case),
"answer_options": self.answer_options,
}
+15 -8
View File
@@ -5,7 +5,7 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Protocol
from .models import Skeleton
from .models import BehaviorSkeleton
from .skeletons import generate_skeletons
@@ -30,6 +30,7 @@ class EvaluationResult:
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
gold_present_probability: float
solver_output: str
model_answer: str
correct: bool
@@ -43,15 +44,18 @@ RENDER_SCHEMA = {
}
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
response = client.request_json(
system=(
"You turn typed logical intermediate representations into self-contained natural-language "
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
"End by requiring exactly one of the supplied answer labels."
"You turn behavioral observations from an unknown stochastic system into one self-contained "
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
"not for an exact probability. End by requiring exactly one supplied answer label."
),
user=f"Render this intermediate representation:\n{prompt}",
schema_name="rendered_problem",
@@ -76,7 +80,9 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
output = client.request_text(
system=(
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
@@ -116,6 +122,7 @@ def run_evaluation(
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
gold_present_probability=skeleton.gold_present_probability,
solver_output=solver_output,
model_answer=answer,
correct=answer == skeleton.gold_answer,
+236 -73
View File
@@ -1,16 +1,26 @@
from __future__ import annotations
import random
from collections.abc import Callable
from dataclasses import dataclass
from itertools import combinations
from typing import Any
from .models import Fact, Query, Skeleton
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
LENGTH_PRESETS = {"short": 2, "medium": 4, "long": 6}
LENGTH_PRESETS = {"short": 4, "medium": 6, "long": 8}
ANSWER_OPTIONS = ("PRESENT", "ABSENT")
@dataclass(frozen=True)
class _State:
present: bool = False
invalidated: bool = False
flags: frozenset[str] = frozenset()
def resolve_length(value: str | int) -> int:
"""Resolve a named preset or an explicit number of proof-bearing operations."""
"""Resolve the number of distinct events in the held-out query."""
if isinstance(value, int):
result = value
elif value in LENGTH_PRESETS:
@@ -21,91 +31,244 @@ def resolve_length(value: str | int) -> int:
except ValueError as exc:
choices = ", ".join(LENGTH_PRESETS)
raise ValueError(f"length must be an integer or one of: {choices}") from exc
if not 1 <= result <= 20:
raise ValueError("length must be between 1 and 20")
if not 4 <= result <= 10:
raise ValueError("length must be between 4 and 10")
return result
def _order_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
nodes = [f"E{i}" for i in range(complexity + 1)]
facts = [Fact("LEFT_OF", (nodes[i], nodes[i + 1])) for i in range(complexity)]
rng.shuffle(facts)
return Skeleton(
id=f"order-{index:04d}",
family="transitive_order",
complexity=complexity,
entities={node: "Entity" for node in nodes},
facts=tuple(facts),
rules=("LEFT_OF is transitive", "RIGHT_OF(x,y) iff LEFT_OF(y,x)"),
query=Query("RELATION", (nodes[0], nodes[-1])),
answer_options=("LEFT_OF", "RIGHT_OF", "UNKNOWN"),
gold_answer="LEFT_OF",
proof=tuple(f"LEFT_OF({nodes[i]},{nodes[i + 1]})" for i in range(complexity)),
def _condition_holds(condition: Condition, attributes: dict[str, Any], state: _State) -> bool:
if condition.source == "attribute":
actual = attributes.get(condition.key)
elif condition.source == "state":
actual = condition.key in state.flags
else:
raise ValueError(f"unknown condition source: {condition.source}")
operations = {
"EQ": lambda left, right: left == right,
"LE": lambda left, right: left <= right,
"GT": lambda left, right: left > right,
"IS_SET": lambda left, right: left is bool(right),
}
try:
return operations[condition.operator](actual, condition.value)
except KeyError as exc:
raise ValueError(f"unknown condition operator: {condition.operator}") from exc
def _apply_action(state: _State, rule: HiddenRule) -> _State:
if rule.action == "SET_FLAG":
return _State(state.present, state.invalidated, state.flags | {rule.target})
if rule.action == "ADD_RECORD":
if state.invalidated:
return state
return _State(True, False, state.flags)
if rule.action == "INVALIDATE_RECORD":
return _State(False, True, state.flags)
raise ValueError(f"unknown action: {rule.action}")
def exact_present_probability(
system: HiddenSystem, attributes: dict[str, Any], events: tuple[str, ...]
) -> float:
"""Enumerate all stochastic branches while respecting event and state order."""
initial = _State(present=system.initial_record == "PRESENT")
distribution: dict[_State, float] = {initial: 1.0}
for event in events:
for rule in (item for item in system.rules if item.on_event == event):
next_distribution: dict[_State, float] = {}
for state, mass in distribution.items():
if not all(_condition_holds(item, attributes, state) for item in rule.conditions):
next_distribution[state] = next_distribution.get(state, 0.0) + mass
continue
inactive_mass = mass * (1.0 - rule.probability)
active_state = _apply_action(state, rule)
next_distribution[state] = next_distribution.get(state, 0.0) + inactive_mass
next_distribution[active_state] = (
next_distribution.get(active_state, 0.0) + mass * rule.probability
)
distribution = next_distribution
return sum(mass for state, mass in distribution.items() if state.present)
def _simulate_once(
system: HiddenSystem,
attributes: dict[str, Any],
events: tuple[str, ...],
rng: random.Random,
) -> bool:
state = _State(present=system.initial_record == "PRESENT")
for event in events:
for rule in (item for item in system.rules if item.on_event == event):
if all(_condition_holds(item, attributes, state) for item in rule.conditions):
if rng.random() < rule.probability:
state = _apply_action(state, rule)
return state.present
def _observe_case(
case_id: str,
system: HiddenSystem,
attributes: dict[str, Any],
events: tuple[str, ...],
trials: int,
rng: random.Random,
) -> BehaviorCase:
present = sum(_simulate_once(system, attributes, events, rng) for _ in range(trials))
return BehaviorCase(
id=case_id,
attributes=attributes,
events=events,
trials=trials,
observed_present=present,
observed_absent=trials - present,
)
def _set_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
item = "E0"
sets = [f"S{i}" for i in range(complexity)]
facts = [Fact("MEMBER_OF", (item, sets[0]))]
facts.extend(Fact("SUBSET_OF", (sets[i], sets[i + 1])) for i in range(complexity - 1))
rng.shuffle(facts)
return Skeleton(
id=f"set-{index:04d}",
family="membership_inheritance",
complexity=complexity,
entities={item: "Entity", **{name: "Set" for name in sets}},
facts=tuple(facts),
rules=("MEMBER_OF(x,A) and SUBSET_OF(A,B) imply MEMBER_OF(x,B)",),
query=Query("MEMBER_OF", (item, sets[-1])),
answer_options=("YES", "NO", "UNKNOWN"),
gold_answer="YES",
proof=tuple(
[f"MEMBER_OF({item},{sets[0]})"]
+ [f"SUBSET_OF({sets[i]},{sets[i + 1]})" for i in range(complexity - 1)]
def _build_system(index: int, complexity: int, rng: random.Random) -> HiddenSystem:
"""Build a finite stochastic program with a gated rule and veto conflicts."""
present_favored = index % 2 == 0
add_probability = rng.choice((0.85, 0.9, 0.95)) if present_favored else rng.choice((0.65, 0.75))
veto_probability = rng.choice((0.15, 0.25)) if present_favored else rng.choice((0.7, 0.8, 0.9))
marker_probability = rng.choice((0.6, 0.75, 0.9))
gated_action = "ADD_RECORD" if present_favored else "INVALIDATE_RECORD"
gated_probability = rng.choice((0.6, 0.75, 0.85))
rules = [
HiddenRule("R0", "E0", (), add_probability, "ADD_RECORD", "record"),
HiddenRule(
"R1",
"E1",
(Condition("attribute", "load", "LE", 4),),
veto_probability,
"INVALIDATE_RECORD",
"record",
),
HiddenRule("R2", "E2", (), marker_probability, "SET_FLAG", "marked"),
HiddenRule(
"R3",
"E3",
(Condition("state", "marked", "IS_SET", True),),
gated_probability,
gated_action,
"record",
),
]
for number in range(4, complexity):
action = "ADD_RECORD" if (number + index) % 3 else "INVALIDATE_RECORD"
probability = rng.choice((0.2, 0.35, 0.5))
condition = Condition("attribute", "class", "EQ", "priority")
rules.append(
HiddenRule(f"R{number}", f"E{number}", (condition,), probability, action, "record")
)
return HiddenSystem(
initial_record="ABSENT",
conflict_policy="PERSISTENT_INVALIDATION_VETO",
rules=tuple(rules),
)
def _state_updates(index: int, complexity: int, rng: random.Random) -> Skeleton:
start = rng.randint(3, 9)
operations: list[tuple[str, int]] = []
value = start
for _ in range(complexity):
amount = rng.randint(1, 4)
predicate = rng.choice(("INCREMENT", "DECREMENT"))
if predicate == "DECREMENT" and value - amount < 0:
predicate = "INCREMENT"
value = value + amount if predicate == "INCREMENT" else value - amount
operations.append((predicate, amount))
def _support_event_sets(complexity: int) -> list[tuple[str, ...]]:
events = [f"E{number}" for number in range(complexity)]
cases: list[tuple[str, ...]] = [(event,) for event in events]
cases.extend((("E2", "E3"), ("E3", "E2"), ("E0", "E1")))
cases.extend(tuple(pair) for pair in combinations(events, 2) if tuple(pair) not in cases)
return cases
facts = [Fact("INITIAL_VALUE", ("E0", str(start)))]
facts.extend(Fact(predicate, ("E0", str(amount))) for predicate, amount in operations)
lower_distractor = value - 1 if value > 0 else value + 2
return Skeleton(
id=f"state-{index:04d}",
family="sequential_state_update",
def _generate_one(index: int, complexity: int, rng: random.Random) -> BehaviorSkeleton:
system = _build_system(index, complexity, rng)
low = {"load": 3, "class": "priority"}
high = {"load": 8, "class": "standard"}
query_attributes = dict(low)
query_events = tuple(f"E{number}" for number in range(complexity))
probability = exact_present_probability(system, query_attributes, query_events)
if abs(probability - 0.5) < 0.08:
raise RuntimeError("generated query has an insufficient probability margin")
gold = "PRESENT" if probability > 0.5 else "ABSENT"
trials = 48 if complexity <= 4 else 64
support: list[BehaviorCase] = []
for case_number, events in enumerate(_support_event_sets(complexity)):
attributes = low if case_number % 3 else high
support.append(
_observe_case(
f"history-{case_number:03d}", system, dict(attributes), events, trials, rng
)
)
# Keeping events fixed while changing load makes the veto guard observable.
next_id = len(support)
support.append(
_observe_case(f"history-{next_id:03d}", system, dict(low), ("E0", "E1"), trials, rng)
)
support.append(
_observe_case(
f"history-{next_id + 1:03d}", system, dict(high), ("E0", "E1"), trials, rng
)
)
support.append(
_observe_case(
f"history-{next_id + 2:03d}",
system,
dict(low),
("E0", "E2", "E3"),
trials,
rng,
)
)
query = BehaviorCase(
id="query",
attributes=query_attributes,
events=query_events,
trials=1,
observed_present=None,
observed_absent=None,
)
trace = tuple(
[f"query events: {' -> '.join(query_events)}"]
+ [
f"{rule.id}: on {rule.on_event}, p={rule.probability}, {rule.action}({rule.target})"
for rule in system.rules
]
+ [f"exact P(PRESENT)={probability:.8f}", f"gold={gold}"]
)
return BehaviorSkeleton(
id=f"dynamic-{index:04d}",
family="stochastic_behavioral_system",
complexity=complexity,
entities={"E0": "Counter"},
facts=tuple(facts),
rules=("Apply updates in the listed order",),
query=Query("FINAL_VALUE", ("E0",)),
answer_options=(str(value), str(value + 1), str(lower_distractor)),
gold_answer=str(value),
proof=tuple(f"{predicate}({amount})" for predicate, amount in operations),
event_vocabulary=tuple(f"E{number}" for number in range(complexity)),
attribute_vocabulary={"load": "integer", "class": "category"},
observation_protocol=(
"Each trial begins with an absent record and no internal flags.",
"Events occur from left to right; the same hidden system is used in every trial.",
"PRESENT and ABSENT counts are final observations over independent repeated trials.",
"For the query, choose which final outcome is more likely; an exact probability is not required.",
),
hidden_system=system,
support_cases=tuple(support),
query_case=query,
answer_options=ANSWER_OPTIONS,
gold_answer=gold,
gold_present_probability=probability,
latent_trace=trace,
)
GENERATORS: tuple[Callable[[int, int, random.Random], Skeleton], ...] = (
_order_chain,
_set_chain,
_state_updates,
)
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[Skeleton]:
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[BehaviorSkeleton]:
if count < 1:
raise ValueError("count must be positive")
complexity = resolve_length(length)
rng = random.Random(seed)
return [GENERATORS[i % len(GENERATORS)](i, complexity, rng) for i in range(count)]
generated: list[BehaviorSkeleton] = []
for index in range(count):
for _attempt in range(100):
try:
generated.append(_generate_one(index, complexity, rng))
break
except RuntimeError as exc:
if "insufficient probability margin" not in str(exc):
raise
else:
raise RuntimeError("could not generate a query with a stable probability margin")
return generated