Replace explicit rules with behavioral hidden systems
This commit is contained in:
+11
-3
@@ -1,6 +1,14 @@
|
||||
"""Logic skeleton benchmark."""
|
||||
"""Behavioral hidden-system benchmark."""
|
||||
|
||||
from .models import Fact, Query, Skeleton
|
||||
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
|
||||
from .skeletons import generate_skeletons, resolve_length
|
||||
|
||||
__all__ = ["Fact", "Query", "Skeleton", "generate_skeletons", "resolve_length"]
|
||||
__all__ = [
|
||||
"BehaviorCase",
|
||||
"BehaviorSkeleton",
|
||||
"Condition",
|
||||
"HiddenRule",
|
||||
"HiddenSystem",
|
||||
"generate_skeletons",
|
||||
"resolve_length",
|
||||
]
|
||||
|
||||
+3
-1
@@ -23,7 +23,9 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="logicbench")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
generate = subparsers.add_parser("generate", help="generate answer-bearing skeleton JSONL")
|
||||
generate = subparsers.add_parser(
|
||||
"generate", help="generate hidden systems and answer-bearing behavior skeletons"
|
||||
)
|
||||
generate.add_argument("--count", type=int, default=3)
|
||||
generate.add_argument("--length", default="short")
|
||||
generate.add_argument("--seed", type=int, default=0)
|
||||
|
||||
+59
-17
@@ -5,36 +5,78 @@ from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fact:
|
||||
predicate: str
|
||||
arguments: tuple[str, ...]
|
||||
class Condition:
|
||||
"""A predicate over immutable inputs or mutable simulator state."""
|
||||
|
||||
source: str
|
||||
key: str
|
||||
operator: str
|
||||
value: Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Query:
|
||||
predicate: str
|
||||
arguments: tuple[str, ...]
|
||||
class HiddenRule:
|
||||
"""One executable rule in the hidden system language."""
|
||||
|
||||
id: str
|
||||
on_event: str
|
||||
conditions: tuple[Condition, ...]
|
||||
probability: float
|
||||
action: str
|
||||
target: str
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Skeleton:
|
||||
class HiddenSystem:
|
||||
"""The latent program. This object must never be sent to the renderer."""
|
||||
|
||||
initial_record: str
|
||||
conflict_policy: str
|
||||
rules: tuple[HiddenRule, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BehaviorCase:
|
||||
"""Observable inputs and aggregate output from repeated independent trials."""
|
||||
|
||||
id: str
|
||||
attributes: dict[str, Any]
|
||||
events: tuple[str, ...]
|
||||
trials: int
|
||||
observed_present: int | None
|
||||
observed_absent: int | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BehaviorSkeleton:
|
||||
"""A hidden program plus its public behavioral evidence and answer-bearing query."""
|
||||
|
||||
id: str
|
||||
family: str
|
||||
complexity: int
|
||||
entities: dict[str, str]
|
||||
facts: tuple[Fact, ...]
|
||||
rules: tuple[str, ...]
|
||||
query: Query
|
||||
event_vocabulary: tuple[str, ...]
|
||||
attribute_vocabulary: dict[str, str]
|
||||
observation_protocol: tuple[str, ...]
|
||||
hidden_system: HiddenSystem
|
||||
support_cases: tuple[BehaviorCase, ...]
|
||||
query_case: BehaviorCase
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
proof: tuple[str, ...]
|
||||
gold_present_probability: float
|
||||
latent_trace: tuple[str, ...]
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return asdict(self)
|
||||
|
||||
def renderer_payload(self) -> dict[str, Any]:
|
||||
"""Return everything needed to render the problem, but never the answer."""
|
||||
payload = self.to_dict()
|
||||
payload.pop("gold_answer")
|
||||
payload.pop("proof")
|
||||
return payload
|
||||
"""Expose behavior only: no program, probability, trace, or answer."""
|
||||
return {
|
||||
"family": self.family,
|
||||
"complexity": self.complexity,
|
||||
"event_vocabulary": self.event_vocabulary,
|
||||
"attribute_vocabulary": self.attribute_vocabulary,
|
||||
"observation_protocol": self.observation_protocol,
|
||||
"support_cases": [asdict(case) for case in self.support_cases],
|
||||
"query_case": asdict(self.query_case),
|
||||
"answer_options": self.answer_options,
|
||||
}
|
||||
|
||||
+15
-8
@@ -5,7 +5,7 @@ from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .models import Skeleton
|
||||
from .models import BehaviorSkeleton
|
||||
from .skeletons import generate_skeletons
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ class EvaluationResult:
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
solver_output: str
|
||||
model_answer: str
|
||||
correct: bool
|
||||
@@ -43,15 +44,18 @@ RENDER_SCHEMA = {
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
|
||||
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"You turn typed logical intermediate representations into self-contained natural-language "
|
||||
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
|
||||
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
|
||||
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
|
||||
"End by requiring exactly one of the supplied answer labels."
|
||||
"You turn behavioral observations from an unknown stochastic system into one self-contained "
|
||||
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
|
||||
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
|
||||
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
|
||||
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
|
||||
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
|
||||
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
|
||||
"not for an exact probability. End by requiring exactly one supplied answer label."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
@@ -76,7 +80,9 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
|
||||
output = client.request_text(
|
||||
system=(
|
||||
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
|
||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
@@ -116,6 +122,7 @@ def run_evaluation(
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
gold_present_probability=skeleton.gold_present_probability,
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
correct=answer == skeleton.gold_answer,
|
||||
|
||||
+236
-73
@@ -1,16 +1,26 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from itertools import combinations
|
||||
from typing import Any
|
||||
|
||||
from .models import Fact, Query, Skeleton
|
||||
from .models import BehaviorCase, BehaviorSkeleton, Condition, HiddenRule, HiddenSystem
|
||||
|
||||
|
||||
LENGTH_PRESETS = {"short": 2, "medium": 4, "long": 6}
|
||||
LENGTH_PRESETS = {"short": 4, "medium": 6, "long": 8}
|
||||
ANSWER_OPTIONS = ("PRESENT", "ABSENT")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _State:
|
||||
present: bool = False
|
||||
invalidated: bool = False
|
||||
flags: frozenset[str] = frozenset()
|
||||
|
||||
|
||||
def resolve_length(value: str | int) -> int:
|
||||
"""Resolve a named preset or an explicit number of proof-bearing operations."""
|
||||
"""Resolve the number of distinct events in the held-out query."""
|
||||
if isinstance(value, int):
|
||||
result = value
|
||||
elif value in LENGTH_PRESETS:
|
||||
@@ -21,91 +31,244 @@ def resolve_length(value: str | int) -> int:
|
||||
except ValueError as exc:
|
||||
choices = ", ".join(LENGTH_PRESETS)
|
||||
raise ValueError(f"length must be an integer or one of: {choices}") from exc
|
||||
if not 1 <= result <= 20:
|
||||
raise ValueError("length must be between 1 and 20")
|
||||
if not 4 <= result <= 10:
|
||||
raise ValueError("length must be between 4 and 10")
|
||||
return result
|
||||
|
||||
|
||||
def _order_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
|
||||
nodes = [f"E{i}" for i in range(complexity + 1)]
|
||||
facts = [Fact("LEFT_OF", (nodes[i], nodes[i + 1])) for i in range(complexity)]
|
||||
rng.shuffle(facts)
|
||||
return Skeleton(
|
||||
id=f"order-{index:04d}",
|
||||
family="transitive_order",
|
||||
complexity=complexity,
|
||||
entities={node: "Entity" for node in nodes},
|
||||
facts=tuple(facts),
|
||||
rules=("LEFT_OF is transitive", "RIGHT_OF(x,y) iff LEFT_OF(y,x)"),
|
||||
query=Query("RELATION", (nodes[0], nodes[-1])),
|
||||
answer_options=("LEFT_OF", "RIGHT_OF", "UNKNOWN"),
|
||||
gold_answer="LEFT_OF",
|
||||
proof=tuple(f"LEFT_OF({nodes[i]},{nodes[i + 1]})" for i in range(complexity)),
|
||||
def _condition_holds(condition: Condition, attributes: dict[str, Any], state: _State) -> bool:
|
||||
if condition.source == "attribute":
|
||||
actual = attributes.get(condition.key)
|
||||
elif condition.source == "state":
|
||||
actual = condition.key in state.flags
|
||||
else:
|
||||
raise ValueError(f"unknown condition source: {condition.source}")
|
||||
|
||||
operations = {
|
||||
"EQ": lambda left, right: left == right,
|
||||
"LE": lambda left, right: left <= right,
|
||||
"GT": lambda left, right: left > right,
|
||||
"IS_SET": lambda left, right: left is bool(right),
|
||||
}
|
||||
try:
|
||||
return operations[condition.operator](actual, condition.value)
|
||||
except KeyError as exc:
|
||||
raise ValueError(f"unknown condition operator: {condition.operator}") from exc
|
||||
|
||||
|
||||
def _apply_action(state: _State, rule: HiddenRule) -> _State:
|
||||
if rule.action == "SET_FLAG":
|
||||
return _State(state.present, state.invalidated, state.flags | {rule.target})
|
||||
if rule.action == "ADD_RECORD":
|
||||
if state.invalidated:
|
||||
return state
|
||||
return _State(True, False, state.flags)
|
||||
if rule.action == "INVALIDATE_RECORD":
|
||||
return _State(False, True, state.flags)
|
||||
raise ValueError(f"unknown action: {rule.action}")
|
||||
|
||||
|
||||
def exact_present_probability(
|
||||
system: HiddenSystem, attributes: dict[str, Any], events: tuple[str, ...]
|
||||
) -> float:
|
||||
"""Enumerate all stochastic branches while respecting event and state order."""
|
||||
initial = _State(present=system.initial_record == "PRESENT")
|
||||
distribution: dict[_State, float] = {initial: 1.0}
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
next_distribution: dict[_State, float] = {}
|
||||
for state, mass in distribution.items():
|
||||
if not all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + mass
|
||||
continue
|
||||
inactive_mass = mass * (1.0 - rule.probability)
|
||||
active_state = _apply_action(state, rule)
|
||||
next_distribution[state] = next_distribution.get(state, 0.0) + inactive_mass
|
||||
next_distribution[active_state] = (
|
||||
next_distribution.get(active_state, 0.0) + mass * rule.probability
|
||||
)
|
||||
distribution = next_distribution
|
||||
return sum(mass for state, mass in distribution.items() if state.present)
|
||||
|
||||
|
||||
def _simulate_once(
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
rng: random.Random,
|
||||
) -> bool:
|
||||
state = _State(present=system.initial_record == "PRESENT")
|
||||
for event in events:
|
||||
for rule in (item for item in system.rules if item.on_event == event):
|
||||
if all(_condition_holds(item, attributes, state) for item in rule.conditions):
|
||||
if rng.random() < rule.probability:
|
||||
state = _apply_action(state, rule)
|
||||
return state.present
|
||||
|
||||
|
||||
def _observe_case(
|
||||
case_id: str,
|
||||
system: HiddenSystem,
|
||||
attributes: dict[str, Any],
|
||||
events: tuple[str, ...],
|
||||
trials: int,
|
||||
rng: random.Random,
|
||||
) -> BehaviorCase:
|
||||
present = sum(_simulate_once(system, attributes, events, rng) for _ in range(trials))
|
||||
return BehaviorCase(
|
||||
id=case_id,
|
||||
attributes=attributes,
|
||||
events=events,
|
||||
trials=trials,
|
||||
observed_present=present,
|
||||
observed_absent=trials - present,
|
||||
)
|
||||
|
||||
|
||||
def _set_chain(index: int, complexity: int, rng: random.Random) -> Skeleton:
|
||||
item = "E0"
|
||||
sets = [f"S{i}" for i in range(complexity)]
|
||||
facts = [Fact("MEMBER_OF", (item, sets[0]))]
|
||||
facts.extend(Fact("SUBSET_OF", (sets[i], sets[i + 1])) for i in range(complexity - 1))
|
||||
rng.shuffle(facts)
|
||||
return Skeleton(
|
||||
id=f"set-{index:04d}",
|
||||
family="membership_inheritance",
|
||||
complexity=complexity,
|
||||
entities={item: "Entity", **{name: "Set" for name in sets}},
|
||||
facts=tuple(facts),
|
||||
rules=("MEMBER_OF(x,A) and SUBSET_OF(A,B) imply MEMBER_OF(x,B)",),
|
||||
query=Query("MEMBER_OF", (item, sets[-1])),
|
||||
answer_options=("YES", "NO", "UNKNOWN"),
|
||||
gold_answer="YES",
|
||||
proof=tuple(
|
||||
[f"MEMBER_OF({item},{sets[0]})"]
|
||||
+ [f"SUBSET_OF({sets[i]},{sets[i + 1]})" for i in range(complexity - 1)]
|
||||
def _build_system(index: int, complexity: int, rng: random.Random) -> HiddenSystem:
|
||||
"""Build a finite stochastic program with a gated rule and veto conflicts."""
|
||||
present_favored = index % 2 == 0
|
||||
add_probability = rng.choice((0.85, 0.9, 0.95)) if present_favored else rng.choice((0.65, 0.75))
|
||||
veto_probability = rng.choice((0.15, 0.25)) if present_favored else rng.choice((0.7, 0.8, 0.9))
|
||||
marker_probability = rng.choice((0.6, 0.75, 0.9))
|
||||
gated_action = "ADD_RECORD" if present_favored else "INVALIDATE_RECORD"
|
||||
gated_probability = rng.choice((0.6, 0.75, 0.85))
|
||||
|
||||
rules = [
|
||||
HiddenRule("R0", "E0", (), add_probability, "ADD_RECORD", "record"),
|
||||
HiddenRule(
|
||||
"R1",
|
||||
"E1",
|
||||
(Condition("attribute", "load", "LE", 4),),
|
||||
veto_probability,
|
||||
"INVALIDATE_RECORD",
|
||||
"record",
|
||||
),
|
||||
HiddenRule("R2", "E2", (), marker_probability, "SET_FLAG", "marked"),
|
||||
HiddenRule(
|
||||
"R3",
|
||||
"E3",
|
||||
(Condition("state", "marked", "IS_SET", True),),
|
||||
gated_probability,
|
||||
gated_action,
|
||||
"record",
|
||||
),
|
||||
]
|
||||
for number in range(4, complexity):
|
||||
action = "ADD_RECORD" if (number + index) % 3 else "INVALIDATE_RECORD"
|
||||
probability = rng.choice((0.2, 0.35, 0.5))
|
||||
condition = Condition("attribute", "class", "EQ", "priority")
|
||||
rules.append(
|
||||
HiddenRule(f"R{number}", f"E{number}", (condition,), probability, action, "record")
|
||||
)
|
||||
return HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=tuple(rules),
|
||||
)
|
||||
|
||||
|
||||
def _state_updates(index: int, complexity: int, rng: random.Random) -> Skeleton:
|
||||
start = rng.randint(3, 9)
|
||||
operations: list[tuple[str, int]] = []
|
||||
value = start
|
||||
for _ in range(complexity):
|
||||
amount = rng.randint(1, 4)
|
||||
predicate = rng.choice(("INCREMENT", "DECREMENT"))
|
||||
if predicate == "DECREMENT" and value - amount < 0:
|
||||
predicate = "INCREMENT"
|
||||
value = value + amount if predicate == "INCREMENT" else value - amount
|
||||
operations.append((predicate, amount))
|
||||
def _support_event_sets(complexity: int) -> list[tuple[str, ...]]:
|
||||
events = [f"E{number}" for number in range(complexity)]
|
||||
cases: list[tuple[str, ...]] = [(event,) for event in events]
|
||||
cases.extend((("E2", "E3"), ("E3", "E2"), ("E0", "E1")))
|
||||
cases.extend(tuple(pair) for pair in combinations(events, 2) if tuple(pair) not in cases)
|
||||
return cases
|
||||
|
||||
facts = [Fact("INITIAL_VALUE", ("E0", str(start)))]
|
||||
facts.extend(Fact(predicate, ("E0", str(amount))) for predicate, amount in operations)
|
||||
lower_distractor = value - 1 if value > 0 else value + 2
|
||||
return Skeleton(
|
||||
id=f"state-{index:04d}",
|
||||
family="sequential_state_update",
|
||||
|
||||
def _generate_one(index: int, complexity: int, rng: random.Random) -> BehaviorSkeleton:
|
||||
system = _build_system(index, complexity, rng)
|
||||
low = {"load": 3, "class": "priority"}
|
||||
high = {"load": 8, "class": "standard"}
|
||||
query_attributes = dict(low)
|
||||
query_events = tuple(f"E{number}" for number in range(complexity))
|
||||
probability = exact_present_probability(system, query_attributes, query_events)
|
||||
if abs(probability - 0.5) < 0.08:
|
||||
raise RuntimeError("generated query has an insufficient probability margin")
|
||||
gold = "PRESENT" if probability > 0.5 else "ABSENT"
|
||||
|
||||
trials = 48 if complexity <= 4 else 64
|
||||
support: list[BehaviorCase] = []
|
||||
for case_number, events in enumerate(_support_event_sets(complexity)):
|
||||
attributes = low if case_number % 3 else high
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{case_number:03d}", system, dict(attributes), events, trials, rng
|
||||
)
|
||||
)
|
||||
# Keeping events fixed while changing load makes the veto guard observable.
|
||||
next_id = len(support)
|
||||
support.append(
|
||||
_observe_case(f"history-{next_id:03d}", system, dict(low), ("E0", "E1"), trials, rng)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 1:03d}", system, dict(high), ("E0", "E1"), trials, rng
|
||||
)
|
||||
)
|
||||
support.append(
|
||||
_observe_case(
|
||||
f"history-{next_id + 2:03d}",
|
||||
system,
|
||||
dict(low),
|
||||
("E0", "E2", "E3"),
|
||||
trials,
|
||||
rng,
|
||||
)
|
||||
)
|
||||
|
||||
query = BehaviorCase(
|
||||
id="query",
|
||||
attributes=query_attributes,
|
||||
events=query_events,
|
||||
trials=1,
|
||||
observed_present=None,
|
||||
observed_absent=None,
|
||||
)
|
||||
trace = tuple(
|
||||
[f"query events: {' -> '.join(query_events)}"]
|
||||
+ [
|
||||
f"{rule.id}: on {rule.on_event}, p={rule.probability}, {rule.action}({rule.target})"
|
||||
for rule in system.rules
|
||||
]
|
||||
+ [f"exact P(PRESENT)={probability:.8f}", f"gold={gold}"]
|
||||
)
|
||||
return BehaviorSkeleton(
|
||||
id=f"dynamic-{index:04d}",
|
||||
family="stochastic_behavioral_system",
|
||||
complexity=complexity,
|
||||
entities={"E0": "Counter"},
|
||||
facts=tuple(facts),
|
||||
rules=("Apply updates in the listed order",),
|
||||
query=Query("FINAL_VALUE", ("E0",)),
|
||||
answer_options=(str(value), str(value + 1), str(lower_distractor)),
|
||||
gold_answer=str(value),
|
||||
proof=tuple(f"{predicate}({amount})" for predicate, amount in operations),
|
||||
event_vocabulary=tuple(f"E{number}" for number in range(complexity)),
|
||||
attribute_vocabulary={"load": "integer", "class": "category"},
|
||||
observation_protocol=(
|
||||
"Each trial begins with an absent record and no internal flags.",
|
||||
"Events occur from left to right; the same hidden system is used in every trial.",
|
||||
"PRESENT and ABSENT counts are final observations over independent repeated trials.",
|
||||
"For the query, choose which final outcome is more likely; an exact probability is not required.",
|
||||
),
|
||||
hidden_system=system,
|
||||
support_cases=tuple(support),
|
||||
query_case=query,
|
||||
answer_options=ANSWER_OPTIONS,
|
||||
gold_answer=gold,
|
||||
gold_present_probability=probability,
|
||||
latent_trace=trace,
|
||||
)
|
||||
|
||||
|
||||
GENERATORS: tuple[Callable[[int, int, random.Random], Skeleton], ...] = (
|
||||
_order_chain,
|
||||
_set_chain,
|
||||
_state_updates,
|
||||
)
|
||||
|
||||
|
||||
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[Skeleton]:
|
||||
def generate_skeletons(count: int, length: str | int, seed: int = 0) -> list[BehaviorSkeleton]:
|
||||
if count < 1:
|
||||
raise ValueError("count must be positive")
|
||||
complexity = resolve_length(length)
|
||||
rng = random.Random(seed)
|
||||
return [GENERATORS[i % len(GENERATORS)](i, complexity, rng) for i in range(count)]
|
||||
generated: list[BehaviorSkeleton] = []
|
||||
for index in range(count):
|
||||
for _attempt in range(100):
|
||||
try:
|
||||
generated.append(_generate_one(index, complexity, rng))
|
||||
break
|
||||
except RuntimeError as exc:
|
||||
if "insufficient probability margin" not in str(exc):
|
||||
raise
|
||||
else:
|
||||
raise RuntimeError("could not generate a query with a stable probability margin")
|
||||
return generated
|
||||
|
||||
Reference in New Issue
Block a user