Replace explicit rules with behavioral hidden systems

This commit is contained in:
2026-09-23 00:10:37 +08:00
parent 3cf3db7b12
commit 9f54f8f5a3
12 changed files with 598 additions and 380 deletions
+15 -8
View File
@@ -5,7 +5,7 @@ from dataclasses import asdict, dataclass
from pathlib import Path
from typing import Any, Protocol
from .models import Skeleton
from .models import BehaviorSkeleton
from .skeletons import generate_skeletons
@@ -30,6 +30,7 @@ class EvaluationResult:
rendered_problem: str
answer_options: tuple[str, ...]
gold_answer: str
gold_present_probability: float
solver_output: str
model_answer: str
correct: bool
@@ -43,15 +44,18 @@ RENDER_SCHEMA = {
}
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
response = client.request_json(
system=(
"You turn typed logical intermediate representations into self-contained natural-language "
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
"End by requiring exactly one of the supplied answer labels."
"You turn behavioral observations from an unknown stochastic system into one self-contained "
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
"not for an exact probability. End by requiring exactly one supplied answer label."
),
user=f"Render this intermediate representation:\n{prompt}",
schema_name="rendered_problem",
@@ -76,7 +80,9 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
output = client.request_text(
system=(
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
@@ -116,6 +122,7 @@ def run_evaluation(
rendered_problem=problem,
answer_options=skeleton.answer_options,
gold_answer=skeleton.gold_answer,
gold_present_probability=skeleton.gold_present_probability,
solver_output=solver_output,
model_answer=answer,
correct=answer == skeleton.gold_answer,