Replace explicit rules with behavioral hidden systems
This commit is contained in:
+15
-8
@@ -5,7 +5,7 @@ from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from .models import Skeleton
|
||||
from .models import BehaviorSkeleton
|
||||
from .skeletons import generate_skeletons
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ class EvaluationResult:
|
||||
rendered_problem: str
|
||||
answer_options: tuple[str, ...]
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
solver_output: str
|
||||
model_answer: str
|
||||
correct: bool
|
||||
@@ -43,15 +44,18 @@ RENDER_SCHEMA = {
|
||||
}
|
||||
|
||||
|
||||
def render_skeleton(client: JsonModel, skeleton: Skeleton) -> str:
|
||||
def render_skeleton(client: JsonModel, skeleton: BehaviorSkeleton) -> str:
|
||||
prompt = json.dumps(skeleton.renderer_payload(), ensure_ascii=False, indent=2)
|
||||
response = client.request_json(
|
||||
system=(
|
||||
"You turn typed logical intermediate representations into self-contained natural-language "
|
||||
"reasoning problems. Invent a concrete or fictional surface scenario and rename entities, but "
|
||||
"preserve every fact, its order when order matters, the rules, the query, and the exact answer "
|
||||
"labels. Do not solve the problem, hint at the answer, mention a hidden skeleton, or add facts. "
|
||||
"End by requiring exactly one of the supplied answer labels."
|
||||
"You turn behavioral observations from an unknown stochastic system into one self-contained "
|
||||
"natural-language inductive reasoning problem. Invent a coherent concrete or fictional domain. "
|
||||
"Give each event code and attribute a stable surface name, and preserve every case's attributes, "
|
||||
"event order, trial count, observed outcome counts, query, and exact answer labels. The histories "
|
||||
"are empirical observations, not explicit system laws: do not state or invent the hidden rules, "
|
||||
"trigger probabilities, intermediate flags, exact query probability, or answer. Make clear that "
|
||||
"all trials use one unchanged system and that the question asks which final outcome is more likely, "
|
||||
"not for an exact probability. End by requiring exactly one supplied answer label."
|
||||
),
|
||||
user=f"Render this intermediate representation:\n{prompt}",
|
||||
schema_name="rendered_problem",
|
||||
@@ -76,7 +80,9 @@ def parse_marked_answer(output: str, answer_options: tuple[str, ...]) -> str:
|
||||
def solve_problem(client: JsonModel, problem: str, answer_options: tuple[str, ...]) -> tuple[str, str]:
|
||||
output = client.request_text(
|
||||
system=(
|
||||
"Solve the reasoning problem from the stated facts only. Give a concise explanation, then put "
|
||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
@@ -116,6 +122,7 @@ def run_evaluation(
|
||||
rendered_problem=problem,
|
||||
answer_options=skeleton.answer_options,
|
||||
gold_answer=skeleton.gold_answer,
|
||||
gold_present_probability=skeleton.gold_present_probability,
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
correct=answer == skeleton.gold_answer,
|
||||
|
||||
Reference in New Issue
Block a user