Replace explicit rules with behavioral hidden systems
This commit is contained in:
+10
-11
@@ -19,19 +19,18 @@ class QueueClient:
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
||||
renderer = QueueClient([
|
||||
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
||||
{"problem": "Choose YES, NO, or UNKNOWN."},
|
||||
{"problem": "Choose 9, 10, or 8."},
|
||||
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
||||
{"problem": "Choose YES, NO, or UNKNOWN."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
])
|
||||
# Seed 20260922 makes the third gold answer 9.
|
||||
solver = QueueClient([
|
||||
"The relation follows by transitivity.\n#### LEFT_OF",
|
||||
"Membership propagates through the subset.\n#### YES",
|
||||
"The two increments produce nine.\n#### 9",
|
||||
"The relation follows by transitivity.\n#### LEFT_OF",
|
||||
"Membership propagates through the subset.\n#### YES",
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
"The veto behavior dominates.\n#### ABSENT",
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
"The veto behavior dominates.\n#### ABSENT",
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
|
||||
+53
-14
@@ -1,36 +1,75 @@
|
||||
import unittest
|
||||
|
||||
from logicbench.skeletons import generate_skeletons, resolve_length
|
||||
from logicbench.models import Condition, HiddenRule, HiddenSystem
|
||||
from logicbench.skeletons import exact_present_probability, generate_skeletons, resolve_length
|
||||
|
||||
|
||||
class SkeletonTests(unittest.TestCase):
|
||||
def test_short_means_two_proof_bearing_operations(self):
|
||||
def test_short_means_four_query_events(self):
|
||||
skeletons = generate_skeletons(count=3, length="short", seed=42)
|
||||
self.assertEqual([item.complexity for item in skeletons], [2, 2, 2])
|
||||
self.assertEqual([item.family for item in skeletons], [
|
||||
"transitive_order",
|
||||
"membership_inheritance",
|
||||
"sequential_state_update",
|
||||
])
|
||||
self.assertTrue(all(len(item.proof) == 2 for item in skeletons))
|
||||
self.assertEqual([item.complexity for item in skeletons], [4, 4, 4])
|
||||
self.assertTrue(all(len(item.query_case.events) == 4 for item in skeletons))
|
||||
self.assertEqual(
|
||||
[item.gold_answer for item in skeletons], ["PRESENT", "ABSENT", "PRESENT"]
|
||||
)
|
||||
|
||||
def test_generation_is_deterministic(self):
|
||||
first = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
|
||||
second = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_renderer_never_receives_gold_or_proof(self):
|
||||
def test_renderer_receives_behavior_but_no_latent_information(self):
|
||||
skeleton = generate_skeletons(1, "short", seed=1)[0]
|
||||
payload = skeleton.renderer_payload()
|
||||
self.assertNotIn("gold_answer", payload)
|
||||
self.assertNotIn("proof", payload)
|
||||
self.assertIn("answer_options", payload)
|
||||
for secret in (
|
||||
"id",
|
||||
"hidden_system",
|
||||
"gold_answer",
|
||||
"gold_present_probability",
|
||||
"latent_trace",
|
||||
):
|
||||
self.assertNotIn(secret, payload)
|
||||
self.assertIn("support_cases", payload)
|
||||
self.assertIsNone(payload["query_case"]["observed_present"])
|
||||
|
||||
def test_exact_enumerator_applies_persistent_veto(self):
|
||||
system = HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=(
|
||||
HiddenRule("add", "A", (), 0.8, "ADD_RECORD", "record"),
|
||||
HiddenRule("veto", "B", (), 0.25, "INVALIDATE_RECORD", "record"),
|
||||
),
|
||||
)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("A", "B")), 0.6)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("B", "A")), 0.6)
|
||||
|
||||
def test_exact_enumerator_respects_stateful_event_order(self):
|
||||
system = HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=(
|
||||
HiddenRule("mark", "A", (), 0.5, "SET_FLAG", "ready"),
|
||||
HiddenRule(
|
||||
"add",
|
||||
"B",
|
||||
(Condition("state", "ready", "IS_SET", True),),
|
||||
0.8,
|
||||
"ADD_RECORD",
|
||||
"record",
|
||||
),
|
||||
),
|
||||
)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("A", "B")), 0.4)
|
||||
self.assertEqual(exact_present_probability(system, {}, ("B", "A")), 0.0)
|
||||
|
||||
def test_length_validation(self):
|
||||
self.assertEqual(resolve_length("medium"), 4)
|
||||
self.assertEqual(resolve_length("medium"), 6)
|
||||
self.assertEqual(resolve_length("5"), 5)
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_length("huge")
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_length(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user