Generate proof-bearing dynamic world histories
This commit is contained in:
+30
-14
@@ -1,3 +1,4 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
@@ -21,21 +22,23 @@ class FailingSolver:
|
||||
raise RuntimeError("solver failed")
|
||||
|
||||
|
||||
class AuditRenderer:
|
||||
def request_json(self, **kwargs):
|
||||
payload = json.loads(kwargs["user"][kwargs["user"].find("{") :])
|
||||
identifiers = [item["id"] for item in payload["initial_observations"]]
|
||||
identifiers.extend(item["id"] for item in payload["events"])
|
||||
return {"problem": "A compact behavioral problem.\n" + "\n".join(identifiers)}
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
||||
renderer = QueueClient([
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
{"problem": "Infer from histories; choose PRESENT or ABSENT."},
|
||||
])
|
||||
renderer = AuditRenderer()
|
||||
solver = QueueClient([
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
"The veto behavior dominates.\n#### ABSENT",
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
"The veto behavior dominates.\n#### ABSENT",
|
||||
"The positive behavior dominates.\n#### PRESENT",
|
||||
"The relation follows through the event chain.\n#### YES",
|
||||
"The target set is disjoint from the inherited membership.\n#### NO",
|
||||
"The reverse ordering is false.\n#### NO",
|
||||
"Membership propagates through the nested sets.\n#### YES",
|
||||
"The relation follows through the event chain.\n#### YES",
|
||||
])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
@@ -70,7 +73,7 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True))
|
||||
|
||||
def test_rendered_problem_is_checkpointed_before_solver_failure(self):
|
||||
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
|
||||
renderer = AuditRenderer()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
with self.assertRaisesRegex(RuntimeError, "solver failed"):
|
||||
@@ -85,7 +88,7 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
|
||||
|
||||
def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
|
||||
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
|
||||
renderer = AuditRenderer()
|
||||
solver = QueueClient(["The response ended before its final label."])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
@@ -101,6 +104,19 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertEqual(summary["correct"], 0)
|
||||
self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
|
||||
|
||||
def test_renderer_omission_is_rejected_before_solving(self):
|
||||
renderer = QueueClient([{"problem": "This rendering omitted its audit IDs."}])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
with self.assertRaisesRegex(RuntimeError, "renderer omitted"):
|
||||
run_evaluation(
|
||||
renderer=renderer,
|
||||
solver=FailingSolver(),
|
||||
count=1,
|
||||
length="short",
|
||||
seed=1,
|
||||
output_dir=Path(directory),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+47
-48
@@ -1,75 +1,74 @@
|
||||
import unittest
|
||||
|
||||
from logicbench.models import Condition, HiddenRule, HiddenSystem
|
||||
from logicbench.skeletons import exact_present_probability, generate_skeletons, resolve_length
|
||||
from logicbench.skeletons import generate_skeletons, resolve_length
|
||||
|
||||
|
||||
class SkeletonTests(unittest.TestCase):
|
||||
def test_short_means_four_query_events(self):
|
||||
skeletons = generate_skeletons(count=3, length="short", seed=42)
|
||||
self.assertEqual([item.complexity for item in skeletons], [4, 4, 4])
|
||||
self.assertTrue(all(len(item.query_case.events) == 4 for item in skeletons))
|
||||
self.assertEqual(
|
||||
[item.gold_answer for item in skeletons], ["PRESENT", "ABSENT", "PRESENT"]
|
||||
)
|
||||
def test_short_world_has_many_entities_and_realized_events(self):
|
||||
skeleton = generate_skeletons(1, "short", seed=42)[0]
|
||||
self.assertEqual(skeleton.entity_count, 128)
|
||||
self.assertEqual(skeleton.simulation_steps, 1000)
|
||||
self.assertGreater(len(skeleton.events), 300)
|
||||
self.assertGreaterEqual(len(skeleton.proof_event_ids), 4)
|
||||
self.assertLess(skeleton.generator_checks["proof_event_ratio"], 0.05)
|
||||
|
||||
def test_generation_is_deterministic(self):
|
||||
first = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
|
||||
second = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
|
||||
first = [item.to_dict() for item in generate_skeletons(4, 4, seed=7)]
|
||||
second = [item.to_dict() for item in generate_skeletons(4, 4, seed=7)]
|
||||
self.assertEqual(first, second)
|
||||
|
||||
def test_renderer_receives_behavior_but_no_latent_information(self):
|
||||
def test_renderer_receives_realized_history_but_no_program_or_answer(self):
|
||||
skeleton = generate_skeletons(1, "short", seed=1)[0]
|
||||
payload = skeleton.renderer_payload()
|
||||
for secret in (
|
||||
"id",
|
||||
"hidden_system",
|
||||
"hidden_program",
|
||||
"hidden_state",
|
||||
"final_state",
|
||||
"gold_answer",
|
||||
"gold_present_probability",
|
||||
"latent_trace",
|
||||
"proof_event_ids",
|
||||
"proof_facts",
|
||||
"generator_checks",
|
||||
):
|
||||
self.assertNotIn(secret, payload)
|
||||
self.assertIn("support_cases", payload)
|
||||
self.assertIsNone(payload["query_case"]["observed_present"])
|
||||
self.assertEqual(len(payload["events"]), len(skeleton.events))
|
||||
self.assertEqual(len(payload["entity_types"]), 128)
|
||||
|
||||
def test_exact_enumerator_applies_persistent_veto(self):
|
||||
system = HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=(
|
||||
HiddenRule("add", "A", (), 0.8, "ADD_RECORD", "record"),
|
||||
HiddenRule("veto", "B", (), 0.25, "INVALIDATE_RECORD", "record"),
|
||||
),
|
||||
)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("A", "B")), 0.6)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("B", "A")), 0.6)
|
||||
def test_query_is_not_directly_stated_and_proof_events_exist(self):
|
||||
for skeleton in generate_skeletons(5, "short", seed=20260922):
|
||||
query_pair = skeleton.query.arguments
|
||||
direct = [
|
||||
event
|
||||
for event in skeleton.events
|
||||
if event.kind == "RELATE"
|
||||
and event.arguments.get("predicate") == skeleton.query.predicate
|
||||
and (event.arguments.get("subject"), event.arguments.get("object")) == query_pair
|
||||
]
|
||||
self.assertFalse(direct)
|
||||
event_ids = {event.id for event in skeleton.events}
|
||||
self.assertTrue(set(skeleton.proof_event_ids) <= event_ids)
|
||||
self.assertGreaterEqual(len(skeleton.proof_event_ids), skeleton.complexity)
|
||||
|
||||
def test_exact_enumerator_respects_stateful_event_order(self):
|
||||
system = HiddenSystem(
|
||||
initial_record="ABSENT",
|
||||
conflict_policy="PERSISTENT_INVALIDATION_VETO",
|
||||
rules=(
|
||||
HiddenRule("mark", "A", (), 0.5, "SET_FLAG", "ready"),
|
||||
HiddenRule(
|
||||
"add",
|
||||
"B",
|
||||
(Condition("state", "ready", "IS_SET", True),),
|
||||
0.8,
|
||||
"ADD_RECORD",
|
||||
"record",
|
||||
),
|
||||
),
|
||||
)
|
||||
self.assertAlmostEqual(exact_present_probability(system, {}, ("A", "B")), 0.4)
|
||||
self.assertEqual(exact_present_probability(system, {}, ("B", "A")), 0.0)
|
||||
def test_world_contains_multiple_logic_domains(self):
|
||||
skeleton = generate_skeletons(1, "short", seed=3)[0]
|
||||
predicates = {
|
||||
event.arguments.get("predicate")
|
||||
for event in skeleton.events
|
||||
if event.kind == "RELATE"
|
||||
}
|
||||
self.assertTrue({"ABOVE", "MEMBER", "SUBSET", "CONNECTED"} <= predicates)
|
||||
kinds = {event.kind for event in skeleton.events}
|
||||
self.assertTrue({"MOVE", "UPDATE_NUMBER", "SET_STATE"} <= kinds)
|
||||
|
||||
def test_length_validation(self):
|
||||
def test_length_and_scale_validation(self):
|
||||
self.assertEqual(resolve_length("medium"), 6)
|
||||
self.assertEqual(resolve_length("5"), 5)
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_length("huge")
|
||||
with self.assertRaises(ValueError):
|
||||
resolve_length(3)
|
||||
resolve_length(2)
|
||||
with self.assertRaises(ValueError):
|
||||
generate_skeletons(1, "short", entity_count=20)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user