Generate proof-bearing dynamic world histories

tests/test_pipeline.py

@@ -1,3 +1,4 @@
+import json
 import tempfile
 import unittest
 from pathlib import Path
@@ -21,21 +22,23 @@ class FailingSolver:
         raise RuntimeError("solver failed")
 
 
+class AuditRenderer:
+    def request_json(self, **kwargs):
+        payload = json.loads(kwargs["user"][kwargs["user"].find("{") :])
+        identifiers = [item["id"] for item in payload["initial_observations"]]
+        identifiers.extend(item["id"] for item in payload["events"])
+        return {"problem": "A compact behavioral problem.\n" + "\n".join(identifiers)}
+
+
 class PipelineTests(unittest.TestCase):
     def test_end_to_end_writes_skeletons_before_scored_results(self):
-        renderer = QueueClient([
-            {"problem": "Infer from histories; choose PRESENT or ABSENT."},
-            {"problem": "Infer from histories; choose PRESENT or ABSENT."},
-            {"problem": "Infer from histories; choose PRESENT or ABSENT."},
-            {"problem": "Infer from histories; choose PRESENT or ABSENT."},
-            {"problem": "Infer from histories; choose PRESENT or ABSENT."},
-        ])
+        renderer = AuditRenderer()
         solver = QueueClient([
-            "The positive behavior dominates.\n#### PRESENT",
-            "The veto behavior dominates.\n#### ABSENT",
-            "The positive behavior dominates.\n#### PRESENT",
-            "The veto behavior dominates.\n#### ABSENT",
-            "The positive behavior dominates.\n#### PRESENT",
+            "The relation follows through the event chain.\n#### YES",
+            "The target set is disjoint from the inherited membership.\n#### NO",
+            "The reverse ordering is false.\n#### NO",
+            "Membership propagates through the nested sets.\n#### YES",
+            "The relation follows through the event chain.\n#### YES",
         ])
         with tempfile.TemporaryDirectory() as directory:
             output = Path(directory)
@@ -70,7 +73,7 @@ class PipelineTests(unittest.TestCase):
         self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True))
 
     def test_rendered_problem_is_checkpointed_before_solver_failure(self):
-        renderer = QueueClient([{"problem": "A compact behavioral problem."}])
+        renderer = AuditRenderer()
         with tempfile.TemporaryDirectory() as directory:
             output = Path(directory)
             with self.assertRaisesRegex(RuntimeError, "solver failed"):
@@ -85,7 +88,7 @@ class PipelineTests(unittest.TestCase):
             self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
 
     def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
-        renderer = QueueClient([{"problem": "A compact behavioral problem."}])
+        renderer = AuditRenderer()
         solver = QueueClient(["The response ended before its final label."])
         with tempfile.TemporaryDirectory() as directory:
             output = Path(directory)
@@ -101,6 +104,19 @@ class PipelineTests(unittest.TestCase):
             self.assertEqual(summary["correct"], 0)
             self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
 
+    def test_renderer_omission_is_rejected_before_solving(self):
+        renderer = QueueClient([{"problem": "This rendering omitted its audit IDs."}])
+        with tempfile.TemporaryDirectory() as directory:
+            with self.assertRaisesRegex(RuntimeError, "renderer omitted"):
+                run_evaluation(
+                    renderer=renderer,
+                    solver=FailingSolver(),
+                    count=1,
+                    length="short",
+                    seed=1,
+                    output_dir=Path(directory),
+                )
+
 
 if __name__ == "__main__":
     unittest.main()