Build initial logic skeleton benchmark

tests/test_pipeline.py

@@ -0,0 +1,46 @@
+import tempfile
+import unittest
+from pathlib import Path
+
+from logicbench.pipeline import run_evaluation
+
+
+class QueueClient:
+    def __init__(self, responses):
+        self.responses = iter(responses)
+
+    def request_json(self, **_kwargs):
+        return next(self.responses)
+
+
+class PipelineTests(unittest.TestCase):
+    def test_end_to_end_writes_skeletons_before_scored_results(self):
+        renderer = QueueClient([
+            {"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
+            {"problem": "Choose YES, NO, or UNKNOWN."},
+            {"problem": "Choose 9, 10, or 8."},
+        ])
+        # Seed 20260922 makes the third gold answer 9.
+        solver = QueueClient([
+            {"answer": "LEFT_OF"},
+            {"answer": "YES"},
+            {"answer": "9"},
+        ])
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory)
+            summary = run_evaluation(
+                renderer=renderer,
+                solver=solver,
+                count=3,
+                length="short",
+                seed=20260922,
+                output_dir=output,
+            )
+            self.assertTrue((output / "skeletons.jsonl").is_file())
+            self.assertTrue((output / "results.jsonl").is_file())
+            self.assertEqual(summary["count"], 3)
+            self.assertEqual(summary["correct"], 3)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_skeletons.py

@@ -0,0 +1,37 @@
+import unittest
+
+from logicbench.skeletons import generate_skeletons, resolve_length
+
+
+class SkeletonTests(unittest.TestCase):
+    def test_short_means_two_proof_bearing_operations(self):
+        skeletons = generate_skeletons(count=3, length="short", seed=42)
+        self.assertEqual([item.complexity for item in skeletons], [2, 2, 2])
+        self.assertEqual([item.family for item in skeletons], [
+            "transitive_order",
+            "membership_inheritance",
+            "sequential_state_update",
+        ])
+        self.assertTrue(all(len(item.proof) == 2 for item in skeletons))
+
+    def test_generation_is_deterministic(self):
+        first = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
+        second = [item.to_dict() for item in generate_skeletons(6, 4, seed=7)]
+        self.assertEqual(first, second)
+
+    def test_renderer_never_receives_gold_or_proof(self):
+        skeleton = generate_skeletons(1, "short", seed=1)[0]
+        payload = skeleton.renderer_payload()
+        self.assertNotIn("gold_answer", payload)
+        self.assertNotIn("proof", payload)
+        self.assertIn("answer_options", payload)
+
+    def test_length_validation(self):
+        self.assertEqual(resolve_length("medium"), 4)
+        self.assertEqual(resolve_length("5"), 5)
+        with self.assertRaises(ValueError):
+            resolve_length("huge")
+
+
+if __name__ == "__main__":
+    unittest.main()