Build initial logic skeleton benchmark
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from logicbench.pipeline import run_evaluation
|
||||
|
||||
|
||||
class QueueClient:
|
||||
def __init__(self, responses):
|
||||
self.responses = iter(responses)
|
||||
|
||||
def request_json(self, **_kwargs):
|
||||
return next(self.responses)
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
||||
renderer = QueueClient([
|
||||
{"problem": "Choose LEFT_OF, RIGHT_OF, or UNKNOWN."},
|
||||
{"problem": "Choose YES, NO, or UNKNOWN."},
|
||||
{"problem": "Choose 9, 10, or 8."},
|
||||
])
|
||||
# Seed 20260922 makes the third gold answer 9.
|
||||
solver = QueueClient([
|
||||
{"answer": "LEFT_OF"},
|
||||
{"answer": "YES"},
|
||||
{"answer": "9"},
|
||||
])
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory)
|
||||
summary = run_evaluation(
|
||||
renderer=renderer,
|
||||
solver=solver,
|
||||
count=3,
|
||||
length="short",
|
||||
seed=20260922,
|
||||
output_dir=output,
|
||||
)
|
||||
self.assertTrue((output / "skeletons.jsonl").is_file())
|
||||
self.assertTrue((output / "results.jsonl").is_file())
|
||||
self.assertEqual(summary["count"], 3)
|
||||
self.assertEqual(summary["correct"], 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user