152 lines
5.8 KiB
Python
152 lines
5.8 KiB
Python
import json
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from logicbench.pipeline import parse_marked_answer, parse_solver_answer, run_evaluation
|
|
|
|
|
|
class QueueClient:
|
|
def __init__(self, responses):
|
|
self.responses = iter(responses)
|
|
|
|
def request_json(self, **_kwargs):
|
|
return next(self.responses)
|
|
|
|
def request_text(self, **_kwargs):
|
|
return next(self.responses)
|
|
|
|
|
|
class FailingSolver:
|
|
def request_text(self, **_kwargs):
|
|
raise RuntimeError("solver failed")
|
|
|
|
|
|
class AuditRenderer:
|
|
def __init__(self):
|
|
self.chunk_sizes = []
|
|
self.surface_name_maps = []
|
|
|
|
def request_json(self, **kwargs):
|
|
if kwargs["schema_name"] == "rendering_plan":
|
|
return {
|
|
"setting_title": "Crystal Archive",
|
|
"setting_premise": "A compact behavioral problem.",
|
|
"style_guide": "Describe every record as an archive log entry.",
|
|
}
|
|
payload = json.loads(kwargs["user"])
|
|
records = payload["records"]
|
|
self.chunk_sizes.append(len(records))
|
|
self.surface_name_maps.append(payload["surface_names"])
|
|
return {"text": "\n".join(f"{item['id']} rendered" for item in records)}
|
|
|
|
|
|
class OmittingRenderer:
|
|
def request_json(self, **kwargs):
|
|
if kwargs["schema_name"] == "rendering_plan":
|
|
return {
|
|
"setting_title": "Incomplete Archive",
|
|
"setting_premise": "Some records are missing.",
|
|
"style_guide": "Use log lines.",
|
|
}
|
|
return {"text": "This rendering omitted its audit IDs."}
|
|
|
|
|
|
class PipelineTests(unittest.TestCase):
|
|
def test_end_to_end_writes_skeletons_before_scored_results(self):
|
|
renderer = AuditRenderer()
|
|
solver = QueueClient([
|
|
"The relation follows through the event chain.\n#### YES",
|
|
"The target set is disjoint from the inherited membership.\n#### NO",
|
|
"The reverse ordering is false.\n#### NO",
|
|
"Membership propagates through the nested sets.\n#### YES",
|
|
"The relation follows through the event chain.\n#### YES",
|
|
])
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
summary = run_evaluation(
|
|
renderer=renderer,
|
|
solver=solver,
|
|
count=5,
|
|
length="short",
|
|
seed=20260922,
|
|
output_dir=output,
|
|
render_chunk_size=17,
|
|
)
|
|
self.assertTrue((output / "skeletons.jsonl").is_file())
|
|
self.assertTrue((output / "rendered.jsonl").is_file())
|
|
self.assertTrue((output / "solver-outputs.jsonl").is_file())
|
|
self.assertTrue((output / "results.jsonl").is_file())
|
|
self.assertEqual(summary["count"], 5)
|
|
self.assertEqual(summary["correct"], 5)
|
|
self.assertEqual(summary["format_compliant"], 5)
|
|
self.assertEqual(summary["render_chunk_size"], 17)
|
|
self.assertTrue(renderer.chunk_sizes)
|
|
self.assertLessEqual(max(renderer.chunk_sizes), 17)
|
|
self.assertTrue(
|
|
all(mapping == renderer.surface_name_maps[0] for mapping in renderer.surface_name_maps)
|
|
)
|
|
|
|
def test_answer_parser_requires_final_marker_and_permitted_label(self):
|
|
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
|
|
with self.assertRaisesRegex(RuntimeError, "must end"):
|
|
parse_marked_answer("#### YES\nextra", ("YES", "NO"))
|
|
with self.assertRaisesRegex(RuntimeError, "invalid answer"):
|
|
parse_marked_answer("#### MAYBE", ("YES", "NO"))
|
|
|
|
def test_answer_parser_recovers_unique_label_on_final_line(self):
|
|
self.assertEqual(
|
|
parse_solver_answer("Reasoning.\nFinal answer: YES", ("YES", "NO")),
|
|
("YES", False),
|
|
)
|
|
self.assertEqual(parse_solver_answer("Reasoning.\n#### YES", ("YES", "NO")), ("YES", True))
|
|
|
|
def test_rendered_problem_is_checkpointed_before_solver_failure(self):
|
|
renderer = AuditRenderer()
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
with self.assertRaisesRegex(RuntimeError, "solver failed"):
|
|
run_evaluation(
|
|
renderer=renderer,
|
|
solver=FailingSolver(),
|
|
count=1,
|
|
length="short",
|
|
seed=1,
|
|
output_dir=output,
|
|
)
|
|
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
|
|
|
|
def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
|
|
renderer = AuditRenderer()
|
|
solver = QueueClient(["The response ended before its final label."])
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
output = Path(directory)
|
|
summary = run_evaluation(
|
|
renderer=renderer,
|
|
solver=solver,
|
|
count=1,
|
|
length="short",
|
|
seed=1,
|
|
output_dir=output,
|
|
)
|
|
self.assertEqual(summary["parsed"], 0)
|
|
self.assertEqual(summary["correct"], 0)
|
|
self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
|
|
|
|
def test_renderer_omission_is_rejected_before_solving(self):
|
|
renderer = OmittingRenderer()
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
with self.assertRaisesRegex(RuntimeError, "renderer omitted"):
|
|
run_evaluation(
|
|
renderer=renderer,
|
|
solver=FailingSolver(),
|
|
count=1,
|
|
length="short",
|
|
seed=1,
|
|
output_dir=Path(directory),
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|