Render dynamic histories in configurable chunks
This commit is contained in:
+34
-5
@@ -23,11 +23,33 @@ class FailingSolver:
|
||||
|
||||
|
||||
class AuditRenderer:
|
||||
def __init__(self):
|
||||
self.chunk_sizes = []
|
||||
self.surface_name_maps = []
|
||||
|
||||
def request_json(self, **kwargs):
|
||||
payload = json.loads(kwargs["user"][kwargs["user"].find("{") :])
|
||||
identifiers = [item["id"] for item in payload["initial_observations"]]
|
||||
identifiers.extend(item["id"] for item in payload["events"])
|
||||
return {"problem": "A compact behavioral problem.\n" + "\n".join(identifiers)}
|
||||
if kwargs["schema_name"] == "rendering_plan":
|
||||
return {
|
||||
"setting_title": "Crystal Archive",
|
||||
"setting_premise": "A compact behavioral problem.",
|
||||
"style_guide": "Describe every record as an archive log entry.",
|
||||
}
|
||||
payload = json.loads(kwargs["user"])
|
||||
records = payload["records"]
|
||||
self.chunk_sizes.append(len(records))
|
||||
self.surface_name_maps.append(payload["surface_names"])
|
||||
return {"text": "\n".join(f"{item['id']} rendered" for item in records)}
|
||||
|
||||
|
||||
class OmittingRenderer:
|
||||
def request_json(self, **kwargs):
|
||||
if kwargs["schema_name"] == "rendering_plan":
|
||||
return {
|
||||
"setting_title": "Incomplete Archive",
|
||||
"setting_premise": "Some records are missing.",
|
||||
"style_guide": "Use log lines.",
|
||||
}
|
||||
return {"text": "This rendering omitted its audit IDs."}
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
@@ -49,6 +71,7 @@ class PipelineTests(unittest.TestCase):
|
||||
length="short",
|
||||
seed=20260922,
|
||||
output_dir=output,
|
||||
render_chunk_size=17,
|
||||
)
|
||||
self.assertTrue((output / "skeletons.jsonl").is_file())
|
||||
self.assertTrue((output / "rendered.jsonl").is_file())
|
||||
@@ -57,6 +80,12 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertEqual(summary["count"], 5)
|
||||
self.assertEqual(summary["correct"], 5)
|
||||
self.assertEqual(summary["format_compliant"], 5)
|
||||
self.assertEqual(summary["render_chunk_size"], 17)
|
||||
self.assertTrue(renderer.chunk_sizes)
|
||||
self.assertLessEqual(max(renderer.chunk_sizes), 17)
|
||||
self.assertTrue(
|
||||
all(mapping == renderer.surface_name_maps[0] for mapping in renderer.surface_name_maps)
|
||||
)
|
||||
|
||||
def test_answer_parser_requires_final_marker_and_permitted_label(self):
|
||||
self.assertEqual(parse_marked_answer("Reason.\n#### YES", ("YES", "NO")), "YES")
|
||||
@@ -105,7 +134,7 @@ class PipelineTests(unittest.TestCase):
|
||||
self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
|
||||
|
||||
def test_renderer_omission_is_rejected_before_solving(self):
|
||||
renderer = QueueClient([{"problem": "This rendering omitted its audit IDs."}])
|
||||
renderer = OmittingRenderer()
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
with self.assertRaisesRegex(RuntimeError, "renderer omitted"):
|
||||
run_evaluation(
|
||||
|
||||
Reference in New Issue
Block a user