Score unparseable answers without aborting batches
This commit is contained in:
@@ -59,7 +59,7 @@ Outputs under the task's `output_dir` are:
|
|||||||
- `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call;
|
- `skeletons.jsonl`: full hidden programs, sampled behavior, exact probabilities, gold labels, and latent traces; written before any LLM call;
|
||||||
- `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering;
|
- `rendered.jsonl`: each natural-language problem, checkpointed immediately after rendering;
|
||||||
- `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing;
|
- `solver-outputs.jsonl`: each complete raw solver response, checkpointed before parsing;
|
||||||
- `results.jsonl`: rendered problems, complete solver responses, parsed labels, gold probabilities, and scores;
|
- `results.jsonl`: rendered problems, complete solver responses, parsed labels, parse errors, format compliance, gold probabilities, and scores;
|
||||||
- `summary.json`: aggregate accuracy and structural metadata.
|
- `summary.json`: aggregate accuracy and structural metadata.
|
||||||
|
|
||||||
## Gitea Actions one-shot test
|
## Gitea Actions one-shot test
|
||||||
|
|||||||
+15
-3
@@ -33,8 +33,9 @@ class EvaluationResult:
|
|||||||
gold_answer: str
|
gold_answer: str
|
||||||
gold_present_probability: float
|
gold_present_probability: float
|
||||||
solver_output: str
|
solver_output: str
|
||||||
model_answer: str
|
model_answer: str | None
|
||||||
format_compliant: bool
|
format_compliant: bool
|
||||||
|
parse_error: str | None
|
||||||
correct: bool
|
correct: bool
|
||||||
|
|
||||||
|
|
||||||
@@ -111,7 +112,8 @@ def request_solver_output(client: JsonModel, problem: str, answer_options: tuple
|
|||||||
system=(
|
system=(
|
||||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||||
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
|
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
|
||||||
|
"200 words, then put "
|
||||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||||
),
|
),
|
||||||
@@ -148,7 +150,13 @@ def run_evaluation(
|
|||||||
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
|
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
|
||||||
solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
|
solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
|
||||||
_write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
|
_write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
|
||||||
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
|
try:
|
||||||
|
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
|
||||||
|
parse_error = None
|
||||||
|
except RuntimeError as exc:
|
||||||
|
answer = None
|
||||||
|
format_compliant = False
|
||||||
|
parse_error = str(exc)
|
||||||
results.append(
|
results.append(
|
||||||
EvaluationResult(
|
EvaluationResult(
|
||||||
skeleton_id=skeleton.id,
|
skeleton_id=skeleton.id,
|
||||||
@@ -161,17 +169,21 @@ def run_evaluation(
|
|||||||
solver_output=solver_output,
|
solver_output=solver_output,
|
||||||
model_answer=answer,
|
model_answer=answer,
|
||||||
format_compliant=format_compliant,
|
format_compliant=format_compliant,
|
||||||
|
parse_error=parse_error,
|
||||||
correct=answer == skeleton.gold_answer,
|
correct=answer == skeleton.gold_answer,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
|
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
|
||||||
|
|
||||||
correct = sum(item.correct for item in results)
|
correct = sum(item.correct for item in results)
|
||||||
|
parsed = sum(item.model_answer is not None for item in results)
|
||||||
format_compliant = sum(item.format_compliant for item in results)
|
format_compliant = sum(item.format_compliant for item in results)
|
||||||
summary = {
|
summary = {
|
||||||
"count": len(results),
|
"count": len(results),
|
||||||
"correct": correct,
|
"correct": correct,
|
||||||
"accuracy": correct / len(results),
|
"accuracy": correct / len(results),
|
||||||
|
"parsed": parsed,
|
||||||
|
"parse_rate": parsed / len(results),
|
||||||
"format_compliant": format_compliant,
|
"format_compliant": format_compliant,
|
||||||
"format_compliance": format_compliant / len(results),
|
"format_compliance": format_compliant / len(results),
|
||||||
"length": length,
|
"length": length,
|
||||||
|
|||||||
@@ -84,6 +84,23 @@ class PipelineTests(unittest.TestCase):
|
|||||||
)
|
)
|
||||||
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
|
self.assertIn("A compact behavioral problem.", (output / "rendered.jsonl").read_text())
|
||||||
|
|
||||||
|
def test_unparseable_solver_output_is_scored_without_stopping_batch(self):
|
||||||
|
renderer = QueueClient([{"problem": "A compact behavioral problem."}])
|
||||||
|
solver = QueueClient(["The response ended before its final label."])
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
output = Path(directory)
|
||||||
|
summary = run_evaluation(
|
||||||
|
renderer=renderer,
|
||||||
|
solver=solver,
|
||||||
|
count=1,
|
||||||
|
length="short",
|
||||||
|
seed=1,
|
||||||
|
output_dir=output,
|
||||||
|
)
|
||||||
|
self.assertEqual(summary["parsed"], 0)
|
||||||
|
self.assertEqual(summary["correct"], 0)
|
||||||
|
self.assertIn('"parse_error":', (output / "results.jsonl").read_text())
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
unittest.main()
|
unittest.main()
|
||||||
|
|||||||
Reference in New Issue
Block a user