Score unparseable answers without aborting batches

This commit is contained in:
2026-09-23 00:52:36 +08:00
parent 06cdd9c556
commit 6f929437a2
3 changed files with 33 additions and 4 deletions
+15 -3
View File
@@ -33,8 +33,9 @@ class EvaluationResult:
gold_answer: str
gold_present_probability: float
solver_output: str
model_answer: str
model_answer: str | None
format_compliant: bool
parse_error: str | None
correct: bool
@@ -111,7 +112,8 @@ def request_solver_output(client: JsonModel, problem: str, answer_options: tuple
system=(
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
"200 words, then put "
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
),
@@ -148,7 +150,13 @@ def run_evaluation(
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
_write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
try:
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
parse_error = None
except RuntimeError as exc:
answer = None
format_compliant = False
parse_error = str(exc)
results.append(
EvaluationResult(
skeleton_id=skeleton.id,
@@ -161,17 +169,21 @@ def run_evaluation(
solver_output=solver_output,
model_answer=answer,
format_compliant=format_compliant,
parse_error=parse_error,
correct=answer == skeleton.gold_answer,
)
)
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
correct = sum(item.correct for item in results)
parsed = sum(item.model_answer is not None for item in results)
format_compliant = sum(item.format_compliant for item in results)
summary = {
"count": len(results),
"correct": correct,
"accuracy": correct / len(results),
"parsed": parsed,
"parse_rate": parsed / len(results),
"format_compliant": format_compliant,
"format_compliance": format_compliant / len(results),
"length": length,