Score unparseable answers without aborting batches
This commit is contained in:
+15
-3
@@ -33,8 +33,9 @@ class EvaluationResult:
|
||||
gold_answer: str
|
||||
gold_present_probability: float
|
||||
solver_output: str
|
||||
model_answer: str
|
||||
model_answer: str | None
|
||||
format_compliant: bool
|
||||
parse_error: str | None
|
||||
correct: bool
|
||||
|
||||
|
||||
@@ -111,7 +112,8 @@ def request_solver_output(client: JsonModel, problem: str, answer_options: tuple
|
||||
system=(
|
||||
"Infer the stable hidden mechanism from the repeated behavioral observations, then predict which "
|
||||
"query outcome is more likely. Event order, attributes, stochastic frequencies, interactions, and "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Give a concise explanation, then put "
|
||||
"counterfactual controls may matter. Do not demand an exact probability. Keep the explanation under "
|
||||
"200 words, then put "
|
||||
"the final answer on the last non-empty line in the exact form '#### <answer>', where <answer> "
|
||||
"is exactly one permitted answer label. Do not use unstated real-world assumptions."
|
||||
),
|
||||
@@ -148,7 +150,13 @@ def run_evaluation(
|
||||
solver_output = request_solver_output(solver, problem, skeleton.answer_options)
|
||||
solver_records.append({"skeleton_id": skeleton.id, "solver_output": solver_output})
|
||||
_write_jsonl(output_dir / "solver-outputs.jsonl", solver_records)
|
||||
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
|
||||
try:
|
||||
answer, format_compliant = parse_solver_answer(solver_output, skeleton.answer_options)
|
||||
parse_error = None
|
||||
except RuntimeError as exc:
|
||||
answer = None
|
||||
format_compliant = False
|
||||
parse_error = str(exc)
|
||||
results.append(
|
||||
EvaluationResult(
|
||||
skeleton_id=skeleton.id,
|
||||
@@ -161,17 +169,21 @@ def run_evaluation(
|
||||
solver_output=solver_output,
|
||||
model_answer=answer,
|
||||
format_compliant=format_compliant,
|
||||
parse_error=parse_error,
|
||||
correct=answer == skeleton.gold_answer,
|
||||
)
|
||||
)
|
||||
_write_jsonl(output_dir / "results.jsonl", [asdict(item) for item in results])
|
||||
|
||||
correct = sum(item.correct for item in results)
|
||||
parsed = sum(item.model_answer is not None for item in results)
|
||||
format_compliant = sum(item.format_compliant for item in results)
|
||||
summary = {
|
||||
"count": len(results),
|
||||
"correct": correct,
|
||||
"accuracy": correct / len(results),
|
||||
"parsed": parsed,
|
||||
"parse_rate": parsed / len(results),
|
||||
"format_compliant": format_compliant,
|
||||
"format_compliance": format_compliant / len(results),
|
||||
"length": length,
|
||||
|
||||
Reference in New Issue
Block a user