1254 lines
29 KiB
JSON
1254 lines
29 KiB
JSON
[
|
|
{
|
|
"blind_id": "P01",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.0,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"extra_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P02",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.667,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P03",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.0,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"extra_hypothesis",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P04",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P05",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.444,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"goal_mismatch",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P06",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P07",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.615,
|
|
"role_conflict_types": [
|
|
"type_conflict",
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 7,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P08",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.667,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P09",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.222,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 8,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P10",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P11",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P12",
|
|
"reference_verdict": "not_formalizable",
|
|
"assistant_verdict": "not_formalizable",
|
|
"assistant_error_types": [
|
|
"other"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P13",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.889,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P14",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P15",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 0.857,
|
|
"role_conflict_types": [
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P16",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.833,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 8,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P17",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.286,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P18",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.4,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P19",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.333,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"extra_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P20",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.727,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P21",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P22",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P23",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.9,
|
|
"role_conflict_types": [
|
|
"type_conflict",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 9,
|
|
"lean_role_count": 11,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P24",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P25",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.909,
|
|
"role_conflict_types": [
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 6,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P26",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"extra_hypothesis",
|
|
"missing_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.769,
|
|
"role_conflict_types": [
|
|
"extra_hypothesis",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 7,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P27",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.333,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object",
|
|
"missing_hypothesis"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P28",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.333,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P29",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.833,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 7,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P30",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.941,
|
|
"role_conflict_types": [
|
|
"type_conflict",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 9,
|
|
"lean_role_count": 8,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P31",
|
|
"reference_verdict": "not_formalizable",
|
|
"assistant_verdict": "not_formalizable",
|
|
"assistant_error_types": [
|
|
"other"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 2,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P32",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P33",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P34",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P35",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"quantifier"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [
|
|
"type_conflict",
|
|
"extra_hypothesis",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P36",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.571,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P37",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.286,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"type_conflict",
|
|
"missing_hypothesis",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P38",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"missing_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.909,
|
|
"role_conflict_types": [
|
|
"missing_hypothesis"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P39",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P40",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.571,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P41",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"extra_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.667,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P42",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion",
|
|
"missing_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"missing_hypothesis",
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P43",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"missing_hypothesis",
|
|
"extra_object",
|
|
"extra_hypothesis",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P44",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.333,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P45",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"quantifier",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object",
|
|
"missing_hypothesis"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P46",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.667,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 2,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P47",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P48",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 5,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P49",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P50",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.286,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P51",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P52",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.8,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P53",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"extra_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 2,
|
|
"lean_role_count": 2,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P54",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.75,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P55",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P56",
|
|
"reference_verdict": "not_formalizable",
|
|
"assistant_verdict": "not_formalizable",
|
|
"assistant_error_types": [
|
|
"other"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": true,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P57",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": false,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.6,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"goal_mismatch"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P58",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object",
|
|
"quantifier"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P59",
|
|
"reference_verdict": "minor_drift",
|
|
"assistant_verdict": "minor_drift",
|
|
"assistant_error_types": [
|
|
"wrong_type"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.75,
|
|
"role_conflict_types": [
|
|
"missing_hypothesis",
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 4,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P60",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"wrong_type",
|
|
"missing_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.25,
|
|
"role_conflict_types": [
|
|
"missing_object",
|
|
"missing_hypothesis",
|
|
"extra_object",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 5,
|
|
"lean_role_count": 3,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P61",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.286,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object",
|
|
"missing_hypothesis"
|
|
],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
},
|
|
{
|
|
"blind_id": "P62",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 0.857,
|
|
"role_conflict_types": [
|
|
"extra_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 4,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P63",
|
|
"reference_verdict": "faithful",
|
|
"assistant_verdict": "faithful",
|
|
"assistant_error_types": [],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": false,
|
|
"role_score": 1.0,
|
|
"role_conflict_types": [],
|
|
"source_role_count": 6,
|
|
"lean_role_count": 6,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P64",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"extra_hypothesis",
|
|
"missing_hypothesis"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": false,
|
|
"role_flagged": true,
|
|
"role_score": 0.875,
|
|
"role_conflict_types": [
|
|
"extra_object",
|
|
"extra_hypothesis"
|
|
],
|
|
"source_role_count": 7,
|
|
"lean_role_count": 9,
|
|
"degenerate_projection": false
|
|
},
|
|
{
|
|
"blind_id": "P65",
|
|
"reference_verdict": "wrong",
|
|
"assistant_verdict": "wrong",
|
|
"assistant_error_types": [
|
|
"degenerate",
|
|
"wrong_conclusion"
|
|
],
|
|
"verdict_agrees": true,
|
|
"compiled": false,
|
|
"lint_flagged": true,
|
|
"role_flagged": true,
|
|
"role_score": 0.5,
|
|
"role_conflict_types": [
|
|
"degenerate",
|
|
"missing_object"
|
|
],
|
|
"source_role_count": 3,
|
|
"lean_role_count": 1,
|
|
"degenerate_projection": true
|
|
}
|
|
]
|