276 lines
7.4 KiB
JSON
276 lines
7.4 KiB
JSON
{
|
|
"experiment": "P0-GPT-diagnostic",
|
|
"status": "complete",
|
|
"n": 65,
|
|
"provenance": {
|
|
"run_kind": "current-chat assistant-authored diagnostic; not an API benchmark",
|
|
"evaluator": "Codex GPT assistant; exact deployed model identifier unavailable",
|
|
"authored_at": "2026-10-28T11:23:06.275167+00:00",
|
|
"items_sha256": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
|
|
"key_sha256": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
|
|
"reference_labels_seen_in_prior_chat": true,
|
|
"lean_seen_while_authoring_source_roles": true,
|
|
"new_external_api_calls": 0,
|
|
"assistant_token_usage": null,
|
|
"limitations": [
|
|
"Same assistant previously authored the reference labels; agreement is not independent accuracy.",
|
|
"Prior context contains the earlier review and detector code. This is not a blinded source-only extraction or a fresh-model replicate.",
|
|
"All judgments are manually authored in this chat, not responses from a GPT API endpoint.",
|
|
"Semantics are assessed conditionally on recoverable notation. No new Lean compilation or library API audit is performed.",
|
|
"The strict Boolean judge flags all verdicts except faithful, including not_formalizable; four-way verdicts are also retained."
|
|
]
|
|
},
|
|
"decision": null,
|
|
"decision_withheld_reason": "Same-assistant, non-blind diagnostic; not the preregistered Qwen pilot or independent ground-truth validation.",
|
|
"hashes": {
|
|
"items": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
|
|
"key": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
|
|
"assessments": "f12735802ba0d4d94b776a91581a13a508eab0ce90ba8973a6c9c4e4d1390272",
|
|
"reference_labels": "1552dba85e75426799151384e7e28c9a0b1072ed4c862df0876df89920d2a638",
|
|
"replay_script": "5c02e3d26761ce0b4e38d4be9e65b95bfd8db048c9c7c73edaac17e654e10c0f",
|
|
"leanbench/align.py": "730e44f1158e6e441e4745dff7324e565b89b6e5148017c3fd93f4c69c22c9ce",
|
|
"leanbench/project.py": "8ed76924b463ba418ceb732d0fcb6c572a548d94fe7f23759c67746da24d9430",
|
|
"leanbench/srcsimplex.py": "d10b785299ceb17cadc4e97737eef74183ed390d04b4571a702cd4fd42b069d0",
|
|
"leanbench/lint.py": "4396e879312f370cd70beeffcbd7ffeab8a41ee1704cc1f235dbbcd6e843fd68"
|
|
},
|
|
"fixed_thresholds": {
|
|
"matching": 0.3,
|
|
"goal": 0.2
|
|
},
|
|
"verdict_counts": {
|
|
"faithful": 25,
|
|
"minor_drift": 13,
|
|
"wrong": 24,
|
|
"not_formalizable": 3
|
|
},
|
|
"role_flagged": 55,
|
|
"lint_flagged": 21,
|
|
"compiled": 10,
|
|
"degenerate_projection": 15,
|
|
"exact_verdict_agreement": 59,
|
|
"changed_verdicts": [
|
|
"P02",
|
|
"P18",
|
|
"P23",
|
|
"P34",
|
|
"P51",
|
|
"P57"
|
|
],
|
|
"primary_reference_agreement": {
|
|
"n": 55,
|
|
"metrics_are_non_independent_reference_agreement": true,
|
|
"detectors": {
|
|
"role": {
|
|
"tp": 25,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tn": 9,
|
|
"n": 55,
|
|
"recall": 1.0,
|
|
"precision": 0.5434782608695652,
|
|
"false_flag_rate": 0.7,
|
|
"f1": 0.704225352112676,
|
|
"recall_ci": [
|
|
0.8668035060468213,
|
|
1.0
|
|
],
|
|
"false_flag_ci": [
|
|
0.5212387898833981,
|
|
0.8333543735604105
|
|
],
|
|
"precision_ci": [
|
|
0.4017993496608042,
|
|
0.67845489578878
|
|
],
|
|
"auc": 0.61
|
|
},
|
|
"assistant_judge": {
|
|
"tp": 25,
|
|
"fp": 5,
|
|
"fn": 0,
|
|
"tn": 25,
|
|
"n": 55,
|
|
"recall": 1.0,
|
|
"precision": 0.8333333333333334,
|
|
"false_flag_rate": 0.16666666666666666,
|
|
"f1": 0.9090909090909091,
|
|
"recall_ci": [
|
|
0.8668035060468213,
|
|
1.0
|
|
],
|
|
"false_flag_ci": [
|
|
0.07336434240351683,
|
|
0.33564705185680177
|
|
],
|
|
"precision_ci": [
|
|
0.6643529481431982,
|
|
0.9266356575964831
|
|
]
|
|
},
|
|
"lint": {
|
|
"tp": 12,
|
|
"fp": 5,
|
|
"fn": 13,
|
|
"tn": 25,
|
|
"n": 55,
|
|
"recall": 0.48,
|
|
"precision": 0.7058823529411765,
|
|
"false_flag_rate": 0.16666666666666666,
|
|
"f1": 0.5714285714285713,
|
|
"recall_ci": [
|
|
0.30031003928960015,
|
|
0.6650178204685269
|
|
],
|
|
"false_flag_ci": [
|
|
0.07336434240351683,
|
|
0.33564705185680177
|
|
],
|
|
"precision_ci": [
|
|
0.468664666737861,
|
|
0.8672020613444361
|
|
]
|
|
},
|
|
"compile_only": {
|
|
"tp": 24,
|
|
"fp": 24,
|
|
"fn": 1,
|
|
"tn": 6,
|
|
"n": 55,
|
|
"recall": 0.96,
|
|
"precision": 0.5,
|
|
"false_flag_rate": 0.8,
|
|
"f1": 0.6575342465753424,
|
|
"recall_ci": [
|
|
0.8045552759497747,
|
|
0.9929039496133008
|
|
],
|
|
"false_flag_ci": [
|
|
0.6269395261897256,
|
|
0.9049502189759876
|
|
],
|
|
"precision_ci": [
|
|
0.36389098932037767,
|
|
0.6361090106796223
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"secondary_reference_agreement": {
|
|
"n": 62,
|
|
"metrics_are_non_independent_reference_agreement": true,
|
|
"detectors": {
|
|
"role": {
|
|
"tp": 32,
|
|
"fp": 21,
|
|
"fn": 0,
|
|
"tn": 9,
|
|
"n": 62,
|
|
"recall": 1.0,
|
|
"precision": 0.6037735849056604,
|
|
"false_flag_rate": 0.7,
|
|
"f1": 0.7529411764705882,
|
|
"recall_ci": [
|
|
0.8928172849426366,
|
|
1.0
|
|
],
|
|
"false_flag_ci": [
|
|
0.5212387898833981,
|
|
0.8333543735604105
|
|
],
|
|
"precision_ci": [
|
|
0.4694123578888073,
|
|
0.7241078808096105
|
|
],
|
|
"auc": 0.590625
|
|
},
|
|
"assistant_judge": {
|
|
"tp": 32,
|
|
"fp": 5,
|
|
"fn": 0,
|
|
"tn": 25,
|
|
"n": 62,
|
|
"recall": 1.0,
|
|
"precision": 0.8648648648648649,
|
|
"false_flag_rate": 0.16666666666666666,
|
|
"f1": 0.927536231884058,
|
|
"recall_ci": [
|
|
0.8928172849426366,
|
|
1.0
|
|
],
|
|
"false_flag_ci": [
|
|
0.07336434240351683,
|
|
0.33564705185680177
|
|
],
|
|
"precision_ci": [
|
|
0.7202227252929444,
|
|
0.9408679273749238
|
|
]
|
|
},
|
|
"lint": {
|
|
"tp": 14,
|
|
"fp": 5,
|
|
"fn": 18,
|
|
"tn": 25,
|
|
"n": 62,
|
|
"recall": 0.4375,
|
|
"precision": 0.7368421052631579,
|
|
"false_flag_rate": 0.16666666666666666,
|
|
"f1": 0.5490196078431372,
|
|
"recall_ci": [
|
|
0.2816508596944317,
|
|
0.6067469796877387
|
|
],
|
|
"false_flag_ci": [
|
|
0.07336434240351683,
|
|
0.33564705185680177
|
|
],
|
|
"precision_ci": [
|
|
0.5120803155892935,
|
|
0.8819376166045982
|
|
]
|
|
},
|
|
"compile_only": {
|
|
"tp": 29,
|
|
"fp": 24,
|
|
"fn": 3,
|
|
"tn": 6,
|
|
"n": 62,
|
|
"recall": 0.90625,
|
|
"precision": 0.5471698113207547,
|
|
"false_flag_rate": 0.8,
|
|
"f1": 0.6823529411764705,
|
|
"recall_ci": [
|
|
0.7578150066422116,
|
|
0.9675990373736805
|
|
],
|
|
"false_flag_ci": [
|
|
0.6269395261897256,
|
|
0.9049502189759876
|
|
],
|
|
"precision_ci": [
|
|
0.41453742646439184,
|
|
0.6734263183985253
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"role_flagged_by_new_verdict": {
|
|
"faithful": {
|
|
"n": 25,
|
|
"flagged": 16
|
|
},
|
|
"minor_drift": {
|
|
"n": 13,
|
|
"flagged": 13
|
|
},
|
|
"not_formalizable": {
|
|
"n": 3,
|
|
"flagged": 2
|
|
},
|
|
"wrong": {
|
|
"n": 24,
|
|
"flagged": 24
|
|
}
|
|
}
|
|
}
|