feat(pilot): add full-cohort GPT-assisted diagnostic
ci / python (pull_request) Successful in 14s
ci / lean (pull_request) Successful in 29s
ci / image (pull_request) Has been skipped

This commit is contained in:
2026-10-28 19:28:07 +08:00
parent 5d531c9c5f
commit 2f0a217a94
6 changed files with 1942 additions and 0 deletions
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,275 @@
{
"experiment": "P0-GPT-diagnostic",
"status": "complete",
"n": 65,
"provenance": {
"run_kind": "current-chat assistant-authored diagnostic; not an API benchmark",
"evaluator": "Codex GPT assistant; exact deployed model identifier unavailable",
"authored_at": "2026-10-28T11:23:06.275167+00:00",
"items_sha256": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
"key_sha256": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
"reference_labels_seen_in_prior_chat": true,
"lean_seen_while_authoring_source_roles": true,
"new_external_api_calls": 0,
"assistant_token_usage": null,
"limitations": [
"Same assistant previously authored the reference labels; agreement is not independent accuracy.",
"Prior context contains the earlier review and detector code. This is not a blinded source-only extraction or a fresh-model replicate.",
"All judgments are manually authored in this chat, not responses from a GPT API endpoint.",
"Semantics are assessed conditionally on recoverable notation. No new Lean compilation or library API audit is performed.",
"The strict Boolean judge flags all verdicts except faithful, including not_formalizable; four-way verdicts are also retained."
]
},
"decision": null,
"decision_withheld_reason": "Same-assistant, non-blind diagnostic; not the preregistered Qwen pilot or independent ground-truth validation.",
"hashes": {
"items": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
"key": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
"assessments": "f12735802ba0d4d94b776a91581a13a508eab0ce90ba8973a6c9c4e4d1390272",
"reference_labels": "1552dba85e75426799151384e7e28c9a0b1072ed4c862df0876df89920d2a638",
"replay_script": "5c02e3d26761ce0b4e38d4be9e65b95bfd8db048c9c7c73edaac17e654e10c0f",
"leanbench/align.py": "730e44f1158e6e441e4745dff7324e565b89b6e5148017c3fd93f4c69c22c9ce",
"leanbench/project.py": "8ed76924b463ba418ceb732d0fcb6c572a548d94fe7f23759c67746da24d9430",
"leanbench/srcsimplex.py": "d10b785299ceb17cadc4e97737eef74183ed390d04b4571a702cd4fd42b069d0",
"leanbench/lint.py": "4396e879312f370cd70beeffcbd7ffeab8a41ee1704cc1f235dbbcd6e843fd68"
},
"fixed_thresholds": {
"matching": 0.3,
"goal": 0.2
},
"verdict_counts": {
"faithful": 25,
"minor_drift": 13,
"wrong": 24,
"not_formalizable": 3
},
"role_flagged": 55,
"lint_flagged": 21,
"compiled": 10,
"degenerate_projection": 15,
"exact_verdict_agreement": 59,
"changed_verdicts": [
"P02",
"P18",
"P23",
"P34",
"P51",
"P57"
],
"primary_reference_agreement": {
"n": 55,
"metrics_are_non_independent_reference_agreement": true,
"detectors": {
"role": {
"tp": 25,
"fp": 21,
"fn": 0,
"tn": 9,
"n": 55,
"recall": 1.0,
"precision": 0.5434782608695652,
"false_flag_rate": 0.7,
"f1": 0.704225352112676,
"recall_ci": [
0.8668035060468213,
1.0
],
"false_flag_ci": [
0.5212387898833981,
0.8333543735604105
],
"precision_ci": [
0.4017993496608042,
0.67845489578878
],
"auc": 0.61
},
"assistant_judge": {
"tp": 25,
"fp": 5,
"fn": 0,
"tn": 25,
"n": 55,
"recall": 1.0,
"precision": 0.8333333333333334,
"false_flag_rate": 0.16666666666666666,
"f1": 0.9090909090909091,
"recall_ci": [
0.8668035060468213,
1.0
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.6643529481431982,
0.9266356575964831
]
},
"lint": {
"tp": 12,
"fp": 5,
"fn": 13,
"tn": 25,
"n": 55,
"recall": 0.48,
"precision": 0.7058823529411765,
"false_flag_rate": 0.16666666666666666,
"f1": 0.5714285714285713,
"recall_ci": [
0.30031003928960015,
0.6650178204685269
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.468664666737861,
0.8672020613444361
]
},
"compile_only": {
"tp": 24,
"fp": 24,
"fn": 1,
"tn": 6,
"n": 55,
"recall": 0.96,
"precision": 0.5,
"false_flag_rate": 0.8,
"f1": 0.6575342465753424,
"recall_ci": [
0.8045552759497747,
0.9929039496133008
],
"false_flag_ci": [
0.6269395261897256,
0.9049502189759876
],
"precision_ci": [
0.36389098932037767,
0.6361090106796223
]
}
}
},
"secondary_reference_agreement": {
"n": 62,
"metrics_are_non_independent_reference_agreement": true,
"detectors": {
"role": {
"tp": 32,
"fp": 21,
"fn": 0,
"tn": 9,
"n": 62,
"recall": 1.0,
"precision": 0.6037735849056604,
"false_flag_rate": 0.7,
"f1": 0.7529411764705882,
"recall_ci": [
0.8928172849426366,
1.0
],
"false_flag_ci": [
0.5212387898833981,
0.8333543735604105
],
"precision_ci": [
0.4694123578888073,
0.7241078808096105
],
"auc": 0.590625
},
"assistant_judge": {
"tp": 32,
"fp": 5,
"fn": 0,
"tn": 25,
"n": 62,
"recall": 1.0,
"precision": 0.8648648648648649,
"false_flag_rate": 0.16666666666666666,
"f1": 0.927536231884058,
"recall_ci": [
0.8928172849426366,
1.0
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.7202227252929444,
0.9408679273749238
]
},
"lint": {
"tp": 14,
"fp": 5,
"fn": 18,
"tn": 25,
"n": 62,
"recall": 0.4375,
"precision": 0.7368421052631579,
"false_flag_rate": 0.16666666666666666,
"f1": 0.5490196078431372,
"recall_ci": [
0.2816508596944317,
0.6067469796877387
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.5120803155892935,
0.8819376166045982
]
},
"compile_only": {
"tp": 29,
"fp": 24,
"fn": 3,
"tn": 6,
"n": 62,
"recall": 0.90625,
"precision": 0.5471698113207547,
"false_flag_rate": 0.8,
"f1": 0.6823529411764705,
"recall_ci": [
0.7578150066422116,
0.9675990373736805
],
"false_flag_ci": [
0.6269395261897256,
0.9049502189759876
],
"precision_ci": [
0.41453742646439184,
0.6734263183985253
]
}
}
},
"role_flagged_by_new_verdict": {
"faithful": {
"n": 25,
"flagged": 16
},
"minor_drift": {
"n": 13,
"flagged": 13
},
"not_formalizable": {
"n": 3,
"flagged": 2
},
"wrong": {
"n": 24,
"flagged": 24
}
}
}