{ "experiment": "P0-GPT-diagnostic", "status": "complete", "n": 65, "provenance": { "run_kind": "current-chat assistant-authored diagnostic; not an API benchmark", "evaluator": "Codex GPT assistant; exact deployed model identifier unavailable", "authored_at": "2026-10-28T11:23:06.275167+00:00", "items_sha256": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed", "key_sha256": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836", "reference_labels_seen_in_prior_chat": true, "lean_seen_while_authoring_source_roles": true, "new_external_api_calls": 0, "assistant_token_usage": null, "limitations": [ "Same assistant previously authored the reference labels; agreement is not independent accuracy.", "Prior context contains the earlier review and detector code. This is not a blinded source-only extraction or a fresh-model replicate.", "All judgments are manually authored in this chat, not responses from a GPT API endpoint.", "Semantics are assessed conditionally on recoverable notation. No new Lean compilation or library API audit is performed.", "The strict Boolean judge flags all verdicts except faithful, including not_formalizable; four-way verdicts are also retained." ] }, "decision": null, "decision_withheld_reason": "Same-assistant, non-blind diagnostic; not the preregistered Qwen pilot or independent ground-truth validation.", "hashes": { "items": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed", "key": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836", "assessments": "f12735802ba0d4d94b776a91581a13a508eab0ce90ba8973a6c9c4e4d1390272", "reference_labels": "1552dba85e75426799151384e7e28c9a0b1072ed4c862df0876df89920d2a638", "replay_script": "5c02e3d26761ce0b4e38d4be9e65b95bfd8db048c9c7c73edaac17e654e10c0f", "leanbench/align.py": "730e44f1158e6e441e4745dff7324e565b89b6e5148017c3fd93f4c69c22c9ce", "leanbench/project.py": "8ed76924b463ba418ceb732d0fcb6c572a548d94fe7f23759c67746da24d9430", "leanbench/srcsimplex.py": "d10b785299ceb17cadc4e97737eef74183ed390d04b4571a702cd4fd42b069d0", "leanbench/lint.py": "4396e879312f370cd70beeffcbd7ffeab8a41ee1704cc1f235dbbcd6e843fd68" }, "fixed_thresholds": { "matching": 0.3, "goal": 0.2 }, "verdict_counts": { "faithful": 25, "minor_drift": 13, "wrong": 24, "not_formalizable": 3 }, "role_flagged": 55, "lint_flagged": 21, "compiled": 10, "degenerate_projection": 15, "exact_verdict_agreement": 59, "changed_verdicts": [ "P02", "P18", "P23", "P34", "P51", "P57" ], "primary_reference_agreement": { "n": 55, "metrics_are_non_independent_reference_agreement": true, "detectors": { "role": { "tp": 25, "fp": 21, "fn": 0, "tn": 9, "n": 55, "recall": 1.0, "precision": 0.5434782608695652, "false_flag_rate": 0.7, "f1": 0.704225352112676, "recall_ci": [ 0.8668035060468213, 1.0 ], "false_flag_ci": [ 0.5212387898833981, 0.8333543735604105 ], "precision_ci": [ 0.4017993496608042, 0.67845489578878 ], "auc": 0.61 }, "assistant_judge": { "tp": 25, "fp": 5, "fn": 0, "tn": 25, "n": 55, "recall": 1.0, "precision": 0.8333333333333334, "false_flag_rate": 0.16666666666666666, "f1": 0.9090909090909091, "recall_ci": [ 0.8668035060468213, 1.0 ], "false_flag_ci": [ 0.07336434240351683, 0.33564705185680177 ], "precision_ci": [ 0.6643529481431982, 0.9266356575964831 ] }, "lint": { "tp": 12, "fp": 5, "fn": 13, "tn": 25, "n": 55, "recall": 0.48, "precision": 0.7058823529411765, "false_flag_rate": 0.16666666666666666, "f1": 0.5714285714285713, "recall_ci": [ 0.30031003928960015, 0.6650178204685269 ], "false_flag_ci": [ 0.07336434240351683, 0.33564705185680177 ], "precision_ci": [ 0.468664666737861, 0.8672020613444361 ] }, "compile_only": { "tp": 24, "fp": 24, "fn": 1, "tn": 6, "n": 55, "recall": 0.96, "precision": 0.5, "false_flag_rate": 0.8, "f1": 0.6575342465753424, "recall_ci": [ 0.8045552759497747, 0.9929039496133008 ], "false_flag_ci": [ 0.6269395261897256, 0.9049502189759876 ], "precision_ci": [ 0.36389098932037767, 0.6361090106796223 ] } } }, "secondary_reference_agreement": { "n": 62, "metrics_are_non_independent_reference_agreement": true, "detectors": { "role": { "tp": 32, "fp": 21, "fn": 0, "tn": 9, "n": 62, "recall": 1.0, "precision": 0.6037735849056604, "false_flag_rate": 0.7, "f1": 0.7529411764705882, "recall_ci": [ 0.8928172849426366, 1.0 ], "false_flag_ci": [ 0.5212387898833981, 0.8333543735604105 ], "precision_ci": [ 0.4694123578888073, 0.7241078808096105 ], "auc": 0.590625 }, "assistant_judge": { "tp": 32, "fp": 5, "fn": 0, "tn": 25, "n": 62, "recall": 1.0, "precision": 0.8648648648648649, "false_flag_rate": 0.16666666666666666, "f1": 0.927536231884058, "recall_ci": [ 0.8928172849426366, 1.0 ], "false_flag_ci": [ 0.07336434240351683, 0.33564705185680177 ], "precision_ci": [ 0.7202227252929444, 0.9408679273749238 ] }, "lint": { "tp": 14, "fp": 5, "fn": 18, "tn": 25, "n": 62, "recall": 0.4375, "precision": 0.7368421052631579, "false_flag_rate": 0.16666666666666666, "f1": 0.5490196078431372, "recall_ci": [ 0.2816508596944317, 0.6067469796877387 ], "false_flag_ci": [ 0.07336434240351683, 0.33564705185680177 ], "precision_ci": [ 0.5120803155892935, 0.8819376166045982 ] }, "compile_only": { "tp": 29, "fp": 24, "fn": 3, "tn": 6, "n": 62, "recall": 0.90625, "precision": 0.5471698113207547, "false_flag_rate": 0.8, "f1": 0.6823529411764705, "recall_ci": [ 0.7578150066422116, 0.9675990373736805 ], "false_flag_ci": [ 0.6269395261897256, 0.9049502189759876 ], "precision_ci": [ 0.41453742646439184, 0.6734263183985253 ] } } }, "role_flagged_by_new_verdict": { "faithful": { "n": 25, "flagged": 16 }, "minor_drift": { "n": 13, "flagged": 13 }, "not_formalizable": { "n": 3, "flagged": 2 }, "wrong": { "n": 24, "flagged": 24 } } }