Zayden-Jung/lean-bench
2
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

feat(pilot): add full-cohort GPT-assisted diagnostic
ci / python (pull_request) Successful in 14s
Details
ci / lean (pull_request) Successful in 29s
Details
ci / image (pull_request) Has been skipped
Details

Browse Source
This commit is contained in:
Codex Bot
2026-10-28 19:28:07 +08:00
parent 5d531c9c5f
commit 2f0a217a94
6 changed files with 1942 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
experiments/P0-pilot/README.md
+13
View File
@@ -52,3 +52,16 @@ Labeling must be finished before anyone opens `results.jsonl`.
by the judge. No threshold, concept table or detector prompt was tuned for these labels.
- Raw passages, labeling HTML and model responses remain in the ignored `data/` directory.
Only verdicts, abstract annotation notes, hashes and text-free summaries belong in Git.
## Separate GPT-assisted diagnostic
The owner requested a complete 65-item reassessment by the current GPT assistant on
2026-10-28. This is separate from the incomplete Qwen run: no outputs are mixed and
neither the earlier labels nor Qwen files are overwritten. The assistant had prior
exposure to the reference labels and Lean; this is not blind or independent validation.
No GO/PARTIAL/NO-GO decision is issued.
See `docs/experiments/P0-gpt-diagnostic.md` for all counts, limitations, artifact locations
and the offline replay command. The tracked `results/gpt-diagnostic-*.json` files contain
only categorical/numeric outputs and audit metadata. The authored source structures,
reasons and full review page stay in the ignored `data/gpt-diagnostic/` directory.
experiments/P0-pilot/assistant_diagnostic.py
+200
View File
@@ -0,0 +1,200 @@
"""Validate and replay a current-chat assistant review without calling any model.
Private source schemas and reasons stay in data/. Public summaries contain only
categories, counts, numeric scores, hashes and an explicit non-independent provenance.
"""
from __future__ import annotations
import argparse
import hashlib
import html
import json
import math
from collections import Counter
from pathlib import Path
from leanbench.align import DEFAULT_THRESHOLD, GOAL_THRESHOLD, align
from leanbench.evalstats import auc, confusion, wilson
from leanbench.lint import lint
from leanbench.project import project
from leanbench.srcsimplex import parse_source_simplex
VERDICTS = {"faithful", "minor_drift", "wrong", "not_formalizable"}
ERRORS = {"missing_hypothesis", "extra_hypothesis", "wrong_type", "quantifier",
"wrong_conclusion", "degenerate", "hallucinated_api", "other"}
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def finite(value):
if isinstance(value, dict):
return {k: finite(v) for k, v in value.items()}
if isinstance(value, (list, tuple)):
return [finite(v) for v in value]
if isinstance(value, float) and not math.isfinite(value):
return None
return value
def unique(rows, field):
indexed = {row[field]: row for row in rows}
if len(indexed) != len(rows):
raise ValueError(f"Duplicate {field}")
return indexed
def evaluate(items, key, assessment):
"""Keep model-authored assessments separate from deterministic detector outputs."""
by_id = unique(items, "id")
records = unique(assessment["records"], "blind_id")
if (not key or len(set(key.values())) != len(key) or set(records) != set(key)
or set(by_id) != set(key.values())):
raise ValueError("Items, key and assessments must cover the same cohort exactly once")
output = []
for blind, item_id in key.items():
item, record = by_id[item_id], records[blind]
if record["id"] != item_id or not item.get("source_text", "").strip():
raise ValueError(f"Missing source text or mismatched item for {blind}")
judge = record["judge"]
if (judge.get("verdict") not in VERDICTS or type(judge.get("faithful")) is not bool
or judge["faithful"] != (judge["verdict"] == "faithful")
or not isinstance(judge.get("reason"), str) or not judge["reason"].strip()
or not isinstance(judge.get("errors"), list)
or not set(judge["errors"]) <= ERRORS):
raise ValueError(f"Invalid authored judgment for {blind}")
source = parse_source_simplex(record["source"])
projection = project(item["lean_code"])
alignment = align(source, projection)
output.append({"id": item_id, "blind_id": blind,
"source": source.as_dict(), "judge": judge,
"projection": projection.as_dict(), "role": alignment.as_dict(),
"lint": lint(projection), "compiled": bool(item.get("verified")),
"errors": {}})
return output
def public_items(rows, labels):
if set(labels) != {row["blind_id"] for row in rows}:
raise ValueError("Reference labels must cover the entire cohort")
out = []
for row in rows:
reference = labels[row["blind_id"]]["verdict"]
if reference not in VERDICTS:
raise ValueError("Invalid reference verdict")
out.append({"blind_id": row["blind_id"], "reference_verdict": reference,
"assistant_verdict": row["judge"]["verdict"],
"assistant_error_types": row["judge"]["errors"],
"verdict_agrees": reference == row["judge"]["verdict"],
"compiled": row["compiled"], "lint_flagged": bool(row["lint"]),
"role_flagged": row["role"]["flagged"],
"role_score": row["role"]["score"],
"role_conflict_types": row["role"]["errors"],
"source_role_count": len(row["source"]["roles"]),
"lean_role_count": len(row["projection"]["roles"]),
"degenerate_projection": row["projection"]["degenerate"]})
return out
def agreement(rows, labels, secondary=False):
cohort = [r for r in rows if labels[r["blind_id"]]["verdict"] != "not_formalizable"
and (secondary or labels[r["blind_id"]]["verdict"] != "minor_drift")]
positives = {"wrong", "minor_drift"} if secondary else {"wrong"}
target = [labels[r["blind_id"]]["verdict"] in positives for r in cohort]
out = {}
for name, flags in {
"role": [r["role"]["flagged"] for r in cohort],
"assistant_judge": [not r["judge"]["faithful"] for r in cohort],
"lint": [bool(r["lint"]) for r in cohort],
"compile_only": [not r["compiled"] for r in cohort],
}.items():
counts = confusion(flags, target)
out[name] = {**counts.as_dict(), "precision_ci": wilson(counts.tp, counts.tp + counts.fp)}
out["role"]["auc"] = auc([-r["role"]["score"] for r in cohort], target)
return {"n": len(cohort), "metrics_are_non_independent_reference_agreement": True,
"detectors": out}
def write_review(path, rows, items):
by_id = {r["id"]: r for r in items}
sections = []
for row in rows:
item = by_id[row["id"]]
sections.append(
f'<section id="{html.escape(row["blind_id"])}"><h2>{row["blind_id"]}</h2>'
f'<h3>Passage</h3><pre>{html.escape(item["source_text"])}</pre>'
f'<h3>Lean</h3><pre>{html.escape(item["lean_code"])}</pre>'
f'<h3>Fresh assistant judgment: {row["judge"]["verdict"]}</h3>'
f'<p>{html.escape(row["judge"]["reason"])}</p>'
f'<h3>Source roles</h3><pre>{html.escape(json.dumps(row["source"], indent=2, ensure_ascii=False))}</pre>'
f'<h3>Deterministic alignment</h3><pre>{html.escape(json.dumps(row["role"], indent=2, ensure_ascii=False))}</pre></section>'
)
path.write_text('<!doctype html><html lang="en"><meta charset="utf-8">'
'<title>P0 GPT-assisted diagnostic</title><style>'
'body{font:16px/1.6 system-ui;max-width:1050px;margin:32px auto;padding:0 20px}'
'pre{white-space:pre-wrap;overflow-wrap:anywhere;background:#f4f5f6;padding:16px}'
'section{border-top:1px solid #ccc;margin:40px 0}</style>'
'<h1>P0: 65-item GPT-assisted diagnostic</h1>'
'<p>Private review. Same-assistant, non-blind diagnosis; no independent accuracy or gate decision. '
'Passages and model-derived descriptions must remain outside Git.</p>'
+ ''.join(sections) + '</html>', encoding="utf-8")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--items", type=Path, required=True)
parser.add_argument("--key", type=Path, required=True)
parser.add_argument("--assessments", type=Path, required=True)
parser.add_argument("--labels", type=Path, required=True)
parser.add_argument("--private-out", type=Path, required=True)
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--public-items", type=Path, required=True)
args = parser.parse_args()
if "data" not in args.private_out.parts:
parser.error("Private output must be under the gitignored data/ directory")
items = [json.loads(x) for x in args.items.read_text().splitlines() if x.strip()]
key = json.loads(args.key.read_text())["key"]
assessment = json.loads(args.assessments.read_text())
provenance = assessment["provenance"]
if (provenance["items_sha256"] != digest(args.items)
or provenance["key_sha256"] != digest(args.key)):
parser.error("Authored review is bound to different inputs or blind key")
rows = evaluate(items, key, assessment)
labels = json.loads(args.labels.read_text())["labels"]
public = public_items(rows, labels)
summary = {
"experiment": "P0-GPT-diagnostic", "status": "complete", "n": len(rows),
"provenance": provenance, "decision": None,
"decision_withheld_reason": "Same-assistant, non-blind diagnostic; not the preregistered Qwen pilot or independent ground-truth validation.",
"hashes": {"items": digest(args.items), "key": digest(args.key),
"assessments": digest(args.assessments), "reference_labels": digest(args.labels),
"replay_script": digest(Path(__file__)),
**{f"leanbench/{name}.py": digest(Path(__file__).resolve().parents[2] / "leanbench" / f"{name}.py")
for name in ("align", "project", "srcsimplex", "lint")}},
"fixed_thresholds": {"matching": DEFAULT_THRESHOLD, "goal": GOAL_THRESHOLD},
"verdict_counts": dict(Counter(r["judge"]["verdict"] for r in rows)),
"role_flagged": sum(r["role"]["flagged"] for r in rows),
"lint_flagged": sum(bool(r["lint"]) for r in rows),
"compiled": sum(r["compiled"] for r in rows),
"degenerate_projection": sum(r["projection"]["degenerate"] for r in rows),
"exact_verdict_agreement": sum(r["verdict_agrees"] for r in public),
"changed_verdicts": [r["blind_id"] for r in public if not r["verdict_agrees"]],
"primary_reference_agreement": agreement(rows, labels),
"secondary_reference_agreement": agreement(rows, labels, secondary=True),
"role_flagged_by_new_verdict": {v: {"n": sum(r["judge"]["verdict"] == v for r in rows),
"flagged": sum(r["judge"]["verdict"] == v and r["role"]["flagged"] for r in rows)}
for v in sorted(VERDICTS)},
}
args.private_out.mkdir(parents=True, exist_ok=True)
(args.private_out / "results.jsonl").write_text(''.join(json.dumps(r, ensure_ascii=False) + '\n' for r in rows))
write_review(args.private_out / "review.html", rows, items)
args.summary.parent.mkdir(parents=True, exist_ok=True)
args.public_items.parent.mkdir(parents=True, exist_ok=True)
args.summary.write_text(json.dumps(finite(summary), indent=2, allow_nan=False) + '\n')
args.public_items.write_text(json.dumps(public, indent=2, allow_nan=False) + '\n')
print(json.dumps({k: summary[k] for k in ("n", "status", "verdict_counts", "role_flagged", "exact_verdict_agreement", "decision")}))
if __name__ == "__main__":
main()
experiments/P0-pilot/results/gpt-diagnostic-items.json
+1253
View File
File diff suppressed because it is too large Load Diff
experiments/P0-pilot/results/gpt-diagnostic-summary.json
+275
View File
@@ -0,0 +1,275 @@
{
"experiment": "P0-GPT-diagnostic",
"status": "complete",
"n": 65,
"provenance": {
"run_kind": "current-chat assistant-authored diagnostic; not an API benchmark",
"evaluator": "Codex GPT assistant; exact deployed model identifier unavailable",
"authored_at": "2026-10-28T11:23:06.275167+00:00",
"items_sha256": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
"key_sha256": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
"reference_labels_seen_in_prior_chat": true,
"lean_seen_while_authoring_source_roles": true,
"new_external_api_calls": 0,
"assistant_token_usage": null,
"limitations": [
"Same assistant previously authored the reference labels; agreement is not independent accuracy.",
"Prior context contains the earlier review and detector code. This is not a blinded source-only extraction or a fresh-model replicate.",
"All judgments are manually authored in this chat, not responses from a GPT API endpoint.",
"Semantics are assessed conditionally on recoverable notation. No new Lean compilation or library API audit is performed.",
"The strict Boolean judge flags all verdicts except faithful, including not_formalizable; four-way verdicts are also retained."
]
},
"decision": null,
"decision_withheld_reason": "Same-assistant, non-blind diagnostic; not the preregistered Qwen pilot or independent ground-truth validation.",
"hashes": {
"items": "487ecd22919a5e3b0174a2846ab8fe7c0daf4d3feccccf8ea6571aec04c41bed",
"key": "f827cac92ee4e907554fadcc7c0e006ef2b829350f74e7f8b54ca11945e62836",
"assessments": "f12735802ba0d4d94b776a91581a13a508eab0ce90ba8973a6c9c4e4d1390272",
"reference_labels": "1552dba85e75426799151384e7e28c9a0b1072ed4c862df0876df89920d2a638",
"replay_script": "5c02e3d26761ce0b4e38d4be9e65b95bfd8db048c9c7c73edaac17e654e10c0f",
"leanbench/align.py": "730e44f1158e6e441e4745dff7324e565b89b6e5148017c3fd93f4c69c22c9ce",
"leanbench/project.py": "8ed76924b463ba418ceb732d0fcb6c572a548d94fe7f23759c67746da24d9430",
"leanbench/srcsimplex.py": "d10b785299ceb17cadc4e97737eef74183ed390d04b4571a702cd4fd42b069d0",
"leanbench/lint.py": "4396e879312f370cd70beeffcbd7ffeab8a41ee1704cc1f235dbbcd6e843fd68"
},
"fixed_thresholds": {
"matching": 0.3,
"goal": 0.2
},
"verdict_counts": {
"faithful": 25,
"minor_drift": 13,
"wrong": 24,
"not_formalizable": 3
},
"role_flagged": 55,
"lint_flagged": 21,
"compiled": 10,
"degenerate_projection": 15,
"exact_verdict_agreement": 59,
"changed_verdicts": [
"P02",
"P18",
"P23",
"P34",
"P51",
"P57"
],
"primary_reference_agreement": {
"n": 55,
"metrics_are_non_independent_reference_agreement": true,
"detectors": {
"role": {
"tp": 25,
"fp": 21,
"fn": 0,
"tn": 9,
"n": 55,
"recall": 1.0,
"precision": 0.5434782608695652,
"false_flag_rate": 0.7,
"f1": 0.704225352112676,
"recall_ci": [
0.8668035060468213,
1.0
],
"false_flag_ci": [
0.5212387898833981,
0.8333543735604105
],
"precision_ci": [
0.4017993496608042,
0.67845489578878
],
"auc": 0.61
},
"assistant_judge": {
"tp": 25,
"fp": 5,
"fn": 0,
"tn": 25,
"n": 55,
"recall": 1.0,
"precision": 0.8333333333333334,
"false_flag_rate": 0.16666666666666666,
"f1": 0.9090909090909091,
"recall_ci": [
0.8668035060468213,
1.0
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.6643529481431982,
0.9266356575964831
]
},
"lint": {
"tp": 12,
"fp": 5,
"fn": 13,
"tn": 25,
"n": 55,
"recall": 0.48,
"precision": 0.7058823529411765,
"false_flag_rate": 0.16666666666666666,
"f1": 0.5714285714285713,
"recall_ci": [
0.30031003928960015,
0.6650178204685269
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.468664666737861,
0.8672020613444361
]
},
"compile_only": {
"tp": 24,
"fp": 24,
"fn": 1,
"tn": 6,
"n": 55,
"recall": 0.96,
"precision": 0.5,
"false_flag_rate": 0.8,
"f1": 0.6575342465753424,
"recall_ci": [
0.8045552759497747,
0.9929039496133008
],
"false_flag_ci": [
0.6269395261897256,
0.9049502189759876
],
"precision_ci": [
0.36389098932037767,
0.6361090106796223
]
}
}
},
"secondary_reference_agreement": {
"n": 62,
"metrics_are_non_independent_reference_agreement": true,
"detectors": {
"role": {
"tp": 32,
"fp": 21,
"fn": 0,
"tn": 9,
"n": 62,
"recall": 1.0,
"precision": 0.6037735849056604,
"false_flag_rate": 0.7,
"f1": 0.7529411764705882,
"recall_ci": [
0.8928172849426366,
1.0
],
"false_flag_ci": [
0.5212387898833981,
0.8333543735604105
],
"precision_ci": [
0.4694123578888073,
0.7241078808096105
],
"auc": 0.590625
},
"assistant_judge": {
"tp": 32,
"fp": 5,
"fn": 0,
"tn": 25,
"n": 62,
"recall": 1.0,
"precision": 0.8648648648648649,
"false_flag_rate": 0.16666666666666666,
"f1": 0.927536231884058,
"recall_ci": [
0.8928172849426366,
1.0
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.7202227252929444,
0.9408679273749238
]
},
"lint": {
"tp": 14,
"fp": 5,
"fn": 18,
"tn": 25,
"n": 62,
"recall": 0.4375,
"precision": 0.7368421052631579,
"false_flag_rate": 0.16666666666666666,
"f1": 0.5490196078431372,
"recall_ci": [
0.2816508596944317,
0.6067469796877387
],
"false_flag_ci": [
0.07336434240351683,
0.33564705185680177
],
"precision_ci": [
0.5120803155892935,
0.8819376166045982
]
},
"compile_only": {
"tp": 29,
"fp": 24,
"fn": 3,
"tn": 6,
"n": 62,
"recall": 0.90625,
"precision": 0.5471698113207547,
"false_flag_rate": 0.8,
"f1": 0.6823529411764705,
"recall_ci": [
0.7578150066422116,
0.9675990373736805
],
"false_flag_ci": [
0.6269395261897256,
0.9049502189759876
],
"precision_ci": [
0.41453742646439184,
0.6734263183985253
]
}
}
},
"role_flagged_by_new_verdict": {
"faithful": {
"n": 25,
"flagged": 16
},
"minor_drift": {
"n": 13,
"flagged": 13
},
"not_formalizable": {
"n": 3,
"flagged": 2
},
"wrong": {
"n": 24,
"flagged": 24
}
}
}