claude-bot
2b255a333a
Raw pilot data stays ignored (copyrighted text); blind key and numeric results are tracked. Runner notes move into docs/infrastructure. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6
100 lines
4.5 KiB
Python
100 lines
4.5 KiB
Python
"""Score the pilot: detectors vs. the blind human labels, with the pre-registered decision.
|
|
|
|
python experiments/P0-pilot/score.py --results experiments/P0-pilot/data/results.jsonl \
|
|
--labels experiments/P0-pilot/labels/labels.json --key experiments/P0-pilot/labels/key.json --summary experiments/P0-pilot/results/summary.json
|
|
|
|
Positive class = unfaithful. Primary: verdict == "wrong", over ALL labeled items (role alignment and
|
|
the judge do not need Lean to accept the code; only 10 of the 65 pilot items compiled, too few to
|
|
decide on). Secondary: wrong or minor_drift; and the compiled-only subset (the paper's claim, reported
|
|
with its interval, not used to decide). "not_formalizable" items are excluded. Decision (fixed before
|
|
looking at labels):
|
|
|
|
GO role recall on wrong items >= 0.5 AND false-flag rate on faithful items <= 0.2
|
|
AND role recall >= judge recall + 0.10 at a false-flag rate no higher than the judge's
|
|
PARTIAL role recall >= 0.4 with false-flag <= 0.3, or role complements the judge
|
|
(flags >= 3 wrong items the judge misses)
|
|
NO-GO otherwise (re-scope to evaluation + taxonomy + resource)
|
|
With ~65 items every number carries a wide Wilson interval; the pilot checks for signal, not effect size.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from leanbench.evalstats import auc, confusion
|
|
|
|
|
|
def load(args):
|
|
results = {json.loads(x)["id"]: json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()}
|
|
labels = json.loads(args.labels.read_text(encoding="utf-8"))["labels"]
|
|
key = json.loads(args.key.read_text(encoding="utf-8"))["key"]
|
|
rows = []
|
|
for blind, item_id in key.items():
|
|
lab = labels.get(blind) or {}
|
|
if item_id in results and lab.get("verdict"):
|
|
rows.append({**results[item_id], "blind": blind, "verdict": lab["verdict"],
|
|
"human_errors": lab.get("errors", [])})
|
|
return rows
|
|
|
|
|
|
def flags(row):
|
|
return {
|
|
"compile_only": not row["compiled"],
|
|
"lint": bool(row.get("lint")),
|
|
"role": bool(row.get("role", {}).get("flagged")),
|
|
"judge": (not row["judge"]["faithful"]) if "judge" in row else False,
|
|
}
|
|
|
|
|
|
def report(rows, positive):
|
|
out = {}
|
|
for name in ("compile_only", "lint", "role", "judge"):
|
|
out[name] = confusion([flags(r)[name] for r in rows], [positive(r) for r in rows]).as_dict()
|
|
out["role"]["auc_score"] = auc([-(r.get("role", {}).get("score", 1.0)) for r in rows],
|
|
[positive(r) for r in rows])
|
|
return out
|
|
|
|
|
|
def decide(sub):
|
|
role, judge = sub["role"], sub["judge"]
|
|
if role["tp"] + role["fn"] == 0:
|
|
return "INSUFFICIENT: no wrong items in the labels"
|
|
go = (role["recall"] >= 0.5 and role["false_flag_rate"] <= 0.2
|
|
and role["recall"] >= judge["recall"] + 0.10
|
|
and role["false_flag_rate"] <= judge["false_flag_rate"])
|
|
partial = (role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
|
|
return "GO" if go else "PARTIAL" if partial else "NO-GO"
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--results", required=True, type=Path)
|
|
ap.add_argument("--labels", required=True, type=Path)
|
|
ap.add_argument("--key", required=True, type=Path)
|
|
ap.add_argument("--summary", type=Path, default=None,
|
|
help="also write the (text-free) summary here, e.g. results/summary.json")
|
|
args = ap.parse_args()
|
|
rows = [r for r in load(args) if r["verdict"] != "not_formalizable"]
|
|
compiled = [r for r in rows if r["compiled"]]
|
|
summary = {
|
|
"n_labeled": len(rows), "n_compiled": len(compiled),
|
|
"verdicts": {v: sum(r["verdict"] == v for r in rows) for v in
|
|
("faithful", "minor_drift", "wrong")},
|
|
"compiled_but_wrong": sum(r["verdict"] == "wrong" for r in compiled),
|
|
"primary_all": report(rows, lambda r: r["verdict"] == "wrong"),
|
|
"secondary_all": report(rows, lambda r: r["verdict"] in ("wrong", "minor_drift")),
|
|
"compiled_only": report(compiled, lambda r: r["verdict"] == "wrong"),
|
|
}
|
|
summary["decision"] = decide(summary["primary_all"])
|
|
text = json.dumps(summary, indent=2, default=str)
|
|
if args.summary:
|
|
args.summary.parent.mkdir(parents=True, exist_ok=True)
|
|
args.summary.write_text(text + "\n", encoding="utf-8")
|
|
print(text)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|