fix(pilot): freeze AI labels and enforce auditable execution
This commit is contained in:
@@ -1,9 +1,9 @@
|
||||
"""Score the pilot: detectors vs. the blind human labels, with the pre-registered decision.
|
||||
"""Score the pilot against frozen blind labels using the pre-registered decision.
|
||||
|
||||
python experiments/P0-pilot/score.py --results experiments/P0-pilot/data/results.jsonl \
|
||||
--labels experiments/P0-pilot/labels/labels.json --key experiments/P0-pilot/labels/key.json --summary experiments/P0-pilot/results/summary.json
|
||||
|
||||
Positive class = unfaithful. Primary: verdict == "wrong", over ALL labeled items (role alignment and
|
||||
Positive class = unfaithful. Primary: wrong versus faithful, excluding minor drift (role alignment and
|
||||
the judge do not need Lean to accept the code; only 10 of the 65 pilot items compiled, too few to
|
||||
decide on). Secondary: wrong or minor_drift; and the compiled-only subset (the paper's claim, reported
|
||||
with its interval, not used to decide). "not_formalizable" items are excluded. Decision (fixed before
|
||||
@@ -19,22 +19,35 @@ With ~65 items every number carries a wide Wilson interval; the pilot checks for
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
from leanbench.evalstats import auc, confusion
|
||||
from leanbench.evalstats import auc, confusion, wilson
|
||||
|
||||
|
||||
def load(args):
|
||||
results = {json.loads(x)["id"]: json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()}
|
||||
result_rows = [json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()]
|
||||
results = {row["id"]: row for row in result_rows}
|
||||
labels = json.loads(args.labels.read_text(encoding="utf-8"))["labels"]
|
||||
key = json.loads(args.key.read_text(encoding="utf-8"))["key"]
|
||||
if not key or len(set(key.values())) != len(key):
|
||||
raise ValueError("The blind key must contain unique item IDs")
|
||||
if len(results) != len(result_rows) or set(results) != set(key.values()):
|
||||
raise ValueError("Results must cover every keyed item exactly once")
|
||||
if set(labels) != set(key):
|
||||
raise ValueError("Labels must cover every blind ID exactly once")
|
||||
rows = []
|
||||
for blind, item_id in key.items():
|
||||
lab = labels.get(blind) or {}
|
||||
if item_id in results and lab.get("verdict"):
|
||||
rows.append({**results[item_id], "blind": blind, "verdict": lab["verdict"],
|
||||
"human_errors": lab.get("errors", [])})
|
||||
if lab.get("verdict") not in {"faithful", "minor_drift", "wrong", "not_formalizable"}:
|
||||
raise ValueError(f"Missing or invalid verdict for {blind}")
|
||||
row = results[item_id]
|
||||
if row.get("errors") or "judge" not in row or "role" not in row:
|
||||
raise ValueError(f"Unresolved detector failure for {blind}; no decision can be issued")
|
||||
rows.append({**row, "blind": blind, "verdict": lab["verdict"],
|
||||
"human_errors": lab.get("errors", [])})
|
||||
return rows
|
||||
|
||||
|
||||
@@ -51,19 +64,24 @@ def report(rows, positive):
|
||||
out = {}
|
||||
for name in ("compile_only", "lint", "role", "judge"):
|
||||
out[name] = confusion([flags(r)[name] for r in rows], [positive(r) for r in rows]).as_dict()
|
||||
out[name]["precision_ci"] = wilson(out[name]["tp"], out[name]["tp"] + out[name]["fp"])
|
||||
out["role"]["auc_score"] = auc([-(r.get("role", {}).get("score", 1.0)) for r in rows],
|
||||
[positive(r) for r in rows])
|
||||
out["role"]["complementary_wrong"] = sum(
|
||||
positive(r) and flags(r)["role"] and not flags(r)["judge"] for r in rows
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def decide(sub):
|
||||
role, judge = sub["role"], sub["judge"]
|
||||
if role["tp"] + role["fn"] == 0:
|
||||
return "INSUFFICIENT: no wrong items in the labels"
|
||||
if role["tp"] + role["fn"] == 0 or role["fp"] + role["tn"] == 0:
|
||||
return "INSUFFICIENT: both wrong and faithful items are required"
|
||||
go = (role["recall"] >= 0.5 and role["false_flag_rate"] <= 0.2
|
||||
and role["recall"] >= judge["recall"] + 0.10
|
||||
and role["false_flag_rate"] <= judge["false_flag_rate"])
|
||||
partial = (role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
|
||||
partial = ((role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
|
||||
or role.get("complementary_wrong", 0) >= 3)
|
||||
return "GO" if go else "PARTIAL" if partial else "NO-GO"
|
||||
|
||||
|
||||
@@ -75,19 +93,51 @@ def main() -> int:
|
||||
ap.add_argument("--summary", type=Path, default=None,
|
||||
help="also write the (text-free) summary here, e.g. results/summary.json")
|
||||
args = ap.parse_args()
|
||||
rows = [r for r in load(args) if r["verdict"] != "not_formalizable"]
|
||||
compiled = [r for r in rows if r["compiled"]]
|
||||
meta = json.loads(args.results.with_suffix(".meta.json").read_text(encoding="utf-8"))
|
||||
if (meta.get("status") != "complete"
|
||||
or meta.get("labels_sha256") != hashlib.sha256(args.labels.read_bytes()).hexdigest()
|
||||
or meta.get("results_sha256") != hashlib.sha256(args.results.read_bytes()).hexdigest()):
|
||||
ap.error("Scoring requires a complete run with unchanged frozen labels and results")
|
||||
all_rows = load(args)
|
||||
rows = [r for r in all_rows if r["verdict"] != "not_formalizable"]
|
||||
primary = [r for r in rows if r["verdict"] != "minor_drift"]
|
||||
compiled = [r for r in primary if r["compiled"]]
|
||||
summary = {
|
||||
"n_labeled": len(rows), "n_compiled": len(compiled),
|
||||
"run_metadata": meta,
|
||||
"n_labeled": len(all_rows), "n_evaluable": len(rows), "n_primary": len(primary),
|
||||
"n_compiled": len(compiled),
|
||||
"verdicts": {v: sum(r["verdict"] == v for r in rows) for v in
|
||||
("faithful", "minor_drift", "wrong")},
|
||||
("faithful", "minor_drift", "wrong", "not_formalizable")},
|
||||
"compiled_but_wrong": sum(r["verdict"] == "wrong" for r in compiled),
|
||||
"primary_all": report(rows, lambda r: r["verdict"] == "wrong"),
|
||||
"primary_all": report(primary, lambda r: r["verdict"] == "wrong"),
|
||||
"secondary_all": report(rows, lambda r: r["verdict"] in ("wrong", "minor_drift")),
|
||||
"compiled_only": report(compiled, lambda r: r["verdict"] == "wrong"),
|
||||
}
|
||||
summary["verdicts"]["not_formalizable"] = len(all_rows) - len(rows)
|
||||
summary["role_recall_by_error"] = {
|
||||
error: {"flagged": sum(flags(r)["role"] for r in rows
|
||||
if r["verdict"] == "wrong" and error in r["human_errors"]),
|
||||
"total": sum(r["verdict"] == "wrong" and error in r["human_errors"] for r in rows)}
|
||||
for error in sorted({e for r in rows for e in r["human_errors"]})
|
||||
}
|
||||
summary["degenerate_projection"] = {
|
||||
"total": sum(bool(r.get("projection", {}).get("degenerate")) for r in all_rows),
|
||||
"wrong": sum(r["verdict"] == "wrong" and bool(r.get("projection", {}).get("degenerate"))
|
||||
for r in all_rows),
|
||||
}
|
||||
summary["disagreements"] = [
|
||||
{"blind": r["blind"], "verdict": r["verdict"], "flags": flags(r)}
|
||||
for r in primary if any(f != (r["verdict"] == "wrong") for f in flags(r).values())
|
||||
]
|
||||
summary["decision"] = decide(summary["primary_all"])
|
||||
text = json.dumps(summary, indent=2, default=str)
|
||||
def finite_json(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: finite_json(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [finite_json(v) for v in value]
|
||||
return None if isinstance(value, float) and not math.isfinite(value) else value
|
||||
|
||||
text = json.dumps(finite_json(summary), indent=2, allow_nan=False)
|
||||
if args.summary:
|
||||
args.summary.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.summary.write_text(text + "\n", encoding="utf-8")
|
||||
|
||||
Reference in New Issue
Block a user