Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6

experiments/pilot/score.py

@@ -0,0 +1,93 @@
+"""Score the pilot: detectors vs. the blind human labels, with the pre-registered decision.
+
+    python experiments/pilot/score.py --results runs/pilot/results.jsonl \
+        --labels runs/pilot/private/labels.json --key runs/pilot/private/key.json
+
+Positive class = unfaithful. Primary: verdict == "wrong", over ALL labeled items (role alignment and
+the judge do not need Lean to accept the code; only 10 of the 65 pilot items compiled, too few to
+decide on). Secondary: wrong or minor_drift; and the compiled-only subset (the paper's claim, reported
+with its interval, not used to decide). "not_formalizable" items are excluded. Decision (fixed before
+looking at labels):
+
+  GO       role recall on wrong items >= 0.5 AND false-flag rate on faithful items <= 0.2
+           AND role recall >= judge recall + 0.10 at a false-flag rate no higher than the judge's
+  PARTIAL  role recall >= 0.4 with false-flag <= 0.3, or role complements the judge
+           (flags >= 3 wrong items the judge misses)
+  NO-GO    otherwise (re-scope to evaluation + taxonomy + resource)
+With ~65 items every number carries a wide Wilson interval; the pilot checks for signal, not effect size.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from leanbench.evalstats import auc, confusion
+
+
+def load(args):
+    results = {json.loads(x)["id"]: json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()}
+    labels = json.loads(args.labels.read_text(encoding="utf-8"))["labels"]
+    key = json.loads(args.key.read_text(encoding="utf-8"))["key"]
+    rows = []
+    for blind, item_id in key.items():
+        lab = labels.get(blind) or {}
+        if item_id in results and lab.get("verdict"):
+            rows.append({**results[item_id], "blind": blind, "verdict": lab["verdict"],
+                         "human_errors": lab.get("errors", [])})
+    return rows
+
+
+def flags(row):
+    return {
+        "compile_only": not row["compiled"],
+        "lint": bool(row.get("lint")),
+        "role": bool(row.get("role", {}).get("flagged")),
+        "judge": (not row["judge"]["faithful"]) if "judge" in row else False,
+    }
+
+
+def report(rows, positive):
+    out = {}
+    for name in ("compile_only", "lint", "role", "judge"):
+        out[name] = confusion([flags(r)[name] for r in rows], [positive(r) for r in rows]).as_dict()
+    out["role"]["auc_score"] = auc([-(r.get("role", {}).get("score", 1.0)) for r in rows],
+                                   [positive(r) for r in rows])
+    return out
+
+
+def decide(sub):
+    role, judge = sub["role"], sub["judge"]
+    if role["tp"] + role["fn"] == 0:
+        return "INSUFFICIENT: no wrong items in the labels"
+    go = (role["recall"] >= 0.5 and role["false_flag_rate"] <= 0.2
+          and role["recall"] >= judge["recall"] + 0.10
+          and role["false_flag_rate"] <= judge["false_flag_rate"])
+    partial = (role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
+    return "GO" if go else "PARTIAL" if partial else "NO-GO"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--results", required=True, type=Path)
+    ap.add_argument("--labels", required=True, type=Path)
+    ap.add_argument("--key", required=True, type=Path)
+    args = ap.parse_args()
+    rows = [r for r in load(args) if r["verdict"] != "not_formalizable"]
+    compiled = [r for r in rows if r["compiled"]]
+    summary = {
+        "n_labeled": len(rows), "n_compiled": len(compiled),
+        "verdicts": {v: sum(r["verdict"] == v for r in rows) for v in
+                     ("faithful", "minor_drift", "wrong")},
+        "compiled_but_wrong": sum(r["verdict"] == "wrong" for r in compiled),
+        "primary_all": report(rows, lambda r: r["verdict"] == "wrong"),
+        "secondary_all": report(rows, lambda r: r["verdict"] in ("wrong", "minor_drift")),
+        "compiled_only": report(compiled, lambda r: r["verdict"] == "wrong"),
+    }
+    summary["decision"] = decide(summary["primary_all"])
+    print(json.dumps(summary, indent=2, default=str))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())