"""Score the pilot: detectors vs. the blind human labels, with the pre-registered decision.

    python experiments/pilot/score.py --results runs/pilot/results.jsonl \
        --labels runs/pilot/private/labels.json --key runs/pilot/private/key.json

Positive class = unfaithful. Primary: verdict == "wrong", over ALL labeled items (role alignment and
the judge do not need Lean to accept the code; only 10 of the 65 pilot items compiled, too few to
decide on). Secondary: wrong or minor_drift; and the compiled-only subset (the paper's claim, reported
with its interval, not used to decide). "not_formalizable" items are excluded. Decision (fixed before
looking at labels):

  GO       role recall on wrong items >= 0.5 AND false-flag rate on faithful items <= 0.2
           AND role recall >= judge recall + 0.10 at a false-flag rate no higher than the judge's
  PARTIAL  role recall >= 0.4 with false-flag <= 0.3, or role complements the judge
           (flags >= 3 wrong items the judge misses)
  NO-GO    otherwise (re-scope to evaluation + taxonomy + resource)
With ~65 items every number carries a wide Wilson interval; the pilot checks for signal, not effect size.
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

from leanbench.evalstats import auc, confusion


def load(args):
    results = {json.loads(x)["id"]: json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()}
    labels = json.loads(args.labels.read_text(encoding="utf-8"))["labels"]
    key = json.loads(args.key.read_text(encoding="utf-8"))["key"]
    rows = []
    for blind, item_id in key.items():
        lab = labels.get(blind) or {}
        if item_id in results and lab.get("verdict"):
            rows.append({**results[item_id], "blind": blind, "verdict": lab["verdict"],
                         "human_errors": lab.get("errors", [])})
    return rows


def flags(row):
    return {
        "compile_only": not row["compiled"],
        "lint": bool(row.get("lint")),
        "role": bool(row.get("role", {}).get("flagged")),
        "judge": (not row["judge"]["faithful"]) if "judge" in row else False,
    }


def report(rows, positive):
    out = {}
    for name in ("compile_only", "lint", "role", "judge"):
        out[name] = confusion([flags(r)[name] for r in rows], [positive(r) for r in rows]).as_dict()
    out["role"]["auc_score"] = auc([-(r.get("role", {}).get("score", 1.0)) for r in rows],
                                   [positive(r) for r in rows])
    return out


def decide(sub):
    role, judge = sub["role"], sub["judge"]
    if role["tp"] + role["fn"] == 0:
        return "INSUFFICIENT: no wrong items in the labels"
    go = (role["recall"] >= 0.5 and role["false_flag_rate"] <= 0.2
          and role["recall"] >= judge["recall"] + 0.10
          and role["false_flag_rate"] <= judge["false_flag_rate"])
    partial = (role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
    return "GO" if go else "PARTIAL" if partial else "NO-GO"


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
    ap.add_argument("--results", required=True, type=Path)
    ap.add_argument("--labels", required=True, type=Path)
    ap.add_argument("--key", required=True, type=Path)
    args = ap.parse_args()
    rows = [r for r in load(args) if r["verdict"] != "not_formalizable"]
    compiled = [r for r in rows if r["compiled"]]
    summary = {
        "n_labeled": len(rows), "n_compiled": len(compiled),
        "verdicts": {v: sum(r["verdict"] == v for r in rows) for v in
                     ("faithful", "minor_drift", "wrong")},
        "compiled_but_wrong": sum(r["verdict"] == "wrong" for r in compiled),
        "primary_all": report(rows, lambda r: r["verdict"] == "wrong"),
        "secondary_all": report(rows, lambda r: r["verdict"] in ("wrong", "minor_drift")),
        "compiled_only": report(compiled, lambda r: r["verdict"] == "wrong"),
    }
    summary["decision"] = decide(summary["primary_all"])
    print(json.dumps(summary, indent=2, default=str))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())