Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6

experiments/pilot/README.md

@@ -0,0 +1,15 @@
+# Pilot (Phase 0)
+
+Question: on the 65 existing items, does role alignment (passage simplex vs. Lean simplex) find
+unfaithful formalizations better than a single LLM judge, compile-only, or static lint?
+Full protocol and decision rule: `docs/experiments/PROTOCOL.md`.
+
+    python experiments/pilot/make_sheet.py --items runs/p85-110/items.jsonl --out runs/pilot/private
+    # a person opens runs/pilot/private/labelsheet.html, labels all items, saves labels.json
+    python experiments/pilot/run_pilot.py --items runs/p85-110/items.jsonl --out runs/pilot/results.jsonl --dry-run
+    python experiments/pilot/run_pilot.py --items runs/p85-110/items.jsonl --out runs/pilot/results.jsonl
+    python experiments/pilot/score.py --results runs/pilot/results.jsonl \
+        --labels runs/pilot/private/labels.json --key runs/pilot/private/key.json
+
+`runs/` and `**/private/` are gitignored: the passages come from a copyrighted book and must not be
+committed. Labeling must finish BEFORE `score.py` is run; do not look at `results.jsonl` while labeling.

experiments/pilot/make_sheet.py

@@ -0,0 +1,176 @@
+"""Build the blind labeling sheet for the pilot.
+
+    python experiments/pilot/make_sheet.py --items runs/p85-110/items.jsonl \
+        --out runs/pilot/private --seed 7
+
+Writes, into a directory that must stay local (it contains passages from a
+copyrighted book; ``runs/`` and ``**/private/`` are gitignored):
+
+    labelsheet.html     open in a browser; label every item; press "Download labels.json"
+    key.json            blind id -> item id (keep it; the scorer needs it)
+
+The sheet shows the passage and the Lean code. It deliberately hides whether Lean
+accepted the code, any model output, and any score, so the labels are independent of
+everything being evaluated. Order is shuffled with --seed.
+"""
+from __future__ import annotations
+
+import argparse
+import html
+import json
+import random
+from pathlib import Path
+
+FIDELITY = [
+    ("faithful", "Faithful", "Says what the passage says (notation and encoding choices aside)."),
+    ("minor_drift", "Minor drift",
+     "Right idea, but a small difference a careful reader would object to."),
+    ("wrong", "Wrong", "States a different or false/vacuous claim, or drops/adds a "
+     "hypothesis that changes the meaning."),
+    ("not_formalizable", "Not formalizable / no claim",
+     "The passage is not a precise statement, or the extraction is garbled."),
+]
+ERRORS = [
+    ("missing_hypothesis", "Missing hypothesis"),
+    ("extra_hypothesis", "Extra hypothesis"),
+    ("wrong_type", "Wrong type / structure (e.g. ℕ for ℤ, Group for CommGroup)"),
+    ("quantifier", "Quantifier error (∀/∃, order)"),
+    ("wrong_conclusion", "Wrong conclusion"),
+    ("degenerate", "Vacuous / trivial / degenerate"),
+    ("hallucinated_api", "Names that do not exist in Mathlib"),
+    ("other", "Other (say in notes)"),
+]
+
+PAGE = """<!doctype html>
+<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
+<title>lean-bench pilot labeling sheet</title>
+<style>
+:root {{ --bg:#fff; --fg:#1b1f24; --mute:#59636e; --line:#d0d7de; --card:#f6f8fa; --accent:#0969da; }}
+@media (prefers-color-scheme: dark) {{ :root {{ --bg:#0d1117; --fg:#e6edf3; --mute:#9198a1;
+  --line:#30363d; --card:#161b22; --accent:#4493f8; }} }}
+body {{ margin:0; background:var(--bg); color:var(--fg); font:15px/1.5 system-ui,sans-serif; }}
+main {{ max-width:980px; margin:0 auto; padding:16px; }}
+h1 {{ font-size:20px; }} .mute {{ color:var(--mute); }}
+.card {{ background:var(--card); border:1px solid var(--line); border-radius:8px;
+  padding:12px 14px; margin:14px 0; }}
+.card.done {{ border-left:4px solid var(--accent); }}
+pre {{ white-space:pre-wrap; word-break:break-word; margin:6px 0; padding:8px;
+  background:var(--bg); border:1px solid var(--line); border-radius:6px;
+  font:13px/1.45 ui-monospace,monospace; }}
+label {{ display:block; margin:2px 0; }} .row {{ display:flex; gap:24px; flex-wrap:wrap; }}
+.row > div {{ flex:1 1 300px; }} textarea {{ width:100%; box-sizing:border-box; }}
+button {{ font:inherit; padding:6px 12px; border:1px solid var(--line); border-radius:6px;
+  background:var(--card); color:var(--fg); cursor:pointer; }}
+#bar {{ position:sticky; top:0; background:var(--bg); padding:8px 0; border-bottom:1px solid var(--line);
+  display:flex; gap:12px; align-items:center; z-index:2; }}
+</style></head><body><main>
+<h1>lean-bench pilot: label {n} items</h1>
+<p class="mute">For each item: <b>if this Lean code compiled, would it state what the passage
+states?</b> Judge the meaning of what is written, not whether Lean accepts it (that is hidden on
+purpose). Notation and harmless encoding choices are fine. Tick the error types that apply
+when the verdict is not "Faithful". Your answers are saved in this browser as you go;
+press <b>Download labels.json</b> when finished and put the file next to key.json.</p>
+<div class="card"><b>Verdicts</b><ul>{legend}</ul></div>
+<div id="bar"><span id="count"></span><button onclick="download()">Download labels.json</button></div>
+{cards}
+<script>
+const IDS = {ids};
+const STORE = "leanbench-pilot-labels-v1";
+let labels = {{}};
+try {{ labels = JSON.parse(localStorage.getItem(STORE) || "{{}}"); }} catch (e) {{ labels = {{}}; }}
+function save() {{ try {{ localStorage.setItem(STORE, JSON.stringify(labels)); }} catch (e) {{}} }}
+function read(id) {{
+  const card = document.getElementById(id);
+  const verdict = card.querySelector('input[name="v_' + id + '"]:checked');
+  const errors = [...card.querySelectorAll('input[name="e_' + id + '"]:checked')].map(x => x.value);
+  const notes = card.querySelector("textarea").value;
+  labels[id] = {{ verdict: verdict ? verdict.value : null, errors: errors, notes: notes }};
+  card.classList.toggle("done", !!verdict);
+  save(); count();
+}}
+function count() {{
+  const done = IDS.filter(i => labels[i] && labels[i].verdict).length;
+  document.getElementById("count").textContent = done + " / " + IDS.length + " labeled";
+}}
+function restore() {{
+  for (const id of IDS) {{
+    const l = labels[id]; if (!l) continue;
+    const card = document.getElementById(id);
+    if (l.verdict) {{ const r = card.querySelector('input[name="v_' + id + '"][value="' + l.verdict + '"]');
+      if (r) r.checked = true; card.classList.add("done"); }}
+    for (const e of l.errors || []) {{ const c = card.querySelector('input[name="e_' + id + '"][value="' + e + '"]');
+      if (c) c.checked = true; }}
+    card.querySelector("textarea").value = l.notes || "";
+  }}
+  count();
+}}
+function download() {{
+  IDS.forEach(i => {{ if (!labels[i]) labels[i] = {{ verdict: null, errors: [], notes: "" }}; }});
+  const blob = new Blob([JSON.stringify({{ labeler: "", labels: labels }}, null, 2)], {{ type: "application/json" }});
+  const a = document.createElement("a"); a.href = URL.createObjectURL(blob);
+  a.download = "labels.json"; a.click();
+}}
+document.querySelectorAll(".card[id]").forEach(c => c.addEventListener("input", () => read(c.id)));
+restore();
+</script></main></body></html>
+"""
+
+
+def esc(text: str) -> str:
+    return html.escape(text or "", quote=True)
+
+
+def card(blind: str, item: dict) -> str:
+    verdicts = "".join(
+        f'<label><input type="radio" name="v_{blind}" value="{key}"> {esc(name)}</label>'
+        for key, name, _ in FIDELITY
+    )
+    errors = "".join(
+        f'<label><input type="checkbox" name="e_{blind}" value="{key}"> {esc(name)}</label>'
+        for key, name in ERRORS
+    )
+    label = esc(item.get("label") or "")
+    return (
+        f'<section class="card" id="{blind}">'
+        f'<b>{blind}</b> <span class="mute">· page {item.get("start_page")} · '
+        f'{esc(item.get("kind", ""))} · {label}</span>'
+        f'<div class="row"><div><div class="mute">Passage</div>'
+        f'<pre>{esc(item.get("source_text", ""))}</pre></div>'
+        f'<div><div class="mute">Lean</div><pre>{esc(item.get("lean_code", ""))}</pre></div></div>'
+        f'<div class="row"><div><b>Verdict</b>{verdicts}</div>'
+        f'<div><b>Errors</b> <span class="mute">(if not faithful)</span>{errors}</div></div>'
+        f'<textarea rows="2" placeholder="notes (optional)"></textarea></section>'
+    )
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    parser.add_argument("--items", required=True, type=Path)
+    parser.add_argument("--out", required=True, type=Path)
+    parser.add_argument("--seed", type=int, default=7)
+    args = parser.parse_args()
+
+    items = [json.loads(line) for line in args.items.read_text(encoding="utf-8").splitlines()
+             if line.strip()]
+    order = list(range(len(items)))
+    random.Random(args.seed).shuffle(order)
+    blind_ids = [f"P{n:02d}" for n in range(1, len(items) + 1)]
+    key = {blind_ids[pos]: items[idx]["id"] for pos, idx in enumerate(order)}
+
+    legend = "".join(f"<li><b>{esc(n)}</b>: {esc(d)}</li>" for _, n, d in FIDELITY)
+    cards = "\n".join(card(blind_ids[pos], items[idx]) for pos, idx in enumerate(order))
+    args.out.mkdir(parents=True, exist_ok=True)
+    (args.out / "labelsheet.html").write_text(
+        PAGE.format(n=len(items), legend=legend, cards=cards, ids=json.dumps(blind_ids)),
+        encoding="utf-8",
+    )
+    (args.out / "key.json").write_text(
+        json.dumps({"seed": args.seed, "items_file": args.items.name, "key": key}, indent=2),
+        encoding="utf-8",
+    )
+    print(f"wrote {args.out/'labelsheet.html'} ({len(items)} items) and key.json")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

experiments/pilot/run_pilot.py

@@ -0,0 +1,110 @@
+"""Run the pilot detectors over the pilot items. Resumable; one JSON line per item.
+
+    python experiments/pilot/run_pilot.py --items runs/p85-110/items.jsonl \
+        --out runs/pilot/results.jsonl --dry-run          # no API calls: shows the plan
+    python experiments/pilot/run_pilot.py --items ... --out ... --max-calls 400
+
+Detectors (all see the passage and/or the Lean code, never the human labels):
+  lint    static patterns on the projected simplex          (no LLM)
+  role    source-side simplex (LLM sees only the passage) aligned to the Lean simplex
+  judge   one LLM call that sees passage + Lean and says faithful / not
+  compile the stored Lean verdict (``verified``), read from the items file
+
+Hard stops: --max-calls bounds API calls; a rerun skips items already in --out.
+"""
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+from leanbench import srcsimplex
+from leanbench.align import align
+from leanbench.config import Settings
+from leanbench.lint import lint
+from leanbench.llm import OpenAICompatibleClient
+from leanbench.project import project
+
+JUDGE_PROMPT_VERSION = "judge-v1"
+JUDGE_SYSTEM = (
+    "You check a Lean 4 (Mathlib) formalization against a textbook passage. Decide whether the "
+    "Lean statement says what the passage says: same objects, same hypotheses, same conclusion, "
+    "same quantifiers. Ignore whether it compiles. Reply with JSON only: "
+    '{"faithful": true|false, "confidence": 0..1, "reason": "one sentence"}'
+)
+
+
+def judge(client, passage: str, lean: str) -> dict:
+    reply = client.complete_json([
+        {"role": "system", "content": JUDGE_SYSTEM},
+        {"role": "user", "content": f"Passage:\n{passage}\n\nLean:\n{lean}"},
+    ])
+    if not isinstance(reply, dict) or "faithful" not in reply:
+        raise ValueError(f"unusable judge reply: {str(reply)[:200]}")
+    return {"faithful": bool(reply["faithful"]), "confidence": reply.get("confidence"),
+            "reason": str(reply.get("reason", ""))[:300]}
+
+
+def run_item(client, item: dict) -> dict:
+    code = item.get("lean_code") or ""
+    row: dict = {"id": item["id"], "compiled": bool(item.get("verified")), "errors": {}}
+    proj = project(code)
+    row["projection"] = proj.as_dict()
+    row["lint"] = lint(proj)
+    try:
+        src = srcsimplex.extract_source_simplex(client, item.get("source_text", ""),
+                                                item.get("label", ""))
+        al = align(src, proj)
+        row["source"] = [r.__dict__ for r in src.roles]
+        row["role"] = al.as_dict()
+    except Exception as error:  # a failed extraction is data, not a crash
+        row["errors"]["role"] = str(error)[:300]
+    try:
+        row["judge"] = judge(client, item.get("source_text", ""), code)
+    except Exception as error:
+        row["errors"]["judge"] = str(error)[:300]
+    return row
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--items", required=True, type=Path)
+    ap.add_argument("--out", required=True, type=Path)
+    ap.add_argument("--max-calls", type=int, default=400)
+    ap.add_argument("--dry-run", action="store_true")
+    args = ap.parse_args()
+
+    items = [json.loads(x) for x in args.items.read_text(encoding="utf-8").splitlines() if x.strip()]
+    done = set()
+    if args.out.exists():
+        done = {json.loads(x)["id"] for x in args.out.read_text(encoding="utf-8").splitlines() if x.strip()}
+    todo = [i for i in items if i["id"] not in done]
+    settings = Settings.from_env()
+    est = len(todo) * 3  # extract (<=2 tries) + judge
+    print(f"items={len(items)} done={len(done)} todo={len(todo)} model={settings.llm.model} "
+          f"est_calls<={est} (extract x2 + judge) cap={args.max_calls}")
+    print("prompts:", srcsimplex.PROMPT_VERSION, JUDGE_PROMPT_VERSION,
+          hashlib.sha256((srcsimplex.SYSTEM_PROMPT + JUDGE_SYSTEM).encode()).hexdigest()[:12])
+    if args.dry_run:
+        return 0
+
+    client = OpenAICompatibleClient.from_settings(settings.llm)
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    with args.out.open("a", encoding="utf-8") as fh:
+        for item in todo:
+            if client.usage.calls >= args.max_calls:
+                print("call cap reached; rerun to resume", file=sys.stderr)
+                break
+            fh.write(json.dumps(run_item(client, item), ensure_ascii=False) + "\n")
+            fh.flush()
+    meta = {"model": settings.llm.model, "usage": client.usage.as_dict(),
+            "prompts": [srcsimplex.PROMPT_VERSION, JUDGE_PROMPT_VERSION]}
+    (args.out.parent / "run_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
+    print(json.dumps(meta))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

experiments/pilot/score.py

@@ -0,0 +1,93 @@
+"""Score the pilot: detectors vs. the blind human labels, with the pre-registered decision.
+
+    python experiments/pilot/score.py --results runs/pilot/results.jsonl \
+        --labels runs/pilot/private/labels.json --key runs/pilot/private/key.json
+
+Positive class = unfaithful. Primary: verdict == "wrong", over ALL labeled items (role alignment and
+the judge do not need Lean to accept the code; only 10 of the 65 pilot items compiled, too few to
+decide on). Secondary: wrong or minor_drift; and the compiled-only subset (the paper's claim, reported
+with its interval, not used to decide). "not_formalizable" items are excluded. Decision (fixed before
+looking at labels):
+
+  GO       role recall on wrong items >= 0.5 AND false-flag rate on faithful items <= 0.2
+           AND role recall >= judge recall + 0.10 at a false-flag rate no higher than the judge's
+  PARTIAL  role recall >= 0.4 with false-flag <= 0.3, or role complements the judge
+           (flags >= 3 wrong items the judge misses)
+  NO-GO    otherwise (re-scope to evaluation + taxonomy + resource)
+With ~65 items every number carries a wide Wilson interval; the pilot checks for signal, not effect size.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from leanbench.evalstats import auc, confusion
+
+
+def load(args):
+    results = {json.loads(x)["id"]: json.loads(x) for x in args.results.read_text(encoding="utf-8").splitlines() if x.strip()}
+    labels = json.loads(args.labels.read_text(encoding="utf-8"))["labels"]
+    key = json.loads(args.key.read_text(encoding="utf-8"))["key"]
+    rows = []
+    for blind, item_id in key.items():
+        lab = labels.get(blind) or {}
+        if item_id in results and lab.get("verdict"):
+            rows.append({**results[item_id], "blind": blind, "verdict": lab["verdict"],
+                         "human_errors": lab.get("errors", [])})
+    return rows
+
+
+def flags(row):
+    return {
+        "compile_only": not row["compiled"],
+        "lint": bool(row.get("lint")),
+        "role": bool(row.get("role", {}).get("flagged")),
+        "judge": (not row["judge"]["faithful"]) if "judge" in row else False,
+    }
+
+
+def report(rows, positive):
+    out = {}
+    for name in ("compile_only", "lint", "role", "judge"):
+        out[name] = confusion([flags(r)[name] for r in rows], [positive(r) for r in rows]).as_dict()
+    out["role"]["auc_score"] = auc([-(r.get("role", {}).get("score", 1.0)) for r in rows],
+                                   [positive(r) for r in rows])
+    return out
+
+
+def decide(sub):
+    role, judge = sub["role"], sub["judge"]
+    if role["tp"] + role["fn"] == 0:
+        return "INSUFFICIENT: no wrong items in the labels"
+    go = (role["recall"] >= 0.5 and role["false_flag_rate"] <= 0.2
+          and role["recall"] >= judge["recall"] + 0.10
+          and role["false_flag_rate"] <= judge["false_flag_rate"])
+    partial = (role["recall"] >= 0.4 and role["false_flag_rate"] <= 0.3)
+    return "GO" if go else "PARTIAL" if partial else "NO-GO"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    ap.add_argument("--results", required=True, type=Path)
+    ap.add_argument("--labels", required=True, type=Path)
+    ap.add_argument("--key", required=True, type=Path)
+    args = ap.parse_args()
+    rows = [r for r in load(args) if r["verdict"] != "not_formalizable"]
+    compiled = [r for r in rows if r["compiled"]]
+    summary = {
+        "n_labeled": len(rows), "n_compiled": len(compiled),
+        "verdicts": {v: sum(r["verdict"] == v for r in rows) for v in
+                     ("faithful", "minor_drift", "wrong")},
+        "compiled_but_wrong": sum(r["verdict"] == "wrong" for r in compiled),
+        "primary_all": report(rows, lambda r: r["verdict"] == "wrong"),
+        "secondary_all": report(rows, lambda r: r["verdict"] in ("wrong", "minor_drift")),
+        "compiled_only": report(compiled, lambda r: r["verdict"] == "wrong"),
+    }
+    summary["decision"] = decide(summary["primary_all"])
+    print(json.dumps(summary, indent=2, default=str))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())