185 lines
8.9 KiB
Python
185 lines
8.9 KiB
Python
"""Build the blind labeling sheet for the pilot.
|
||
|
||
python experiments/P0-pilot/make_sheet.py --items experiments/P0-pilot/data/items.jsonl \
|
||
--out experiments/P0-pilot/data --key-dir experiments/P0-pilot/labels --seed 7
|
||
|
||
Writes:
|
||
|
||
<out>/labelsheet.html open in a browser; label every item; press "Download labels.json".
|
||
Contains passages from a copyrighted book, so ``data/`` is gitignored.
|
||
<key-dir>/key.json blind id -> item id. Text-free, so it is tracked in git; save the
|
||
downloaded labels.json next to it.
|
||
|
||
The sheet shows the passage and the Lean code. It deliberately hides whether Lean
|
||
accepted the code, any model output, and any score, so the labels are independent of
|
||
everything being evaluated. Order is shuffled with --seed.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import html
|
||
import json
|
||
import random
|
||
from pathlib import Path
|
||
|
||
FIDELITY = [
|
||
("faithful", "Faithful", "Says what the passage says (notation and encoding choices aside)."),
|
||
("minor_drift", "Minor drift",
|
||
"Right idea, but a small difference a careful reader would object to."),
|
||
("wrong", "Wrong", "States a different or false/vacuous claim, or drops/adds a "
|
||
"hypothesis that changes the meaning."),
|
||
("not_formalizable", "Not formalizable / no claim",
|
||
"The passage is not a precise statement, or the extraction is garbled."),
|
||
]
|
||
ERRORS = [
|
||
("missing_hypothesis", "Missing hypothesis"),
|
||
("extra_hypothesis", "Extra hypothesis"),
|
||
("wrong_type", "Wrong type / structure (e.g. ℕ for ℤ, Group for CommGroup)"),
|
||
("quantifier", "Quantifier error (∀/∃, order)"),
|
||
("wrong_conclusion", "Wrong conclusion"),
|
||
("degenerate", "Vacuous / trivial / degenerate"),
|
||
("hallucinated_api", "Names that do not exist in Mathlib"),
|
||
("other", "Other (say in notes)"),
|
||
]
|
||
|
||
PAGE = """<!doctype html>
|
||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width">
|
||
<title>lean-bench pilot labeling sheet</title>
|
||
<style>
|
||
:root {{ --bg:#fff; --fg:#1b1f24; --mute:#59636e; --line:#d0d7de; --card:#f6f8fa; --accent:#0969da; }}
|
||
@media (prefers-color-scheme: dark) {{ :root {{ --bg:#0d1117; --fg:#e6edf3; --mute:#9198a1;
|
||
--line:#30363d; --card:#161b22; --accent:#4493f8; }} }}
|
||
body {{ margin:0; background:var(--bg); color:var(--fg); font:15px/1.5 system-ui,sans-serif; }}
|
||
main {{ max-width:980px; margin:0 auto; padding:16px; }}
|
||
h1 {{ font-size:20px; }} .mute {{ color:var(--mute); }}
|
||
.card {{ background:var(--card); border:1px solid var(--line); border-radius:8px;
|
||
padding:12px 14px; margin:14px 0; }}
|
||
.card.done {{ border-left:4px solid var(--accent); }}
|
||
pre {{ white-space:pre-wrap; word-break:break-word; margin:6px 0; padding:8px;
|
||
background:var(--bg); border:1px solid var(--line); border-radius:6px;
|
||
font:13px/1.45 ui-monospace,monospace; }}
|
||
label {{ display:block; margin:2px 0; }} .row {{ display:flex; gap:24px; flex-wrap:wrap; }}
|
||
.row > div {{ flex:1 1 300px; }} textarea {{ width:100%; box-sizing:border-box; }}
|
||
button {{ font:inherit; padding:6px 12px; border:1px solid var(--line); border-radius:6px;
|
||
background:var(--card); color:var(--fg); cursor:pointer; }}
|
||
#bar {{ position:sticky; top:0; background:var(--bg); padding:8px 0; border-bottom:1px solid var(--line);
|
||
display:flex; gap:12px; align-items:center; z-index:2; }}
|
||
</style></head><body><main>
|
||
<h1>lean-bench pilot: label {n} items</h1>
|
||
<p class="mute">For each item: <b>if this Lean code compiled, would it state what the passage
|
||
states?</b> Judge the meaning of what is written, not whether Lean accepts it (that is hidden on
|
||
purpose). Notation and harmless encoding choices are fine. Tick the error types that apply
|
||
when the verdict is not "Faithful". Your answers are saved in this browser as you go;
|
||
press <b>Download labels.json</b> when finished and put the file next to key.json.</p>
|
||
<div class="card"><b>Verdicts</b><ul>{legend}</ul></div>
|
||
<div id="bar"><span id="count"></span><button onclick="download()">Download labels.json</button></div>
|
||
{cards}
|
||
<script>
|
||
const IDS = {ids};
|
||
const STORE = "leanbench-pilot-labels-v1";
|
||
let labels = {{}};
|
||
try {{ labels = JSON.parse(localStorage.getItem(STORE) || "{{}}"); }} catch (e) {{ labels = {{}}; }}
|
||
function save() {{ try {{ localStorage.setItem(STORE, JSON.stringify(labels)); }} catch (e) {{}} }}
|
||
function read(id) {{
|
||
const card = document.getElementById(id);
|
||
const verdict = card.querySelector('input[name="v_' + id + '"]:checked');
|
||
const errors = [...card.querySelectorAll('input[name="e_' + id + '"]:checked')].map(x => x.value);
|
||
const notes = card.querySelector("textarea").value;
|
||
labels[id] = {{ verdict: verdict ? verdict.value : null, errors: errors, notes: notes }};
|
||
card.classList.toggle("done", !!verdict);
|
||
save(); count();
|
||
}}
|
||
function count() {{
|
||
const done = IDS.filter(i => labels[i] && labels[i].verdict).length;
|
||
document.getElementById("count").textContent = done + " / " + IDS.length + " labeled";
|
||
}}
|
||
function restore() {{
|
||
for (const id of IDS) {{
|
||
const l = labels[id]; if (!l) continue;
|
||
const card = document.getElementById(id);
|
||
if (l.verdict) {{ const r = card.querySelector('input[name="v_' + id + '"][value="' + l.verdict + '"]');
|
||
if (r) r.checked = true; card.classList.add("done"); }}
|
||
for (const e of l.errors || []) {{ const c = card.querySelector('input[name="e_' + id + '"][value="' + e + '"]');
|
||
if (c) c.checked = true; }}
|
||
card.querySelector("textarea").value = l.notes || "";
|
||
}}
|
||
count();
|
||
}}
|
||
function download() {{
|
||
IDS.forEach(i => {{ if (!labels[i]) labels[i] = {{ verdict: null, errors: [], notes: "" }}; }});
|
||
const blob = new Blob([JSON.stringify({{ labeler: "", labels: labels }}, null, 2)], {{ type: "application/json" }});
|
||
const a = document.createElement("a"); a.href = URL.createObjectURL(blob);
|
||
a.download = "labels.json"; a.click();
|
||
}}
|
||
document.querySelectorAll(".card[id]").forEach(c => c.addEventListener("input", () => read(c.id)));
|
||
restore();
|
||
</script></main></body></html>
|
||
"""
|
||
|
||
|
||
def esc(text: str) -> str:
|
||
return html.escape(text or "", quote=True)
|
||
|
||
|
||
def card(blind: str, item: dict) -> str:
|
||
verdicts = "".join(
|
||
f'<label><input type="radio" name="v_{blind}" value="{key}"> {esc(name)}</label>'
|
||
for key, name, _ in FIDELITY
|
||
)
|
||
errors = "".join(
|
||
f'<label><input type="checkbox" name="e_{blind}" value="{key}"> {esc(name)}</label>'
|
||
for key, name in ERRORS
|
||
)
|
||
label = esc(item.get("label") or "")
|
||
return (
|
||
f'<section class="card" id="{blind}">'
|
||
f'<b>{blind}</b> <span class="mute">· page {item.get("start_page")} · '
|
||
f'{esc(item.get("kind", ""))} · {label}</span>'
|
||
f'<div class="row"><div><div class="mute">Passage</div>'
|
||
f'<pre>{esc(item.get("source_text", ""))}</pre></div>'
|
||
f'<div><div class="mute">Lean</div><pre>{esc(item.get("lean_code", ""))}</pre></div></div>'
|
||
f'<div class="row"><div><b>Verdict</b>{verdicts}</div>'
|
||
f'<div><b>Errors</b> <span class="mute">(if not faithful)</span>{errors}</div></div>'
|
||
f'<textarea rows="2" placeholder="notes (optional)"></textarea></section>'
|
||
)
|
||
|
||
|
||
def main() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||
parser.add_argument("--items", required=True, type=Path)
|
||
parser.add_argument("--out", required=True, type=Path, help="where labelsheet.html goes (ignored by git)")
|
||
parser.add_argument("--key-dir", type=Path, default=None, help="where key.json goes (default: ../labels)")
|
||
parser.add_argument("--seed", type=int, default=7)
|
||
args = parser.parse_args()
|
||
|
||
items = [json.loads(line) for line in args.items.read_text(encoding="utf-8").splitlines()
|
||
if line.strip()]
|
||
if not items or any(not item.get("source_text", "").strip() for item in items):
|
||
parser.error("Every item needs nonempty source_text; use a private source-inclusive export.")
|
||
if len({item["id"] for item in items}) != len(items):
|
||
parser.error("Item IDs must be unique.")
|
||
order = list(range(len(items)))
|
||
random.Random(args.seed).shuffle(order)
|
||
blind_ids = [f"P{n:02d}" for n in range(1, len(items) + 1)]
|
||
key = {blind_ids[pos]: items[idx]["id"] for pos, idx in enumerate(order)}
|
||
|
||
legend = "".join(f"<li><b>{esc(n)}</b>: {esc(d)}</li>" for _, n, d in FIDELITY)
|
||
cards = "\n".join(card(blind_ids[pos], items[idx]) for pos, idx in enumerate(order))
|
||
key_dir = args.key_dir or args.out.parent / "labels"
|
||
args.out.mkdir(parents=True, exist_ok=True)
|
||
key_dir.mkdir(parents=True, exist_ok=True)
|
||
(args.out / "labelsheet.html").write_text(
|
||
PAGE.format(n=len(items), legend=legend, cards=cards, ids=json.dumps(blind_ids)),
|
||
encoding="utf-8",
|
||
)
|
||
(key_dir / "key.json").write_text(
|
||
json.dumps({"seed": args.seed, "items_file": args.items.name, "key": key}, indent=2),
|
||
encoding="utf-8",
|
||
)
|
||
print(f"wrote {args.out/'labelsheet.html'} ({len(items)} items) and {key_dir/'key.json'}")
|
||
return 0
|
||
|
||
|
||
if __name__ == "__main__":
|
||
raise SystemExit(main())
|