fix(pilot): freeze AI labels and enforce auditable execution
ci / python (pull_request) Successful in 22s
ci / lean (pull_request) Successful in 1m16s
ci / image (pull_request) Has been skipped

This commit is contained in:
2026-10-28 18:46:24 +08:00
parent 21d422fb35
commit 6875c79c40
9 changed files with 826 additions and 50 deletions
+138 -30
View File
@@ -15,8 +15,11 @@ Hard stops: --max-calls bounds API calls; a rerun skips items already in --out.
from __future__ import annotations
import argparse
import fcntl
import hashlib
import inspect
import json
import os
import sys
from pathlib import Path
@@ -24,7 +27,7 @@ from leanbench import srcsimplex
from leanbench.align import align
from leanbench.config import Settings
from leanbench.lint import lint
from leanbench.llm import OpenAICompatibleClient
from leanbench.llm import OpenAICompatibleClient, Usage
from leanbench.project import project
JUDGE_PROMPT_VERSION = "judge-v1"
@@ -41,13 +44,15 @@ def judge(client, passage: str, lean: str) -> dict:
{"role": "system", "content": JUDGE_SYSTEM},
{"role": "user", "content": f"Passage:\n{passage}\n\nLean:\n{lean}"},
])
if not isinstance(reply, dict) or "faithful" not in reply:
if not isinstance(reply, dict) or type(reply.get("faithful")) is not bool:
raise ValueError(f"unusable judge reply: {str(reply)[:200]}")
return {"faithful": bool(reply["faithful"]), "confidence": reply.get("confidence"),
return {"faithful": reply["faithful"], "confidence": reply.get("confidence"),
"reason": str(reply.get("reason", ""))[:300]}
def run_item(client, item: dict) -> dict:
if not item.get("source_text", "").strip():
raise ValueError("Missing source_text; use a private source-inclusive export")
code = item.get("lean_code") or ""
row: dict = {"id": item["id"], "compiled": bool(item.get("verified")), "errors": {}}
proj = project(code)
@@ -59,51 +64,154 @@ def run_item(client, item: dict) -> dict:
al = align(src, proj)
row["source"] = [r.__dict__ for r in src.roles]
row["role"] = al.as_dict()
except CallBudgetExceeded:
raise
except Exception as error: # a failed extraction is data, not a crash
row["errors"]["role"] = str(error)[:300]
try:
row["judge"] = judge(client, item.get("source_text", ""), code)
except CallBudgetExceeded:
raise
except Exception as error:
row["errors"]["judge"] = str(error)[:300]
return row
class CallBudgetExceeded(RuntimeError):
"""Stop before issuing a request that would exceed the cumulative budget."""
def digest(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def atomic_json(path: Path, value: dict) -> None:
temporary = path.with_suffix(path.suffix + ".tmp")
temporary.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n", encoding="utf-8")
os.replace(temporary, path)
class AuditedClient(OpenAICompatibleClient):
"""Persist reservations before HTTP requests, including transport and JSON retries."""
def complete(self, *args, **kwargs):
try:
return super().complete(*args, **kwargs)
finally:
self.checkpoint()
def execute(args, items, settings) -> int:
rows = []
if args.out.exists():
rows = [json.loads(x) for x in args.out.read_text(encoding="utf-8").splitlines() if x.strip()]
ids = {item["id"] for item in items}
done = {row["id"] for row in rows}
if len(done) != len(rows) or not done <= ids:
raise ValueError("Results have duplicate or unknown item IDs")
hashes = {
"source": hashlib.sha256((srcsimplex.SYSTEM_PROMPT + inspect.getsource(srcsimplex.passage_prompt)).encode()).hexdigest(),
"judge": hashlib.sha256(JUDGE_SYSTEM.encode()).hexdigest(),
}
identity = {"model": settings.llm.model, "base_url": settings.llm.base_url,
"temperature": settings.llm.temperature, "items_sha256": digest(args.items),
"prompt_hashes": hashes,
"prompts": [srcsimplex.PROMPT_VERSION, JUDGE_PROMPT_VERSION]}
labels = getattr(args, "labels", None)
if labels is not None:
identity["labels_sha256"] = digest(labels)
meta_path = args.out.with_suffix(".meta.json")
meta = {**identity, "request_attempts": 0, "usage": Usage().as_dict(), "status": "ready"}
if meta_path.exists():
meta = json.loads(meta_path.read_text(encoding="utf-8"))
if any(meta.get(k) != v for k, v in identity.items()):
raise ValueError("Run identity changed; use a new output path instead of mixing runs")
elif rows:
raise ValueError("Cannot resume results without their audit metadata")
todo = [item for item in items if item["id"] not in done]
print(f"items={len(items)} done={len(done)} todo={len(todo)} model={settings.llm.model} "
f"nominal_calls={len(todo) * 2} request_attempts={meta['request_attempts']} cap={args.max_calls}")
print("prompt_hashes:", json.dumps(hashes))
if args.dry_run:
return 0
if settings.llm.model != "qwen-max" or settings.llm.temperature != 0:
raise ValueError("P0 is preregistered for qwen-max at temperature 0")
client = AuditedClient.from_settings(settings.llm)
client.usage = Usage(**meta["usage"])
def checkpoint():
meta["usage"] = client.usage.as_dict()
meta["completed_items"] = len(done)
atomic_json(meta_path, meta)
def reserve():
if meta["request_attempts"] >= args.max_calls:
raise CallBudgetExceeded("Cumulative request cap reached; no further HTTP request was sent")
meta["request_attempts"] += 1
checkpoint()
client.checkpoint = checkpoint
client.request_guard = reserve
meta["max_calls"] = args.max_calls
meta["status"] = "running"
checkpoint()
try:
with args.out.open("a", encoding="utf-8") as stream:
for item in todo:
row = run_item(client, item)
stream.write(json.dumps(row, ensure_ascii=False) + "\n")
stream.flush()
os.fsync(stream.fileno())
done.add(item["id"])
checkpoint()
print(f"completed={len(done)}/{len(items)} requests={meta['request_attempts']}", flush=True)
meta["status"] = "complete"
except CallBudgetExceeded as error:
meta["status"] = "budget_exhausted"
print(str(error), file=sys.stderr)
except BaseException:
meta["status"] = "interrupted"
raise
finally:
if args.out.exists():
meta["results_sha256"] = digest(args.out)
checkpoint()
print(json.dumps(meta))
return 0 if meta["status"] == "complete" else 2
def main() -> int:
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
ap.add_argument("--items", required=True, type=Path)
ap.add_argument("--out", required=True, type=Path)
ap.add_argument("--max-calls", type=int, default=400)
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--labels", type=Path, help="Frozen labels; their hash is bound to the run")
args = ap.parse_args()
if not 1 <= args.max_calls <= 400:
ap.error("P0 requires a request cap between 1 and 400")
items = [json.loads(x) for x in args.items.read_text(encoding="utf-8").splitlines() if x.strip()]
done = set()
if args.out.exists():
done = {json.loads(x)["id"] for x in args.out.read_text(encoding="utf-8").splitlines() if x.strip()}
todo = [i for i in items if i["id"] not in done]
settings = Settings.from_env()
est = len(todo) * 3 # extract (<=2 tries) + judge
print(f"items={len(items)} done={len(done)} todo={len(todo)} model={settings.llm.model} "
f"est_calls<={est} (extract x2 + judge) cap={args.max_calls}")
print("prompts:", srcsimplex.PROMPT_VERSION, JUDGE_PROMPT_VERSION,
hashlib.sha256((srcsimplex.SYSTEM_PROMPT + JUDGE_SYSTEM).encode()).hexdigest()[:12])
if args.dry_run:
return 0
client = OpenAICompatibleClient.from_settings(settings.llm)
if not items or len({item["id"] for item in items}) != len(items):
ap.error("Input must contain unique item IDs")
if any(not item.get("source_text", "").strip() for item in items):
ap.error("Every item needs nonempty source_text; use a private source-inclusive export")
if not args.dry_run and args.labels is None:
ap.error("Freeze all blind labels first, then pass --labels")
if args.labels is not None:
annotation = json.loads(args.labels.read_text(encoding="utf-8"))
key_path = args.labels.with_name("key.json")
key = json.loads(key_path.read_text(encoding="utf-8"))["key"]
verdicts = {"faithful", "minor_drift", "wrong", "not_formalizable"}
if (set(key.values()) != {item["id"] for item in items}
or set(annotation["labels"]) != set(key)
or any(row.get("verdict") not in verdicts for row in annotation["labels"].values())
or annotation.get("items_sha256") != digest(args.items)
or annotation.get("key_sha256") != digest(key_path)):
ap.error("Frozen labels, key and source-inclusive items must be complete and match")
args.out.parent.mkdir(parents=True, exist_ok=True)
with args.out.open("a", encoding="utf-8") as fh:
for item in todo:
if client.usage.calls >= args.max_calls:
print("call cap reached; rerun to resume", file=sys.stderr)
break
fh.write(json.dumps(run_item(client, item), ensure_ascii=False) + "\n")
fh.flush()
meta = {"model": settings.llm.model, "usage": client.usage.as_dict(),
"prompts": [srcsimplex.PROMPT_VERSION, JUDGE_PROMPT_VERSION]}
(args.out.parent / "run_meta.json").write_text(json.dumps(meta, indent=2), encoding="utf-8")
print(json.dumps(meta))
return 0
with args.out.with_suffix(args.out.suffix + ".lock").open("w") as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return execute(args, items, Settings.from_env())
if __name__ == "__main__":