Pilot prep: simplex schema, projection, alignment, lint, stats, pilot scripts, protocol v1
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FYqwNMyPUHeTbFWg83GQq6
This commit is contained in:
@@ -0,0 +1,359 @@
|
||||
"""Compare the source-side simplex with the Lean-side one, role by role.
|
||||
|
||||
The result says *which* parameter disagrees -- a hypothesis the Lean dropped, one it
|
||||
invented, a number system that changed, an ``∃`` turned into a ``∀`` -- which a
|
||||
single "faithful: yes/no" from a judge cannot. Matching is deterministic and cheap:
|
||||
shared symbols (``G``, ``n``) and shared concepts (``group``, ``finite``, ``prime``)
|
||||
through a small synonym table. An LLM pairing (``llm_pairing``) can replace the
|
||||
matching step, so the two can be compared as an ablation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .llm import ChatClient
|
||||
from .project import Projection
|
||||
from .schema import KIND_FACT, KIND_GOAL, KIND_OBJECT, KIND_OWN, Role
|
||||
from .srcsimplex import SourceRole, SourceSimplex
|
||||
|
||||
DEFAULT_THRESHOLD = 0.3
|
||||
GOAL_THRESHOLD = 0.2
|
||||
|
||||
ERROR_TYPES = (
|
||||
"missing_hypothesis", "missing_object", "extra_hypothesis", "extra_object",
|
||||
"type_conflict", "quantifier", "goal_mismatch", "degenerate",
|
||||
)
|
||||
# extra_object is reported but does not flag an item: Lean often introduces helpers.
|
||||
FLAG_ERRORS = frozenset(ERROR_TYPES) - {"extra_object"}
|
||||
|
||||
CONCEPTS: dict[str, set[str]] = {
|
||||
"group": {"group", "grp"}, "abelian": {"abelian", "comm", "commutative"},
|
||||
"finite": {"finite", "fintype", "card"}, "infinite": {"infinite"},
|
||||
"ring": {"ring"}, "field": {"field"}, "domain": {"domain"},
|
||||
"integer": {"integer", "int", "ℤ"}, "natural": {"natural", "nat", "ℕ"},
|
||||
"real": {"real", "ℝ"}, "complex": {"complex", "ℂ"}, "rational": {"rational", "ℚ"},
|
||||
"prime": {"prime"}, "coprime": {"coprime", "relatively"},
|
||||
"divides": {"divides", "divisor", "dvd", "∣", "divisible", "divisibility"},
|
||||
"subgroup": {"subgroup"}, "ideal": {"ideal"}, "normal": {"normal"},
|
||||
"homomorphism": {"homomorphism", "hom", "morphism"},
|
||||
"isomorphism": {"isomorphism", "equiv", "≃", "≅", "isomorphic"},
|
||||
"set": {"set", "subset"}, "function": {"function", "map"},
|
||||
"even": {"even"}, "odd": {"odd"}, "cyclic": {"cyclic"}, "order": {"order", "orderof"},
|
||||
"gcd": {"gcd"}, "lcm": {"lcm"}, "positive": {"positive", "pos"},
|
||||
"nonzero": {"nonzero"}, "unit": {"unit", "units"}, "matrix": {"matrix"},
|
||||
"vector": {"vector"}, "space": {"space"}, "continuous": {"continuous"},
|
||||
"sequence": {"sequence", "seq"}, "polynomial": {"polynomial"},
|
||||
"kernel": {"kernel", "ker"}, "image": {"image", "range"},
|
||||
"injective": {"injective"}, "surjective": {"surjective"}, "bijective": {"bijective"},
|
||||
}
|
||||
_TOKEN_TO_CONCEPT = {tok: concept for concept, toks in CONCEPTS.items() for tok in toks}
|
||||
NUMBER_SYSTEMS = {"integer", "natural", "real", "complex", "rational"}
|
||||
|
||||
_TOKEN = re.compile(r"[A-Z]+(?![a-z])|[A-Z]?[a-z]+|[ℕℤℚℝℂ∣≃≅]")
|
||||
_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_']*")
|
||||
|
||||
|
||||
def tokens(text: str) -> set[str]:
|
||||
out = set()
|
||||
for raw in _TOKEN.findall(text):
|
||||
word = raw.lower()
|
||||
out.add(word)
|
||||
if word.endswith("s") and len(word) > 3:
|
||||
out.add(word[:-1])
|
||||
return out
|
||||
|
||||
|
||||
def concepts(text_or_words: str | list[str]) -> set[str]:
|
||||
words = tokens(text_or_words) if isinstance(text_or_words, str) else {
|
||||
w for item in text_or_words for w in tokens(item)}
|
||||
return {_TOKEN_TO_CONCEPT[w] for w in words if w in _TOKEN_TO_CONCEPT}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Unit:
|
||||
"""One thing on the Lean side a source role can be matched to."""
|
||||
|
||||
key: str
|
||||
kind: str # object | fact | instance
|
||||
role: Role | None
|
||||
owner: str = ""
|
||||
symbols: set[str] = field(default_factory=set)
|
||||
concept_set: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Match:
|
||||
source: str
|
||||
lean: str
|
||||
score: float
|
||||
how: str
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {"source": self.source, "lean": self.lean, "score": round(self.score, 3),
|
||||
"how": self.how}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Conflict:
|
||||
kind: str
|
||||
detail: str
|
||||
source: str = ""
|
||||
lean: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {"kind": self.kind, "detail": self.detail, "source": self.source,
|
||||
"lean": self.lean}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Alignment:
|
||||
matches: list[Match] = field(default_factory=list)
|
||||
conflicts: list[Conflict] = field(default_factory=list)
|
||||
score: float = 0.0
|
||||
flag_errors: frozenset[str] = FLAG_ERRORS
|
||||
|
||||
@property
|
||||
def errors(self) -> list[str]:
|
||||
seen: list[str] = []
|
||||
for conflict in self.conflicts:
|
||||
if conflict.kind not in seen:
|
||||
seen.append(conflict.kind)
|
||||
return seen
|
||||
|
||||
@property
|
||||
def flagged(self) -> bool:
|
||||
return any(c.kind in self.flag_errors for c in self.conflicts)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"score": round(self.score, 3), "flagged": self.flagged, "errors": self.errors,
|
||||
"matches": [m.as_dict() for m in self.matches],
|
||||
"conflicts": [c.as_dict() for c in self.conflicts],
|
||||
}
|
||||
|
||||
|
||||
def lean_units(projection: Projection) -> list[Unit]:
|
||||
units: list[Unit] = []
|
||||
for role in projection.roles:
|
||||
if role.kind == KIND_OBJECT:
|
||||
classes = [str(c) for c in role.attrs.get("instances", [])]
|
||||
units.append(Unit(
|
||||
key=role.name, kind="object", role=role, symbols={role.name},
|
||||
concept_set=concepts(role.type_text) | concepts(" ".join(classes))))
|
||||
for text in classes:
|
||||
units.append(Unit(
|
||||
key=f"inst:{role.name}:{text}", kind="instance", role=None,
|
||||
owner=role.name, symbols={role.name}, concept_set=concepts(text)))
|
||||
elif role.kind == KIND_FACT:
|
||||
units.append(Unit(
|
||||
key=role.name, kind="fact", role=role,
|
||||
symbols=set(_IDENT.findall(role.type_text)) | {role.name},
|
||||
concept_set=concepts(role.type_text) | _relation_concepts(role.type_text)))
|
||||
return units
|
||||
|
||||
|
||||
def _relation_concepts(text: str) -> set[str]:
|
||||
found: set[str] = set()
|
||||
if re.search(r"0\s*<|>\s*0", text):
|
||||
found.add("positive")
|
||||
if re.search(r"≠\s*0|0\s*≠", text):
|
||||
found.add("nonzero")
|
||||
if "∣" in text:
|
||||
found.add("divides")
|
||||
return found
|
||||
|
||||
|
||||
def _score(source: SourceRole, unit: Unit) -> float:
|
||||
symbol = 1.0 if (set(source.symbols) & unit.symbols) else 0.0
|
||||
wanted = concepts(source.keywords) | concepts(source.description)
|
||||
if wanted and unit.concept_set:
|
||||
concept = len(wanted & unit.concept_set) / min(len(wanted), len(unit.concept_set))
|
||||
else:
|
||||
concept = 0.0
|
||||
return 0.6 * symbol + 0.4 * concept
|
||||
|
||||
|
||||
def _compatible(source: SourceRole, unit: Unit) -> bool:
|
||||
if source.kind == "object":
|
||||
return unit.kind == "object"
|
||||
if source.kind == "fact":
|
||||
return unit.kind in ("fact", "instance")
|
||||
return False
|
||||
|
||||
|
||||
def _pair_deterministic(
|
||||
source: SourceSimplex, units: list[Unit], threshold: float
|
||||
) -> dict[str, tuple[Unit, float, str]]:
|
||||
candidates = []
|
||||
for role in source.roles:
|
||||
for unit in units:
|
||||
if _compatible(role, unit):
|
||||
score = _score(role, unit)
|
||||
if score >= threshold:
|
||||
candidates.append((score, role.alias, unit.key, role, unit))
|
||||
candidates.sort(key=lambda c: (-c[0], c[1], c[2]))
|
||||
taken_sources: set[str] = set()
|
||||
taken_units: set[str] = set()
|
||||
pairs: dict[str, tuple[Unit, float, str]] = {}
|
||||
for score, alias, key, _role, unit in candidates:
|
||||
if alias in taken_sources:
|
||||
continue
|
||||
if key in taken_units and unit.kind != "instance":
|
||||
continue
|
||||
taken_sources.add(alias)
|
||||
taken_units.add(key)
|
||||
pairs[alias] = (unit, score, "symbol+concept")
|
||||
return pairs
|
||||
|
||||
|
||||
def _existential_names(goal_text: str) -> set[str]:
|
||||
names: set[str] = set()
|
||||
for match in re.finditer(r"∃([^,]*),", goal_text):
|
||||
names.update(_IDENT.findall(match.group(1)))
|
||||
return names
|
||||
|
||||
|
||||
def align(
|
||||
source: SourceSimplex,
|
||||
projection: Projection,
|
||||
*,
|
||||
threshold: float = DEFAULT_THRESHOLD,
|
||||
pairing: dict[str, str] | None = None,
|
||||
flag_errors: frozenset[str] = FLAG_ERRORS,
|
||||
) -> Alignment:
|
||||
out = Alignment(flag_errors=flag_errors)
|
||||
if projection.degenerate:
|
||||
out.conflicts.append(Conflict("degenerate", "; ".join(projection.notes) or "no roles"))
|
||||
units = lean_units(projection)
|
||||
by_key = {u.key: u for u in units}
|
||||
|
||||
if pairing is not None:
|
||||
pairs = {a: (by_key[k], 1.0, "llm") for a, k in pairing.items() if k in by_key}
|
||||
else:
|
||||
pairs = _pair_deterministic(source, units, threshold)
|
||||
|
||||
goal_role = next((r for r in projection.roles if r.kind in (KIND_GOAL, KIND_OWN)), None)
|
||||
goal_text = goal_role.type_text if goal_role else ""
|
||||
exist_names = _existential_names(goal_text)
|
||||
|
||||
used_units = {u.key for u, _s, _h in pairs.values()}
|
||||
matched_objects: dict[str, Unit] = {}
|
||||
for role in source.roles:
|
||||
if role.kind == "goal":
|
||||
continue
|
||||
hit = pairs.get(role.alias)
|
||||
if hit:
|
||||
unit, score, how = hit
|
||||
out.matches.append(Match(role.alias, unit.key, score, how))
|
||||
if role.kind == "object":
|
||||
matched_objects[role.alias] = unit
|
||||
_check_object(role, unit, exist_names, out)
|
||||
continue
|
||||
symbols = set(role.symbols)
|
||||
if role.kind == "object":
|
||||
if symbols & exist_names and role.introduced == "exists":
|
||||
out.matches.append(Match(role.alias, "∃-in-goal", 1.0, "existential"))
|
||||
elif symbols & exist_names:
|
||||
out.conflicts.append(Conflict(
|
||||
"quantifier", f"{sorted(symbols & exist_names)} is ∃-bound in Lean "
|
||||
f"but the source says '{role.introduced}'", role.alias))
|
||||
else:
|
||||
out.conflicts.append(Conflict("missing_object", role.description, role.alias))
|
||||
else:
|
||||
if _absorbed(role, units):
|
||||
out.matches.append(Match(role.alias, "type", 0.5, "absorbed"))
|
||||
else:
|
||||
out.conflicts.append(Conflict("missing_hypothesis", role.description, role.alias))
|
||||
|
||||
for unit in units:
|
||||
if unit.kind == "fact" and unit.key not in used_units:
|
||||
out.conflicts.append(Conflict("extra_hypothesis", unit.role.type_text, "", unit.key))
|
||||
elif unit.kind == "object" and unit.key not in used_units:
|
||||
out.conflicts.append(Conflict(
|
||||
"extra_object", f"{unit.key} : {unit.role.type_text}", "", unit.key))
|
||||
|
||||
src_goal = source.goal
|
||||
goal_ok = True
|
||||
if src_goal is not None and goal_role is not None:
|
||||
goal_symbols = set(_IDENT.findall(goal_text))
|
||||
sym = 1.0 if set(src_goal.symbols) & goal_symbols else 0.0
|
||||
wanted = concepts(src_goal.keywords) | concepts(src_goal.description)
|
||||
have = concepts(goal_text) | _relation_concepts(goal_text)
|
||||
con = len(wanted & have) / min(len(wanted), len(have)) if wanted and have else 0.0
|
||||
informative = bool(src_goal.symbols or wanted)
|
||||
if informative and 0.5 * sym + 0.5 * con < GOAL_THRESHOLD:
|
||||
goal_ok = False
|
||||
out.conflicts.append(Conflict(
|
||||
"goal_mismatch", f"source: {src_goal.description} | lean: {goal_text[:120]}",
|
||||
src_goal.alias, goal_role.name))
|
||||
lean_exists = goal_text.count("∃") > 0
|
||||
if source.goal_has_exists != lean_exists:
|
||||
out.conflicts.append(Conflict(
|
||||
"quantifier",
|
||||
f"source goal ∃={source.goal_has_exists}, lean goal ∃={lean_exists}",
|
||||
src_goal.alias, goal_role.name))
|
||||
if goal_ok and src_goal is not None and goal_role is not None:
|
||||
out.matches.append(Match(src_goal.alias, goal_role.name, 1.0, "goal"))
|
||||
|
||||
n_source = len(source.roles)
|
||||
n_lean = len([u for u in units if u.kind != "instance"]) + (1 if goal_role else 0)
|
||||
out.score = 2 * len(out.matches) / (n_source + n_lean) if (n_source + n_lean) else 0.0
|
||||
return out
|
||||
|
||||
|
||||
def _check_object(role: SourceRole, unit: Unit, exist_names: set[str], out: Alignment) -> None:
|
||||
wanted = (concepts(role.keywords) | concepts(role.description)) & NUMBER_SYSTEMS
|
||||
have = unit.concept_set & NUMBER_SYSTEMS
|
||||
if wanted and have and wanted.isdisjoint(have):
|
||||
out.conflicts.append(Conflict(
|
||||
"type_conflict", f"source {sorted(wanted)} vs lean {sorted(have)}",
|
||||
role.alias, unit.key))
|
||||
bound_exists = unit.key in exist_names
|
||||
if role.introduced == "exists" and not bound_exists:
|
||||
out.conflicts.append(Conflict(
|
||||
"quantifier", f"source asserts {unit.key} exists, lean takes it as given",
|
||||
role.alias, unit.key))
|
||||
|
||||
|
||||
def _absorbed(role: SourceRole, units: list[Unit]) -> bool:
|
||||
wanted = concepts(role.keywords) | concepts(role.description)
|
||||
if not wanted:
|
||||
return False
|
||||
for unit in units:
|
||||
if unit.kind == "object" and wanted & unit.concept_set:
|
||||
if not role.symbols or set(role.symbols) & unit.symbols:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
PAIRING_PROMPT = """\
|
||||
You pair the logical parameters of a math passage with the binders of a formal
|
||||
statement. For each SOURCE role, give the key of the LEAN item that expresses the
|
||||
same parameter, or null if none does. Pair by meaning, not by spelling. A LEAN item can
|
||||
be used once, except instance items. Reply with JSON: {"pairs": {"<source alias>":
|
||||
"<lean key or null>"}}."""
|
||||
|
||||
|
||||
def llm_pairing(
|
||||
client: ChatClient, source: SourceSimplex, projection: Projection
|
||||
) -> dict[str, str]:
|
||||
"""Ablation: let a model do the matching step (it sees roles, not the passage)."""
|
||||
units = lean_units(projection)
|
||||
listing = {
|
||||
"source": [r.as_dict() for r in source.roles if r.kind != "goal"],
|
||||
"lean": [
|
||||
{"key": u.key, "kind": u.kind,
|
||||
"text": (u.role.type_text if u.role else u.key.split(":", 2)[-1])}
|
||||
for u in units
|
||||
],
|
||||
}
|
||||
import json
|
||||
reply = client.complete_json([ # type: ignore[attr-defined]
|
||||
{"role": "system", "content": PAIRING_PROMPT},
|
||||
{"role": "user", "content": json.dumps(listing, ensure_ascii=False)},
|
||||
])
|
||||
pairs = reply.get("pairs", {}) if isinstance(reply, dict) else {}
|
||||
valid = {u.key for u in units}
|
||||
return {a: k for a, k in pairs.items() if isinstance(k, str) and k in valid}
|
||||
@@ -0,0 +1,93 @@
|
||||
"""Small, dependency-free statistics for evaluating a detector against human labels."""
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
def wilson(successes: int, total: int, z: float = 1.96) -> tuple[float, float]:
|
||||
"""Wilson score interval for a proportion; (0, 1) when there is no data."""
|
||||
if total <= 0:
|
||||
return 0.0, 1.0
|
||||
p = successes / total
|
||||
denom = 1 + z * z / total
|
||||
centre = (p + z * z / (2 * total)) / denom
|
||||
half = z * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total)) / denom
|
||||
return max(0.0, centre - half), min(1.0, centre + half)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Confusion:
|
||||
"""``positive`` means 'the item is unfaithful' and a detector 'flags' it."""
|
||||
|
||||
tp: int = 0
|
||||
fp: int = 0
|
||||
fn: int = 0
|
||||
tn: int = 0
|
||||
|
||||
@property
|
||||
def n(self) -> int:
|
||||
return self.tp + self.fp + self.fn + self.tn
|
||||
|
||||
@property
|
||||
def recall(self) -> float:
|
||||
return self.tp / (self.tp + self.fn) if (self.tp + self.fn) else float("nan")
|
||||
|
||||
@property
|
||||
def precision(self) -> float:
|
||||
return self.tp / (self.tp + self.fp) if (self.tp + self.fp) else float("nan")
|
||||
|
||||
@property
|
||||
def false_flag_rate(self) -> float:
|
||||
return self.fp / (self.fp + self.tn) if (self.fp + self.tn) else float("nan")
|
||||
|
||||
@property
|
||||
def f1(self) -> float:
|
||||
p, r = self.precision, self.recall
|
||||
return 2 * p * r / (p + r) if (p == p and r == r and (p + r) > 0) else float("nan")
|
||||
|
||||
def as_dict(self) -> dict[str, float | int]:
|
||||
return {
|
||||
"tp": self.tp, "fp": self.fp, "fn": self.fn, "tn": self.tn, "n": self.n,
|
||||
"recall": self.recall, "precision": self.precision,
|
||||
"false_flag_rate": self.false_flag_rate, "f1": self.f1,
|
||||
"recall_ci": wilson(self.tp, self.tp + self.fn),
|
||||
"false_flag_ci": wilson(self.fp, self.fp + self.tn),
|
||||
}
|
||||
|
||||
|
||||
def confusion(flagged: Sequence[bool], unfaithful: Sequence[bool]) -> Confusion:
|
||||
if len(flagged) != len(unfaithful):
|
||||
raise ValueError("flagged and unfaithful must have the same length")
|
||||
c = Confusion()
|
||||
for f, u in zip(flagged, unfaithful, strict=True):
|
||||
if f and u:
|
||||
c.tp += 1
|
||||
elif f and not u:
|
||||
c.fp += 1
|
||||
elif u:
|
||||
c.fn += 1
|
||||
else:
|
||||
c.tn += 1
|
||||
return c
|
||||
|
||||
|
||||
def auc(scores: Sequence[float], unfaithful: Sequence[bool]) -> float:
|
||||
"""ROC-AUC of 'higher score means unfaithful' (rank-sum form, ties count half)."""
|
||||
pos = [s for s, u in zip(scores, unfaithful, strict=True) if u]
|
||||
neg = [s for s, u in zip(scores, unfaithful, strict=True) if not u]
|
||||
if not pos or not neg:
|
||||
return float("nan")
|
||||
wins = sum((p > n) + 0.5 * (p == n) for p in pos for n in neg)
|
||||
return wins / (len(pos) * len(neg))
|
||||
|
||||
|
||||
def cohen_kappa(a: Sequence[str], b: Sequence[str]) -> float:
|
||||
if len(a) != len(b) or not a:
|
||||
raise ValueError("need two label lists of equal, non-zero length")
|
||||
n = len(a)
|
||||
observed = sum(x == y for x, y in zip(a, b, strict=True)) / n
|
||||
labels = set(a) | set(b)
|
||||
expected = sum((a.count(k) / n) * (b.count(k) / n) for k in labels)
|
||||
return 1.0 if expected == 1 else (observed - expected) / (1 - expected)
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Cheap static checks for formalizations that compile and are probably wrong.
|
||||
|
||||
These are a baseline for the fidelity filter, not a replacement for one: each looks
|
||||
at the projected roles and the text of the declaration only, with no model and no
|
||||
Lean. They catch encoding hazards (natural-number subtraction, integer division),
|
||||
binders the statement never uses, and goals that say nothing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from .project import Projection
|
||||
from .schema import KIND_FACT, KIND_GOAL, KIND_OBJECT
|
||||
|
||||
_NAT = re.compile(r"\b(ℕ|Nat)\b")
|
||||
_OTHER_NUMBERS = re.compile(r"[ℤℚℝℂ]|\b(Int|Rat|Real|Complex)\b")
|
||||
_SUBTRACTION = re.compile(r"[\w)\]]\s*-\s*[\w(]")
|
||||
_DIVISION = re.compile(r"[\w)\]]\s*/\s*[\w(]")
|
||||
_IDENT = re.compile(r"[A-Za-z_][A-Za-z0-9_']*")
|
||||
|
||||
|
||||
def lint(projection: Projection) -> list[str]:
|
||||
findings: list[str] = []
|
||||
if projection.degenerate:
|
||||
findings.append("degenerate")
|
||||
texts = [r.type_text for r in projection.roles if r.kind in (KIND_FACT, KIND_GOAL)]
|
||||
body = " ".join(texts)
|
||||
declared = " ".join(r.type_text for r in projection.roles if r.kind == KIND_OBJECT)
|
||||
|
||||
for role in projection.roles:
|
||||
if role.kind != KIND_OBJECT or role.attrs.get("binder_kind") != "explicit":
|
||||
continue
|
||||
if role.type_text.strip().startswith(("Type", "Sort")):
|
||||
continue # a carrier: used by the instances that constrain it
|
||||
used = set(_IDENT.findall(body)) | set(_IDENT.findall(declared.replace(role.type_text, "")))
|
||||
if role.name != "_" and role.name not in used:
|
||||
findings.append(f"unused_binder:{role.name}")
|
||||
|
||||
nat_only = _NAT.search(declared + " " + body) and not _OTHER_NUMBERS.search(declared + body)
|
||||
if nat_only and _SUBTRACTION.search(body):
|
||||
findings.append("nat_subtraction")
|
||||
if (_NAT.search(declared) or re.search(r"\b(ℤ|Int)\b", declared)) and _DIVISION.search(body):
|
||||
findings.append("integer_division")
|
||||
|
||||
goal = next((r for r in projection.roles if r.kind == KIND_GOAL), None)
|
||||
if goal is not None and goal.type_text.strip() in ("", "True", "False", "1 = 1", "0 = 0"):
|
||||
findings.append("trivial_goal")
|
||||
return findings
|
||||
@@ -0,0 +1,260 @@
|
||||
"""Read a Lean declaration's parameters as roles.
|
||||
|
||||
``project(code)`` is deterministic and never calls Lean or a model. It splits the
|
||||
signature into
|
||||
|
||||
* **objects** explicit/implicit binders whose type is data or a sort: ``(G : Type*)``,
|
||||
``(n : ℕ)``; instance binders ``[Group G]`` attach to the object they
|
||||
constrain (``attrs["instances"]``);
|
||||
* **facts** binders whose type is a proposition: ``(h : a ∣ b)``, and, when the goal
|
||||
is an implication, its premises;
|
||||
* **goal** the conclusion of a theorem, or **own** for what a definition is.
|
||||
|
||||
Leading ``∀`` binders of the goal are unpacked into objects, so ``(n : ℕ) : n = n`` and
|
||||
``∀ n : ℕ, n = n`` project to the same roles. Whether a binder is a fact is decided by
|
||||
a syntactic heuristic (relation symbols, known predicate heads, the ``h…`` naming
|
||||
convention) and every role records how it was decided in ``attrs["classified_by"]``,
|
||||
so the heuristic's accuracy can be measured against a manual parse.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .schema import (
|
||||
KIND_FACT,
|
||||
KIND_GOAL,
|
||||
KIND_OBJECT,
|
||||
KIND_OWN,
|
||||
Role,
|
||||
Simplex,
|
||||
)
|
||||
from .typeindex import _DECL, _IDENT, PROOF_KEYWORDS, _split_top_level, _strip_comments
|
||||
|
||||
OPENERS = {"(": ")", "{": "}", "[": "]", "⦃": "⦄", "⟨": "⟩"}
|
||||
CLOSERS = set(OPENERS.values())
|
||||
BINDER_KIND = {"(": "explicit", "{": "implicit", "⦃": "strict_implicit", "[": "instance"}
|
||||
|
||||
_SORT = re.compile(r"^(Type|Sort)(\b|\*|_)")
|
||||
_RELATION = re.compile(r"[=≠<>≤≥∣∈∉⊆⊂⊇⊃≡∧∨¬↔∀∃≃≅≈]|\bTrue\b|\bFalse\b")
|
||||
_PROP_HEAD = re.compile(
|
||||
r"^(Is[A-Z]\w*|Nat\.Prime|Prime|Nat\.Coprime|Int\.Coprime|Coprime|Function\.\w+|"
|
||||
r"Injective|Surjective|Bijective|Continuous\w*|Differentiable\w*|Monotone\w*|Antitone\w*|"
|
||||
r"Finite|Set\.Finite|Set\.Infinite|Infinite|Odd|Even|Irreducible|Squarefree|"
|
||||
r"Nonempty|Set\.Nonempty|Nat\.Even|Nat\.Odd)\b"
|
||||
)
|
||||
_HYPOTHESIS_NAME = re.compile(r"^h[\w'₀-₉]*$")
|
||||
|
||||
|
||||
def _split_depth0(text: str, separator: str) -> list[str]:
|
||||
"""Split on ``separator`` outside every bracket pair."""
|
||||
parts: list[str] = []
|
||||
depth = 0
|
||||
start = 0
|
||||
for index, char in enumerate(text):
|
||||
if char in OPENERS:
|
||||
depth += 1
|
||||
elif char in CLOSERS:
|
||||
depth = max(0, depth - 1)
|
||||
elif depth == 0 and text.startswith(separator, index):
|
||||
parts.append(text[start:index])
|
||||
start = index + len(separator)
|
||||
parts.append(text[start:])
|
||||
return parts
|
||||
|
||||
|
||||
def _first_depth0(text: str, needle: str) -> int:
|
||||
depth = 0
|
||||
for index, char in enumerate(text):
|
||||
if char in OPENERS:
|
||||
depth += 1
|
||||
elif char in CLOSERS:
|
||||
depth = max(0, depth - 1)
|
||||
elif depth == 0 and text.startswith(needle, index):
|
||||
if needle == ":" and text.startswith(":=", index):
|
||||
continue
|
||||
return index
|
||||
return -1
|
||||
|
||||
|
||||
def classify(type_text: str, names: list[str], objects: set[str]) -> tuple[str, str]:
|
||||
"""(kind, how) for a binder with these names and this type."""
|
||||
text = type_text.strip()
|
||||
if not text:
|
||||
return KIND_OBJECT, "untyped"
|
||||
if _SORT.match(text):
|
||||
return KIND_OBJECT, "sort"
|
||||
if _RELATION.search(text):
|
||||
return KIND_FACT, "relation"
|
||||
if _PROP_HEAD.match(text):
|
||||
return KIND_FACT, "predicate"
|
||||
if names and all(_HYPOTHESIS_NAME.match(n) for n in names):
|
||||
tokens = {t for t in _IDENT.findall(text) if t[:1].islower()}
|
||||
if tokens & objects:
|
||||
return KIND_FACT, "name"
|
||||
return KIND_OBJECT, "default"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Projection:
|
||||
name: str
|
||||
keyword: str
|
||||
roles: list[Role] = field(default_factory=list)
|
||||
simplex: Simplex | None = None
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def degenerate(self) -> bool:
|
||||
return self.simplex is None
|
||||
|
||||
def by_kind(self, kind: str) -> list[Role]:
|
||||
return [r for r in self.roles if r.kind == kind]
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"keyword": self.keyword,
|
||||
"degenerate": self.degenerate,
|
||||
"roles": [r.as_dict() for r in self.roles],
|
||||
"notes": self.notes,
|
||||
}
|
||||
|
||||
|
||||
def _cut_signature(rest: str) -> str:
|
||||
depth = 0
|
||||
for index, char in enumerate(rest):
|
||||
if char in OPENERS:
|
||||
depth += 1
|
||||
elif char in CLOSERS:
|
||||
depth = max(0, depth - 1)
|
||||
elif depth == 0:
|
||||
if rest.startswith(":=", index) or rest.startswith("\nwhere", index):
|
||||
return rest[:index]
|
||||
if rest.startswith(" where", index) and index > 0:
|
||||
return rest[:index]
|
||||
return rest
|
||||
|
||||
|
||||
def _roles_from_group(opener: str, inner: str, objects: set[str], origin: str) -> list[Role]:
|
||||
binder_kind = BINDER_KIND.get(opener, "explicit")
|
||||
head, _, tail = inner.partition(":")
|
||||
names = head.split()
|
||||
type_text = tail.strip() if ":" in inner else ""
|
||||
if not names or any(not _IDENT.fullmatch(n) and n != "_" for n in names):
|
||||
return []
|
||||
kind, how = classify(type_text, names, objects)
|
||||
roles = []
|
||||
for name in names:
|
||||
attrs = {"binder_kind": binder_kind, "classified_by": how, "origin": origin}
|
||||
roles.append(Role(name, kind, type_text=type_text, attrs=attrs))
|
||||
if kind == KIND_OBJECT:
|
||||
objects.add(name)
|
||||
return roles
|
||||
|
||||
|
||||
def _attach_instance(inner: str, roles: dict[str, Role], notes: list[str]) -> None:
|
||||
text = inner.split(":", 1)[1].strip() if ":" in inner else inner.strip()
|
||||
owner = None
|
||||
for token in _IDENT.findall(text):
|
||||
if token in roles and roles[token].kind == KIND_OBJECT:
|
||||
owner = roles[token]
|
||||
if owner is None:
|
||||
notes.append(f"instance [{text}] constrains no named object")
|
||||
return
|
||||
owner.attrs.setdefault("instances", []).append(text)
|
||||
|
||||
|
||||
def _unpack_forall(goal: str, objects: set[str]) -> tuple[list[Role], str]:
|
||||
"""Lift leading ``∀`` binders of the goal into object/fact roles."""
|
||||
lifted: list[Role] = []
|
||||
text = goal.strip()
|
||||
while text.startswith("∀"):
|
||||
body = text[1:].lstrip()
|
||||
comma = _first_depth0(body, ",")
|
||||
if comma < 0:
|
||||
break
|
||||
binders, text = body[:comma].strip(), body[comma + 1 :].strip()
|
||||
groups = _split_top_level(binders)
|
||||
if groups and binders.startswith(tuple(OPENERS)):
|
||||
for opener, inner in groups:
|
||||
lifted.extend(_roles_from_group(opener, inner, objects, "forall_in_goal"))
|
||||
continue
|
||||
bound = re.match(r"^([\w'₀-₉ ]+?)\s*(∈|∉|<|≤|>|≥|≠)\s*(.+)$", binders)
|
||||
if bound and ":" not in binders:
|
||||
for name in bound.group(1).split():
|
||||
lifted.append(Role(name, KIND_OBJECT, attrs={
|
||||
"binder_kind": "explicit", "classified_by": "bounded", "origin": "forall_in_goal"}))
|
||||
objects.add(name)
|
||||
lifted.append(Role(
|
||||
f"h_{name}", KIND_FACT, type_text=f"{name} {bound.group(2)} {bound.group(3)}",
|
||||
attrs={"binder_kind": "explicit", "classified_by": "bounded",
|
||||
"origin": "forall_in_goal"}))
|
||||
continue
|
||||
lifted.extend(_roles_from_group("(", binders, objects, "forall_in_goal"))
|
||||
return lifted, text
|
||||
|
||||
|
||||
def project(code: str) -> Projection:
|
||||
clean = _strip_comments(code)
|
||||
match = _DECL.search(clean)
|
||||
if not match:
|
||||
return Projection("", "", notes=["no declaration found"])
|
||||
keyword, name = match.group(1), match.group(2)
|
||||
result = Projection(name, keyword)
|
||||
signature = _cut_signature(clean[match.end() :])
|
||||
|
||||
colon = _first_depth0(signature, ":")
|
||||
binder_text = signature if colon < 0 else signature[:colon]
|
||||
goal_text = "" if colon < 0 else signature[colon + 1 :].strip()
|
||||
|
||||
objects: set[str] = set()
|
||||
named: dict[str, Role] = {}
|
||||
instances: list[str] = []
|
||||
for opener, inner in _split_top_level(binder_text):
|
||||
if opener == "[":
|
||||
instances.append(inner)
|
||||
continue
|
||||
if opener == "⟨":
|
||||
continue
|
||||
for role in _roles_from_group(opener, inner, objects, "binder"):
|
||||
result.roles.append(role)
|
||||
named[role.name] = role
|
||||
for inner in instances:
|
||||
_attach_instance(inner, named, result.notes)
|
||||
|
||||
if keyword in PROOF_KEYWORDS:
|
||||
lifted, goal_text = _unpack_forall(goal_text, objects)
|
||||
for role in lifted:
|
||||
result.roles.append(role)
|
||||
named[role.name] = role
|
||||
parts = _split_depth0(goal_text, "→") if goal_text else []
|
||||
if len(parts) >= 2:
|
||||
for index, premise in enumerate(parts[:-1], start=1):
|
||||
result.roles.append(Role(
|
||||
f"premise{index}", KIND_FACT, type_text=premise.strip(),
|
||||
attrs={"binder_kind": "goal_implication", "classified_by": "implication",
|
||||
"origin": "goal"}))
|
||||
goal_text = parts[-1].strip()
|
||||
result.roles.append(Role(
|
||||
"goal", KIND_GOAL, type_text=goal_text,
|
||||
attrs={
|
||||
"exists_count": goal_text.count("∃"),
|
||||
"forall_count": goal_text.count("∀"),
|
||||
"is_iff": "↔" in goal_text,
|
||||
}))
|
||||
else:
|
||||
result.roles.append(Role(
|
||||
"own", KIND_OWN, type_text=goal_text or keyword, attrs={"keyword": keyword}))
|
||||
|
||||
seen: dict[str, int] = {}
|
||||
aliased: dict[str, Role] = {}
|
||||
for role in result.roles:
|
||||
count = seen.get(role.name, 0) + 1
|
||||
seen[role.name] = count
|
||||
aliased[role.name if count == 1 else f"{role.name}_{count}"] = role
|
||||
if len(aliased) >= 2:
|
||||
result.simplex = Simplex(name, aliased)
|
||||
else:
|
||||
result.notes.append(f"degenerate: only {len(aliased)} role(s)")
|
||||
return result
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Roles and simplices: the parameters of a declaration as a small typed schema.
|
||||
|
||||
A Lean declaration is a bundle of roles: the objects it is about (``(G : Type*)``),
|
||||
the facts it assumes (``(h : a ∣ b)``), and the goal it asserts or the thing it
|
||||
defines. A ``Simplex`` is such a bundle with at least two roles. The same schema can
|
||||
be read off the prose of a book, so the two can be compared role by role.
|
||||
|
||||
Design notes (from the archived ``schema-simplex`` prototype):
|
||||
|
||||
* a role may be *constrained* to a closed set of allowed values (``choices``);
|
||||
* a simplex is itself a role, so simplices nest (a lemma used by another is a role
|
||||
of it), which is how a dependency structure appears;
|
||||
* validation lives in ``__init__``. The prototype put it in a dataclass
|
||||
``__post_init__`` next to a hand-written ``__init__``, where it never ran.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
KIND_OBJECT = "object" # something the statement is about: a group G, an integer n
|
||||
KIND_FACT = "fact" # something assumed: n > 0, G is finite
|
||||
KIND_GOAL = "goal" # what is asserted, or asked to be shown
|
||||
KIND_OWN = "own" # what a definition is
|
||||
KIND_SIMPLEX = "simplex"
|
||||
KINDS = (KIND_OBJECT, KIND_FACT, KIND_GOAL, KIND_OWN, KIND_SIMPLEX)
|
||||
|
||||
MIN_ROLES = 2
|
||||
|
||||
|
||||
class Role:
|
||||
"""A named slot with a kind, a description and optionally a closed value set."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
kind: str = KIND_OBJECT,
|
||||
*,
|
||||
type_text: str = "",
|
||||
context: str = "",
|
||||
choices: Sequence[Any] | None = None,
|
||||
attrs: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
if not name:
|
||||
raise ValueError("a role needs a name")
|
||||
if kind not in KINDS:
|
||||
raise ValueError(f"unknown role kind {kind!r}; expected one of {KINDS}")
|
||||
self.name = name
|
||||
self.kind = kind
|
||||
self.type_text = type_text
|
||||
self.context = context
|
||||
self.choices: list[Any] | None = list(choices) if choices is not None else None
|
||||
self.attrs: dict[str, Any] = dict(attrs or {})
|
||||
|
||||
def is_constrained(self) -> bool:
|
||||
return self.choices is not None
|
||||
|
||||
def validate_value(self, value: Any) -> bool:
|
||||
return self.choices is None or value in self.choices
|
||||
|
||||
def check_value(self, value: Any) -> None:
|
||||
if not self.validate_value(value):
|
||||
raise ValueError(
|
||||
f"{value!r} is not allowed for role {self.name!r}; allowed: {self.choices}"
|
||||
)
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {"name": self.name, "kind": self.kind}
|
||||
if self.type_text:
|
||||
out["type"] = self.type_text
|
||||
if self.context:
|
||||
out["context"] = self.context
|
||||
if self.choices is not None:
|
||||
out["choices"] = self.choices
|
||||
if self.attrs:
|
||||
out["attrs"] = self.attrs
|
||||
return out
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"<Role {self.kind}:{self.name}>"
|
||||
|
||||
|
||||
class Simplex(Role):
|
||||
"""A reusable bundle of at least two roles, keyed by alias. Itself a role."""
|
||||
|
||||
def __init__(self, name: str, roles: dict[str, Role], *, context: str = "") -> None:
|
||||
super().__init__(name, KIND_SIMPLEX, context=context)
|
||||
if len(roles) < MIN_ROLES:
|
||||
raise ValueError(
|
||||
f"simplex {name!r} needs at least {MIN_ROLES} roles, got {len(roles)}"
|
||||
)
|
||||
self.roles = dict(roles)
|
||||
|
||||
def by_kind(self, kind: str) -> list[Role]:
|
||||
return [role for role in self.roles.values() if role.kind == kind]
|
||||
|
||||
def walk(self) -> list[Role]:
|
||||
"""Every non-simplex role, descending into nested simplices."""
|
||||
out: list[Role] = []
|
||||
for role in self.roles.values():
|
||||
if isinstance(role, Simplex):
|
||||
out.extend(role.walk())
|
||||
else:
|
||||
out.append(role)
|
||||
return out
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"roles": {alias: role.as_dict() for alias, role in self.roles.items()},
|
||||
}
|
||||
|
||||
|
||||
class State:
|
||||
"""A role together with a value; unresolved until it has one."""
|
||||
|
||||
def __init__(self, role: Role, value: Any = None) -> None:
|
||||
self.role = role
|
||||
self.value = value
|
||||
|
||||
def is_resolved(self) -> bool:
|
||||
return self.value is not None
|
||||
|
||||
|
||||
class Fact(State):
|
||||
"""Known at the start."""
|
||||
|
||||
def __init__(self, role: Role, value: Any) -> None:
|
||||
if value is None:
|
||||
raise ValueError("a Fact must carry a value")
|
||||
super().__init__(role, value)
|
||||
|
||||
|
||||
class Goal(State):
|
||||
"""To be resolved."""
|
||||
@@ -0,0 +1,165 @@
|
||||
"""The source-side simplex: what the *prose* gives, assumes and claims.
|
||||
|
||||
One model call reads only the book passage -- never the Lean -- and lists its roles.
|
||||
Keeping the Lean out of view is the point: the schema must come from the source, so
|
||||
that comparing it with the Lean-side schema is a check and not an echo.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from .llm import ChatClient, parse_json
|
||||
|
||||
PROMPT_VERSION = "srcsimplex-v1"
|
||||
|
||||
INTRODUCED = ("given", "forall", "exists")
|
||||
|
||||
SYSTEM_PROMPT = """\
|
||||
You read ONE passage from a mathematics book and describe its logical parameters.
|
||||
Do not write Lean. Do not solve anything. Describe only what the passage itself says.
|
||||
|
||||
Return a JSON object:
|
||||
{
|
||||
"roles": [
|
||||
{"kind": "object", "symbols": ["G"], "description": "finite group",
|
||||
"keywords": ["finite", "group"], "introduced": "given"},
|
||||
{"kind": "fact", "symbols": ["n"], "description": "n is positive",
|
||||
"keywords": ["positive"], "introduced": "given"},
|
||||
{"kind": "goal", "symbols": [], "description": "every element's order divides |G|",
|
||||
"keywords": ["order", "divides"], "introduced": "given"}
|
||||
],
|
||||
"goal_has_exists": false
|
||||
}
|
||||
|
||||
Rules:
|
||||
- "object": a mathematical object the passage is about (a group G, an integer n, a
|
||||
function f). One entry per object, even if several share a description.
|
||||
- "fact": a stated assumption or condition on those objects ("n > 0", "G is
|
||||
finite", "a and b are coprime"). Include conditions hidden in prose ("let G be a
|
||||
finite group" gives an object G AND a fact that G is finite).
|
||||
- "goal": exactly one entry: the claim being asserted, the property being defined, or
|
||||
for an exercise the thing to be shown. For a definition, the goal is what is defined.
|
||||
- "symbols": the letters/names used in the text, as written. Empty if none.
|
||||
- "keywords": 1-5 lowercase words naming the concepts involved (e.g. "integer",
|
||||
"prime", "subgroup", "homomorphism", "finite"). Words from the text, not synonyms.
|
||||
- "introduced" (objects only): "given" if the passage fixes it ("let n be..."),
|
||||
"forall" if it ranges over all of them ("for every n"), "exists" if the passage asserts
|
||||
one exists ("there is an n such that"). Use "given" when unsure.
|
||||
- "goal_has_exists": true only if the claim itself asserts that something exists.
|
||||
- At least two roles in total, and exactly one goal.
|
||||
Reply with the JSON object only."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceRole:
|
||||
kind: str
|
||||
symbols: list[str] = field(default_factory=list)
|
||||
description: str = ""
|
||||
keywords: list[str] = field(default_factory=list)
|
||||
introduced: str = "given"
|
||||
alias: str = ""
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"alias": self.alias, "kind": self.kind, "symbols": self.symbols,
|
||||
"description": self.description, "keywords": self.keywords,
|
||||
"introduced": self.introduced,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class SourceSimplex:
|
||||
roles: list[SourceRole]
|
||||
goal_has_exists: bool = False
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
def by_kind(self, kind: str) -> list[SourceRole]:
|
||||
return [r for r in self.roles if r.kind == kind]
|
||||
|
||||
@property
|
||||
def goal(self) -> SourceRole | None:
|
||||
goals = self.by_kind("goal")
|
||||
return goals[0] if goals else None
|
||||
|
||||
def as_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"roles": [r.as_dict() for r in self.roles],
|
||||
"goal_has_exists": self.goal_has_exists,
|
||||
"notes": self.notes,
|
||||
}
|
||||
|
||||
|
||||
def _strings(value: Any) -> list[str]:
|
||||
if isinstance(value, str):
|
||||
value = [value]
|
||||
if not isinstance(value, list):
|
||||
return []
|
||||
return [str(v).strip() for v in value if str(v).strip()]
|
||||
|
||||
|
||||
def parse_source_simplex(payload: Any) -> SourceSimplex:
|
||||
"""Validate a model reply; raises ``ValueError`` when it is not a usable schema."""
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("roles"), list):
|
||||
raise ValueError("reply has no 'roles' list")
|
||||
roles: list[SourceRole] = []
|
||||
counters: dict[str, int] = {}
|
||||
notes: list[str] = []
|
||||
for item in payload["roles"]:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
kind = str(item.get("kind", "")).strip().lower()
|
||||
if kind not in ("object", "fact", "goal"):
|
||||
notes.append(f"dropped role with kind {kind!r}")
|
||||
continue
|
||||
introduced = str(item.get("introduced", "given")).strip().lower()
|
||||
counters[kind] = counters.get(kind, 0) + 1
|
||||
roles.append(SourceRole(
|
||||
kind=kind,
|
||||
symbols=_strings(item.get("symbols")),
|
||||
description=str(item.get("description", "")).strip(),
|
||||
keywords=[k.lower() for k in _strings(item.get("keywords"))],
|
||||
introduced=introduced if introduced in INTRODUCED else "given",
|
||||
alias=f"{kind}{counters[kind]}",
|
||||
))
|
||||
goals = [r for r in roles if r.kind == "goal"]
|
||||
if len(goals) != 1:
|
||||
raise ValueError(f"expected exactly one goal role, got {len(goals)}")
|
||||
if len(roles) < 2:
|
||||
raise ValueError("a simplex needs at least two roles")
|
||||
return SourceSimplex(roles, bool(payload.get("goal_has_exists")), notes)
|
||||
|
||||
|
||||
def passage_prompt(text: str, label: str = "") -> list[dict[str, str]]:
|
||||
header = f"Label: {label}\n\n" if label else ""
|
||||
body = re.sub(r"[ \t]+", " ", text).strip()
|
||||
return [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": f"{header}Passage:\n<<<BEGIN>>>\n{body}\n<<<END>>>"},
|
||||
]
|
||||
|
||||
|
||||
def extract_source_simplex(
|
||||
client: ChatClient, text: str, label: str = "", *, attempts: int = 2
|
||||
) -> SourceSimplex:
|
||||
"""Ask the model for the passage's roles; retry once if the reply is unusable."""
|
||||
messages = passage_prompt(text, label)
|
||||
last: ValueError | None = None
|
||||
for _ in range(attempts):
|
||||
payload = client.complete_json(messages) # type: ignore[attr-defined]
|
||||
try:
|
||||
return parse_source_simplex(payload)
|
||||
except ValueError as error:
|
||||
last = error
|
||||
messages = messages + [
|
||||
{"role": "assistant", "content": str(payload)[:3000]},
|
||||
{"role": "user", "content": f"Not usable: {error}. Reply with the JSON only."},
|
||||
]
|
||||
raise ValueError(f"no usable source simplex: {last}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"PROMPT_VERSION", "SYSTEM_PROMPT", "SourceRole", "SourceSimplex",
|
||||
"extract_source_simplex", "parse_json", "parse_source_simplex", "passage_prompt",
|
||||
]
|
||||
Reference in New Issue
Block a user