Drop the generated search index; derive it from the source instead
The index file was 12KB of which ~96% was a byte-for-byte copy of
database/goodman-1.6.jsonl. Its only original content was three derived
fields: `name` (literally id.replace('-','_')), `status` (a two-line
computation over proof_checked), and `identifiers` (a regex over the
statement). It was a cache for work that costs microseconds, and it
created a real hazard: nothing checked that it matched its source, so
editing a statement and forgetting to regenerate would leave
search_database.py serving a premise whose text no longer matched the
generated .lean file.
- search_database.py now reads database/goodman-1.6.jsonl directly via a
new load_database(), deriving name/identifiers/status on the fly and
importing those functions from build_database.py so there is one
definition of each rather than two that can drift.
- build_database.py drops --out-index and index generation, and its
proof/status derivation is factored into resolve_proof() shared with
retrieval.
- database/goodman-1.6.index.jsonl deleted.
Also fixes a retrieval bug the removal surfaced: the tokenizer only
matched ASCII identifiers, so a goal that is pure algebra
("a * (b + c) = a * b + a * c") tokenized to nothing and retrieved
nothing at all -- it now keeps type and operator symbols, and drops
single-letter variable names, which say nothing when matched. That goal
now correctly finds Proposition 1.6.1, and the identifier-rich goals
still rank their exact premise first by a wide margin (9.3 vs 4.4).
Tests restructured accordingly: the proof gate is now pinned directly on
resolve_proof in both directions, plus new coverage for load_database
deriving status from proof_checked and for the symbol tokenization.
49 tests, 48 green (the one failure remains the pre-existing unrelated
lean_agent import error in test_dataset.py).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017vbLFFJiHSZqGZHEkjvG5x
This commit is contained in:
+58
-12
@@ -7,6 +7,13 @@ for entries that might apply to the goal in front of it. The ranked results are
|
||||
gets shown to the LLM alongside the goal, so the next line of Lean it writes can
|
||||
cite a real premise by name instead of inventing a lemma.
|
||||
|
||||
It reads the database JSONL directly and derives each premise's declaration name,
|
||||
identifiers and proof status on the fly, sharing those functions with
|
||||
build_database.py. There is no precomputed index: at this scale deriving them costs
|
||||
microseconds, and a second generated copy of the data would only create something
|
||||
that can silently go stale against the source. A precomputed index earns its place
|
||||
when it holds something genuinely expensive, i.e. embeddings.
|
||||
|
||||
Scoring here is deliberately simple and dependency-free: overlap of identifiers and
|
||||
symbols between the goal and each premise, weighted so that a rare identifier
|
||||
matters more than a common one (`IsGCDOfList` appearing in both is a much stronger
|
||||
@@ -23,22 +30,61 @@ import re
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
TOKEN_RE = re.compile(r'[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+|[A-Za-z_][A-Za-z0-9_]*|[∣∀∃↔→∧∨¬≤≥≠∈]')
|
||||
# Qualified names (Nat.Prime) first so they aren't split, then bare identifiers, then
|
||||
# the type and operator symbols that carry signal. Arithmetic operators are included
|
||||
# deliberately: a goal that is pure algebra ("a * (b + c) = a * b + a * c") has no
|
||||
# identifiers at all, and without these it would tokenize to nothing and retrieve
|
||||
# nothing. They appear in nearly every premise, so inverse-document-frequency
|
||||
# weighting already discounts them to almost zero -- they only decide ties.
|
||||
TOKEN_RE = re.compile(
|
||||
r'[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+'
|
||||
r'|[A-Za-z_][A-Za-z0-9_]*'
|
||||
r'|[∣∀∃↔→∧∨¬≤≥≠∈ℤℕℚℝ×∑∏]'
|
||||
r'|[+*=/-]')
|
||||
STOPWORDS = {'let', 'in', 'fun', 'if', 'then', 'else', 'true', 'false', 'Prop', 'Type', 'this'}
|
||||
|
||||
|
||||
def load_index(path):
|
||||
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||
try:
|
||||
from .build_database import decl_name, resolve_proof, statement_identifiers
|
||||
except ImportError:
|
||||
from build_database import decl_name, resolve_proof, statement_identifiers
|
||||
|
||||
|
||||
def load_database(path, results=None):
|
||||
"""Database entries in the shape search() ranks, derived from the source file."""
|
||||
entries = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||
records = []
|
||||
for entry in entries:
|
||||
proof, status = resolve_proof(entry, (results or {}).get(entry['id']))
|
||||
records.append({
|
||||
"name": decl_name(entry['id']),
|
||||
"id": entry['id'],
|
||||
"source_label": entry.get('source_label'),
|
||||
"kind": entry.get('kind'),
|
||||
"status": status,
|
||||
"proof": proof or entry.get('proof'),
|
||||
"statement": entry['statement'],
|
||||
"natural_language": entry.get('natural_language', '').split('[fix note:')[0].strip(),
|
||||
"identifiers": statement_identifiers(entry['statement']),
|
||||
"requires": entry.get('requires', []),
|
||||
})
|
||||
return records
|
||||
|
||||
|
||||
SYMBOLS = set('∣∀∃↔→∧∨¬≤≥≠∈ℤℕℚℝ×∑∏+*=/-')
|
||||
|
||||
|
||||
def tokenize(text):
|
||||
return [t for t in TOKEN_RE.findall(text or '') if t not in STOPWORDS and len(t) > 1]
|
||||
"""Identifiers and meaningful symbols. Single-letter variable names are dropped
|
||||
(matching `a` against `a` says nothing), but single-character symbols are kept."""
|
||||
return [t for t in TOKEN_RE.findall(text or '')
|
||||
if t not in STOPWORDS and (len(t) > 1 or t in SYMBOLS)]
|
||||
|
||||
|
||||
def document_frequencies(index):
|
||||
def document_frequencies(records):
|
||||
"""How many premises each token appears in, for inverse-document-frequency weighting."""
|
||||
frequencies = Counter()
|
||||
for record in index:
|
||||
for record in records:
|
||||
for token in set(tokenize(record['statement']) + record.get('identifiers', [])
|
||||
+ record.get('requires', [])):
|
||||
frequencies[token] += 1
|
||||
@@ -57,13 +103,13 @@ def score_premise(goal_tokens, record, frequencies, total):
|
||||
return score
|
||||
|
||||
|
||||
def search(index, goal, limit=5):
|
||||
def search(records, goal, limit=5):
|
||||
"""Premises ranked against one goal, best first. Zero-score entries are dropped."""
|
||||
frequencies = document_frequencies(index)
|
||||
total = max(len(index), 1)
|
||||
frequencies = document_frequencies(records)
|
||||
total = max(len(records), 1)
|
||||
goal_tokens = tokenize(goal)
|
||||
scored = []
|
||||
for record in index:
|
||||
for record in records:
|
||||
score = score_premise(goal_tokens, record, frequencies, total)
|
||||
if score > 0:
|
||||
scored.append((score, record))
|
||||
@@ -85,13 +131,13 @@ def format_for_prompt(hits):
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Rank database premises against a proof goal')
|
||||
parser.add_argument('index', type=Path, help='search index from build_database.py')
|
||||
parser.add_argument('database', type=Path, help='database JSONL, e.g. database/goodman-1.6.jsonl')
|
||||
parser.add_argument('goal', help='the current goal, as Lean prints it')
|
||||
parser.add_argument('--limit', type=int, default=5)
|
||||
parser.add_argument('--json', action='store_true', help='emit raw JSON instead of prompt text')
|
||||
args = parser.parse_args()
|
||||
|
||||
hits = search(load_index(args.index), args.goal, args.limit)
|
||||
hits = search(load_database(args.database), args.goal, args.limit)
|
||||
print(json.dumps(hits, ensure_ascii=False, indent=2) if args.json else format_for_prompt(hits))
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user