Zayden-Jung/lean-bench
2
1
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Drop the generated search index; derive it from the source instead

Browse Source 
The index file was 12KB of which ~96% was a byte-for-byte copy of
database/goodman-1.6.jsonl. Its only original content was three derived
fields: `name` (literally id.replace('-','_')), `status` (a two-line
computation over proof_checked), and `identifiers` (a regex over the
statement). It was a cache for work that costs microseconds, and it
created a real hazard: nothing checked that it matched its source, so
editing a statement and forgetting to regenerate would leave
search_database.py serving a premise whose text no longer matched the
generated .lean file.

- search_database.py now reads database/goodman-1.6.jsonl directly via a
  new load_database(), deriving name/identifiers/status on the fly and
  importing those functions from build_database.py so there is one
  definition of each rather than two that can drift.
- build_database.py drops --out-index and index generation, and its
  proof/status derivation is factored into resolve_proof() shared with
  retrieval.
- database/goodman-1.6.index.jsonl deleted.

Also fixes a retrieval bug the removal surfaced: the tokenizer only
matched ASCII identifiers, so a goal that is pure algebra
("a * (b + c) = a * b + a * c") tokenized to nothing and retrieved
nothing at all -- it now keeps type and operator symbols, and drops
single-letter variable names, which say nothing when matched. That goal
now correctly finds Proposition 1.6.1, and the identifier-rich goals
still rank their exact premise first by a wide margin (9.3 vs 4.4).

Tests restructured accordingly: the proof gate is now pinned directly on
resolve_proof in both directions, plus new coverage for load_database
deriving status from proof_checked and for the symbol tokenization.
49 tests, 48 green (the one failure remains the pre-existing unrelated
lean_agent import error in test_dataset.py).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017vbLFFJiHSZqGZHEkjvG5x
This commit is contained in:
Claude Bot
2026-09-18 19:50:10 +00:00
parent fbcb0c3383
commit 9e0457a0e1
6 changed files with 230 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_build_database.py
+61 -36
View File
@@ -1,22 +1,24 @@
import unittest
from textbook.build_database import build, decl_name, split_prelude, statement_identifiers
from textbook.build_database import build, decl_name, resolve_proof, split_prelude, statement_identifiers
ENTRIES = [
{"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
"statement": "\u2200 (a : \u2124), a + 0 = a", "prelude": "import Mathlib",
"statement": "∀ (a : ℤ), a + 0 = a", "prelude": "import Mathlib",
"natural_language": "Zero is an additive identity. [fix note: irrelevant]",
"requires": ["Int.add_zero"], "elaborates": True},
{"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
"statement": "\u2200 (as : List \u2124), IsGCDOfList 1 as", "prelude": "import Mathlib\ndef IsGCDOfList (d : \u2115) (as : List \u2124) : Prop := True",
"statement": "∀ (as : List ℤ), IsGCDOfList 1 as",
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
"natural_language": "", "requires": [], "elaborates": True},
{"id": "goodman-1-6-99", "source_label": "Proposition 1.6.99", "kind": "proposition",
"statement": "\u2200 (as : List \u2124), IsGCDOfList 2 as", "prelude": "import Mathlib\ndef IsGCDOfList (d : \u2115) (as : List \u2124) : Prop := True",
"statement": "∀ (as : List ℤ), IsGCDOfList 2 as",
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
"natural_language": "", "requires": [], "elaborates": True},
]
class BuildDatabaseTests(unittest.TestCase):
class HelperTests(unittest.TestCase):
def test_decl_name_is_a_legal_lean_identifier(self):
self.assertEqual(decl_name('goodman-1-6-2-a'), 'goodman_1_6_2_a')
@@ -25,60 +27,83 @@ class BuildDatabaseTests(unittest.TestCase):
self.assertEqual(imports, ['import Mathlib'])
self.assertEqual(other, 'def Foo := True')
def test_statement_identifiers_skips_stopwords(self):
self.assertNotIn('let', statement_identifiers('let x := 1'))
self.assertIn('Nat.Prime', statement_identifiers('Nat.Prime p'))
class ResolveProofTests(unittest.TestCase):
"""A proof only counts once Lean has verified it -- this is the gate that keeps
the library from claiming more than was actually checked."""
def test_no_proof_is_asserted(self):
self.assertEqual(resolve_proof({"id": "x"}), (None, 'asserted'))
def test_unverified_proof_is_pending_not_proved(self):
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": False})
self.assertEqual(status, 'proof_pending')
self.assertIsNone(proof)
def test_verified_proof_is_proved(self):
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": True})
self.assertEqual((proof, status), ('ring', 'proved'))
def test_successful_bench_run_proves_an_entry_with_no_recorded_proof(self):
result = {"success": True, "attempts": [{"proof": "simp"}]}
self.assertEqual(resolve_proof({"id": "x"}, result), ('simp', 'proved'))
def test_failed_bench_run_does_not_prove(self):
result = {"success": False, "attempts": []}
self.assertEqual(resolve_proof({"id": "x"}, result), (None, 'asserted'))
class BuildDatabaseTests(unittest.TestCase):
def test_unproven_entries_become_axioms(self):
text, index, emitted, skipped = build(ENTRIES)
text, emitted, skipped = build(ENTRIES)
self.assertIn('axiom goodman_1_6_1 :', text)
self.assertNotIn('theorem goodman_1_6_1 :', text)
self.assertEqual(len(emitted), 3)
self.assertEqual(skipped, [])
self.assertTrue(all(r['status'] == 'asserted' for r in index))
def test_proven_entries_become_real_theorems_with_their_proof(self):
results = {"goodman-1-6-1": {"id": "goodman-1-6-1", "success": True,
"attempts": [{"proof": "simp"}]}}
text, index, _emitted, _skipped = build(ENTRIES, results)
text, _emitted, _skipped = build(ENTRIES, results)
self.assertIn('theorem goodman_1_6_1 :', text)
self.assertNotIn('axiom goodman_1_6_1 :', text)
self.assertIn(' simp', text)
statuses = {r['id']: r['status'] for r in index}
self.assertEqual(statuses['goodman-1-6-1'], 'proved')
self.assertEqual(statuses['goodman-1-6-24'], 'asserted')
self.assertIn('Proof verified by Lean.', text)
def test_recorded_proof_is_only_emitted_once_checked(self):
pending = [dict(ENTRIES[0], proof='ring', proof_checked=False)]
text, _e, _s = build(pending)
self.assertIn('axiom goodman_1_6_1 :', text)
self.assertIn('not yet verified', text)
checked = [dict(ENTRIES[0], proof='ring', proof_checked=True)]
text, _e, _s = build(checked)
self.assertIn('theorem goodman_1_6_1 :', text)
self.assertIn(' ring', text)
def test_shared_def_in_prelude_is_emitted_once_not_per_entry(self):
# Two entries share the same `def`; declaring it twice in one Lean file is an
# "already declared" error, so it must be hoisted and deduplicated.
text, _index, _emitted, _skipped = build(ENTRIES)
text, _emitted, _skipped = build(ENTRIES)
self.assertEqual(text.count('def IsGCDOfList'), 1)
self.assertEqual(text.count('import Mathlib'), 1)
def test_require_checked_skips_unverified_entries(self):
entries = [dict(ENTRIES[0]), dict(ENTRIES[1], elaborates=False)]
_text, index, emitted, skipped = build(entries, require_checked=True)
text, emitted, skipped = build(entries, require_checked=True)
self.assertEqual(emitted, ['goodman-1-6-1'])
self.assertEqual(skipped, ['goodman-1-6-24'])
self.assertEqual(len(index), 1)
self.assertNotIn('goodman_1_6_24', text)
def test_recorded_proof_counts_only_once_checked(self):
# An unverified proof must NOT be emitted as a theorem -- the library may
# never claim more than Lean has confirmed.
pending = [dict(ENTRIES[0], proof='ring', proof_checked=False)]
text, index, _e, _s = build(pending)
self.assertIn('axiom goodman_1_6_1 :', text)
self.assertNotIn('theorem goodman_1_6_1 :', text)
self.assertEqual(index[0]['status'], 'proof_pending')
self.assertEqual(index[0]['proof'], 'ring')
checked = [dict(ENTRIES[0], proof='ring', proof_checked=True)]
text, index, _e, _s = build(checked)
self.assertIn('theorem goodman_1_6_1 :', text)
self.assertIn(' ring', text)
self.assertEqual(index[0]['status'], 'proved')
def test_index_carries_identifiers_and_strips_fix_notes(self):
_text, index, _emitted, _skipped = build(ENTRIES)
first = index[0]
self.assertIn('Int.add_zero', first['requires'])
self.assertNotIn('fix note', first['natural_language'])
self.assertNotIn('let', statement_identifiers('let x := 1'))
def test_docstring_records_provenance_and_axiom_warning(self):
text, _emitted, _skipped = build(ENTRIES)
self.assertIn('Proposition 1.6.1', text)
self.assertIn('ASSERTED from the textbook', text)
self.assertNotIn('fix note', text)
if __name__ == '__main__':