claude-bot
9e0457a0e1
The index file was 12KB of which ~96% was a byte-for-byte copy of
database/goodman-1.6.jsonl. Its only original content was three derived
fields: `name` (literally id.replace('-','_')), `status` (a two-line
computation over proof_checked), and `identifiers` (a regex over the
statement). It was a cache for work that costs microseconds, and it
created a real hazard: nothing checked that it matched its source, so
editing a statement and forgetting to regenerate would leave
search_database.py serving a premise whose text no longer matched the
generated .lean file.
- search_database.py now reads database/goodman-1.6.jsonl directly via a
new load_database(), deriving name/identifiers/status on the fly and
importing those functions from build_database.py so there is one
definition of each rather than two that can drift.
- build_database.py drops --out-index and index generation, and its
proof/status derivation is factored into resolve_proof() shared with
retrieval.
- database/goodman-1.6.index.jsonl deleted.
Also fixes a retrieval bug the removal surfaced: the tokenizer only
matched ASCII identifiers, so a goal that is pure algebra
("a * (b + c) = a * b + a * c") tokenized to nothing and retrieved
nothing at all -- it now keeps type and operator symbols, and drops
single-letter variable names, which say nothing when matched. That goal
now correctly finds Proposition 1.6.1, and the identifier-rich goals
still rank their exact premise first by a wide margin (9.3 vs 4.4).
Tests restructured accordingly: the proof gate is now pinned directly on
resolve_proof in both directions, plus new coverage for load_database
deriving status from proof_checked and for the symbol tokenization.
49 tests, 48 green (the one failure remains the pre-existing unrelated
lean_agent import error in test_dataset.py).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017vbLFFJiHSZqGZHEkjvG5x
111 lines
5.1 KiB
Python
111 lines
5.1 KiB
Python
import unittest
|
||
|
||
from textbook.build_database import build, decl_name, resolve_proof, split_prelude, statement_identifiers
|
||
|
||
ENTRIES = [
|
||
{"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
|
||
"statement": "∀ (a : ℤ), a + 0 = a", "prelude": "import Mathlib",
|
||
"natural_language": "Zero is an additive identity. [fix note: irrelevant]",
|
||
"requires": ["Int.add_zero"], "elaborates": True},
|
||
{"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
|
||
"statement": "∀ (as : List ℤ), IsGCDOfList 1 as",
|
||
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
|
||
"natural_language": "", "requires": [], "elaborates": True},
|
||
{"id": "goodman-1-6-99", "source_label": "Proposition 1.6.99", "kind": "proposition",
|
||
"statement": "∀ (as : List ℤ), IsGCDOfList 2 as",
|
||
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
|
||
"natural_language": "", "requires": [], "elaborates": True},
|
||
]
|
||
|
||
|
||
class HelperTests(unittest.TestCase):
|
||
def test_decl_name_is_a_legal_lean_identifier(self):
|
||
self.assertEqual(decl_name('goodman-1-6-2-a'), 'goodman_1_6_2_a')
|
||
|
||
def test_split_prelude_separates_imports_from_defs(self):
|
||
imports, other = split_prelude('import Mathlib\ndef Foo := True')
|
||
self.assertEqual(imports, ['import Mathlib'])
|
||
self.assertEqual(other, 'def Foo := True')
|
||
|
||
def test_statement_identifiers_skips_stopwords(self):
|
||
self.assertNotIn('let', statement_identifiers('let x := 1'))
|
||
self.assertIn('Nat.Prime', statement_identifiers('Nat.Prime p'))
|
||
|
||
|
||
class ResolveProofTests(unittest.TestCase):
|
||
"""A proof only counts once Lean has verified it -- this is the gate that keeps
|
||
the library from claiming more than was actually checked."""
|
||
|
||
def test_no_proof_is_asserted(self):
|
||
self.assertEqual(resolve_proof({"id": "x"}), (None, 'asserted'))
|
||
|
||
def test_unverified_proof_is_pending_not_proved(self):
|
||
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": False})
|
||
self.assertEqual(status, 'proof_pending')
|
||
self.assertIsNone(proof)
|
||
|
||
def test_verified_proof_is_proved(self):
|
||
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": True})
|
||
self.assertEqual((proof, status), ('ring', 'proved'))
|
||
|
||
def test_successful_bench_run_proves_an_entry_with_no_recorded_proof(self):
|
||
result = {"success": True, "attempts": [{"proof": "simp"}]}
|
||
self.assertEqual(resolve_proof({"id": "x"}, result), ('simp', 'proved'))
|
||
|
||
def test_failed_bench_run_does_not_prove(self):
|
||
result = {"success": False, "attempts": []}
|
||
self.assertEqual(resolve_proof({"id": "x"}, result), (None, 'asserted'))
|
||
|
||
|
||
class BuildDatabaseTests(unittest.TestCase):
|
||
def test_unproven_entries_become_axioms(self):
|
||
text, emitted, skipped = build(ENTRIES)
|
||
self.assertIn('axiom goodman_1_6_1 :', text)
|
||
self.assertNotIn('theorem goodman_1_6_1 :', text)
|
||
self.assertEqual(len(emitted), 3)
|
||
self.assertEqual(skipped, [])
|
||
|
||
def test_proven_entries_become_real_theorems_with_their_proof(self):
|
||
results = {"goodman-1-6-1": {"id": "goodman-1-6-1", "success": True,
|
||
"attempts": [{"proof": "simp"}]}}
|
||
text, _emitted, _skipped = build(ENTRIES, results)
|
||
self.assertIn('theorem goodman_1_6_1 :', text)
|
||
self.assertNotIn('axiom goodman_1_6_1 :', text)
|
||
self.assertIn(' simp', text)
|
||
self.assertIn('Proof verified by Lean.', text)
|
||
|
||
def test_recorded_proof_is_only_emitted_once_checked(self):
|
||
pending = [dict(ENTRIES[0], proof='ring', proof_checked=False)]
|
||
text, _e, _s = build(pending)
|
||
self.assertIn('axiom goodman_1_6_1 :', text)
|
||
self.assertIn('not yet verified', text)
|
||
|
||
checked = [dict(ENTRIES[0], proof='ring', proof_checked=True)]
|
||
text, _e, _s = build(checked)
|
||
self.assertIn('theorem goodman_1_6_1 :', text)
|
||
self.assertIn(' ring', text)
|
||
|
||
def test_shared_def_in_prelude_is_emitted_once_not_per_entry(self):
|
||
# Two entries share the same `def`; declaring it twice in one Lean file is an
|
||
# "already declared" error, so it must be hoisted and deduplicated.
|
||
text, _emitted, _skipped = build(ENTRIES)
|
||
self.assertEqual(text.count('def IsGCDOfList'), 1)
|
||
self.assertEqual(text.count('import Mathlib'), 1)
|
||
|
||
def test_require_checked_skips_unverified_entries(self):
|
||
entries = [dict(ENTRIES[0]), dict(ENTRIES[1], elaborates=False)]
|
||
text, emitted, skipped = build(entries, require_checked=True)
|
||
self.assertEqual(emitted, ['goodman-1-6-1'])
|
||
self.assertEqual(skipped, ['goodman-1-6-24'])
|
||
self.assertNotIn('goodman_1_6_24', text)
|
||
|
||
def test_docstring_records_provenance_and_axiom_warning(self):
|
||
text, _emitted, _skipped = build(ENTRIES)
|
||
self.assertIn('Proposition 1.6.1', text)
|
||
self.assertIn('ASSERTED from the textbook', text)
|
||
self.assertNotIn('fix note', text)
|
||
|
||
|
||
if __name__ == '__main__':
|
||
unittest.main()
|