Drop the generated search index; derive it from the source instead

The index file was 12KB of which ~96% was a byte-for-byte copy of
database/goodman-1.6.jsonl. Its only original content was three derived
fields: `name` (literally id.replace('-','_')), `status` (a two-line
computation over proof_checked), and `identifiers` (a regex over the
statement). It was a cache for work that costs microseconds, and it
created a real hazard: nothing checked that it matched its source, so
editing a statement and forgetting to regenerate would leave
search_database.py serving a premise whose text no longer matched the
generated .lean file.

- search_database.py now reads database/goodman-1.6.jsonl directly via a
  new load_database(), deriving name/identifiers/status on the fly and
  importing those functions from build_database.py so there is one
  definition of each rather than two that can drift.
- build_database.py drops --out-index and index generation, and its
  proof/status derivation is factored into resolve_proof() shared with
  retrieval.
- database/goodman-1.6.index.jsonl deleted.

Also fixes a retrieval bug the removal surfaced: the tokenizer only
matched ASCII identifiers, so a goal that is pure algebra
("a * (b + c) = a * b + a * c") tokenized to nothing and retrieved
nothing at all -- it now keeps type and operator symbols, and drops
single-letter variable names, which say nothing when matched. That goal
now correctly finds Proposition 1.6.1, and the identifier-rich goals
still rank their exact premise first by a wide margin (9.3 vs 4.4).

Tests restructured accordingly: the proof gate is now pinned directly on
resolve_proof in both directions, plus new coverage for load_database
deriving status from proof_checked and for the symbol tokenization.
49 tests, 48 green (the one failure remains the pre-existing unrelated
lean_agent import error in test_dataset.py).

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017vbLFFJiHSZqGZHEkjvG5x

tests/test_search_database.py

@@ -1,6 +1,10 @@
 import unittest
 
-from textbook.search_database import format_for_prompt, search, tokenize
+import json
+import tempfile
+from pathlib import Path
+
+from textbook.search_database import format_for_prompt, load_database, search, tokenize
 
 INDEX = [
     {"name": "goodman_1_6_20", "id": "goodman-1-6-20", "source_label": "Corollary 1.6.20",
@@ -18,12 +22,63 @@ INDEX = [
 ]
 
 
+SOURCE_ENTRIES = [
+    {"id": "goodman-1-6-20", "source_label": "Corollary 1.6.20", "kind": "corollary",
+     "statement": "\u2200 (p : \u2115), Nat.Prime p \u2192 (p:\u2124) \u2223 List.prod as",
+     "prelude": "import Mathlib", "natural_language": "x [fix note: dropped]",
+     "requires": ["Nat.Prime"]},
+    {"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
+     "statement": "\u2200 (as : List \u2124), \u2203 d, IsGCDOfList d as",
+     "prelude": "import Mathlib", "natural_language": "", "requires": [],
+     "proof": "exact foo", "proof_checked": True},
+    {"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
+     "statement": "\u2200 (a : \u2124), a + 0 = a", "prelude": "import Mathlib",
+     "natural_language": "", "requires": [], "proof": "ring", "proof_checked": False},
+]
+
+
+class LoadDatabaseTests(unittest.TestCase):
+    """load_database derives name/status/identifiers from the source file, so there is
+    no separate index that could go stale against it."""
+
+    def _load(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'db.jsonl'
+            path.write_text('\n'.join(json.dumps(e) for e in SOURCE_ENTRIES) + '\n',
+                            encoding='utf-8')
+            return load_database(path)
+
+    def test_derives_lean_name_and_strips_fix_notes(self):
+        records = {r['id']: r for r in self._load()}
+        self.assertEqual(records['goodman-1-6-20']['name'], 'goodman_1_6_20')
+        self.assertNotIn('fix note', records['goodman-1-6-20']['natural_language'])
+
+    def test_derives_proof_status_from_proof_checked(self):
+        records = {r['id']: r for r in self._load()}
+        self.assertEqual(records['goodman-1-6-24']['status'], 'proved')
+        self.assertEqual(records['goodman-1-6-1']['status'], 'proof_pending')
+        self.assertEqual(records['goodman-1-6-20']['status'], 'asserted')
+
+    def test_search_works_directly_on_loaded_source(self):
+        hits = search(self._load(), '\u22a2 IsGCDOfList d as', limit=2)
+        self.assertEqual(hits[0]['name'], 'goodman_1_6_24')
+
+
 class SearchDatabaseTests(unittest.TestCase):
     def test_tokenize_keeps_qualified_names_and_drops_stopwords(self):
         tokens = tokenize('let x := Nat.Prime p')
         self.assertIn('Nat.Prime', tokens)
         self.assertNotIn('let', tokens)
 
+    def test_tokenize_keeps_symbols_but_drops_single_letter_variables(self):
+        # A pure-algebra goal has no identifiers at all, so the symbols have to
+        # survive or it retrieves nothing; matching 'a' against 'a' says nothing.
+        tokens = tokenize('\u2200 (a b : \u2124), a * (b + c) = 0')
+        self.assertIn('\u2124', tokens)
+        self.assertIn('*', tokens)
+        self.assertIn('+', tokens)
+        self.assertNotIn('a', tokens)
+
     def test_exact_premise_ranks_first(self):
         hits = search(INDEX, '\u22a2 IsGCDOfList d as', limit=3)
         self.assertEqual(hits[0]['name'], 'goodman_1_6_24')