Drop the generated search index; derive it from the source instead
The index file was 12KB of which ~96% was a byte-for-byte copy of
database/goodman-1.6.jsonl. Its only original content was three derived
fields: `name` (literally id.replace('-','_')), `status` (a two-line
computation over proof_checked), and `identifiers` (a regex over the
statement). It was a cache for work that costs microseconds, and it
created a real hazard: nothing checked that it matched its source, so
editing a statement and forgetting to regenerate would leave
search_database.py serving a premise whose text no longer matched the
generated .lean file.
- search_database.py now reads database/goodman-1.6.jsonl directly via a
new load_database(), deriving name/identifiers/status on the fly and
importing those functions from build_database.py so there is one
definition of each rather than two that can drift.
- build_database.py drops --out-index and index generation, and its
proof/status derivation is factored into resolve_proof() shared with
retrieval.
- database/goodman-1.6.index.jsonl deleted.
Also fixes a retrieval bug the removal surfaced: the tokenizer only
matched ASCII identifiers, so a goal that is pure algebra
("a * (b + c) = a * b + a * c") tokenized to nothing and retrieved
nothing at all -- it now keeps type and operator symbols, and drops
single-letter variable names, which say nothing when matched. That goal
now correctly finds Proposition 1.6.1, and the identifier-rich goals
still rank their exact premise first by a wide margin (9.3 vs 4.4).
Tests restructured accordingly: the proof gate is now pinned directly on
resolve_proof in both directions, plus new coverage for load_database
deriving status from proof_checked and for the symbol tokenization.
49 tests, 48 green (the one failure remains the pre-existing unrelated
lean_agent import error in test_dataset.py).
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017vbLFFJiHSZqGZHEkjvG5x
This commit is contained in:
@@ -1,21 +0,0 @@
|
|||||||
{"name": "goodman_1_6_1", "id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition", "status": "proof_pending", "proof": "intro a b c\nrefine ⟨by ring, by ring, by ring, by ring, by ring, by ring, by ring, by ring, ?_⟩\nintro x y h\nexact ⟨fun hx => h (by simp [hx]), fun hy => h (by simp [hy])⟩", "statement": "∀ (a b c : ℤ), (a + b = b + a) ∧ (a + (b + c) = (a + b) + c) ∧ (0 + a = a) ∧ (a + (-a) = 0) ∧ (a * b = b * a) ∧ (a * (b * c) = (a * b) * c) ∧ (1 * a = a) ∧ (a * (b + c) = a * b + a * c) ∧ (∀ (x y : ℤ), x * y ≠ 0 → x ≠ 0 ∧ y ≠ 0)", "natural_language": "Addition and multiplication on the integers are commutative and associative, 0 and 1 are their respective identities, the distributive law holds, and a product of integers is nonzero iff both factors are nonzero.", "identifiers": [], "requires": ["Int.add_comm", "Int.add_assoc", "Int.zero_add", "Int.add_left_neg", "Int.mul_comm", "Int.mul_assoc", "Int.one_mul", "Int.mul_add", "Int.mul_ne_zero"]}
|
|
||||||
{"name": "goodman_1_6_4", "id": "goodman-1-6-4", "source_label": "Proposition 1.6.4", "kind": "proposition", "status": "proof_pending", "proof": "intro n hn\nobtain ⟨p, hp, hpd⟩ := Nat.exists_prime_and_dvd (show n ≠ 1 by omega)\nexact ⟨p, hp, hpd⟩", "statement": "∀ (n : ℕ), n > 1 → ∃ (p : ℕ) (h : Nat.Prime p), p ∣ n", "natural_language": "Any natural number greater than 1 can be written as a product of prime numbers.", "identifiers": ["Nat.Prime"], "requires": ["Nat.exists_prime_factor"]}
|
|
||||||
{"name": "goodman_1_6_6", "id": "goodman-1-6-6", "source_label": "Theorem 1.6.6", "kind": "theorem", "status": "asserted", "proof": null, "statement": "∃ (p : ℕ → ℕ), ∀ (n : ℕ), Nat.Prime (p n) ∧ ∀ (i j : ℕ), i ≠ j → p i ≠ p j", "natural_language": "There are infinitely many prime numbers.", "identifiers": ["Nat.Prime"], "requires": ["Nat.infinite_primes"]}
|
|
||||||
{"name": "goodman_1_6_7", "id": "goodman-1-6-7", "source_label": "Proposition 1.6.7", "kind": "proposition", "status": "asserted", "proof": null, "statement": "∀ (a d : ℤ), d ≥ 1 → ∃! qr : ℤ × ℤ, a = qr.1 * d + qr.2 ∧ 0 ≤ qr.2 ∧ qr.2 < d", "natural_language": "Given integers a and d with d ≥ 1, there exist unique integers q and r such that a = qd + r and 0 ≤ r < d.", "identifiers": ["qr"], "requires": ["Int.division_theorem"]}
|
|
||||||
{"name": "goodman_1_6_9", "id": "goodman-1-6-9", "source_label": "Proposition 1.6.9", "kind": "proposition", "status": "asserted", "proof": null, "statement": "∀ (m n : ℤ), let I := {v : ℤ | ∃ a b : ℤ, v = a * m + b * n}; (∀ x ∈ I, ∀ y ∈ I, x + y ∈ I) ∧ (∀ x ∈ I, -x ∈ I) ∧ (∀ x : ℤ, ∀ z ∈ I, x * z ∈ I) ∧ (∀ b : ℤ, b ∣ m ∧ b ∣ n → ∀ z ∈ I, b ∣ z)", "natural_language": "I(m,n) = {am+bn} is closed under addition and negation, absorbs multiplication by any integer, and any common divisor of m and n divides every element of I(m,n).", "identifiers": [], "requires": []}
|
|
||||||
{"name": "goodman_1_6_10", "id": "goodman-1-6-10", "source_label": "Lemma 1.6.10", "kind": "lemma", "status": "asserted", "proof": null, "statement": "∀ (m n d : ℤ), d ∣ m ∧ d ∣ n ∧ d ∈ {v : ℤ | ∃ a b : ℤ, v = a * m + b * n} → ∀ (x : ℤ), x ∣ m ∧ x ∣ n → x ∣ d", "natural_language": "If a natural number d is a common divisor of m and n and an element of I(m, n), then d is the greatest common divisor of m and n.", "identifiers": [], "requires": ["Int.gcd_is_greatest_common_divisor"]}
|
|
||||||
{"name": "goodman_1_6_11", "id": "goodman-1-6-11", "source_label": "Proposition 1.6.11", "kind": "proposition", "status": "asserted", "proof": null, "statement": "∀ (m n : ℤ), m ≠ 0 ∧ n ≠ 0 → let d := Nat.gcd (Int.natAbs m) (Int.natAbs n); (d : ℤ) = Int.gcd m n ∧ (d : ℤ) ∈ {v : ℤ | ∃ a b : ℤ, v = a * m + b * n}", "natural_language": "The natural number d, which is the greatest common divisor of the absolute values of m and n, is also the greatest common divisor of m and n, and d is an element of I(m, n).", "identifiers": ["Nat.gcd", "Int.natAbs", "Int.gcd"], "requires": ["Int.gcd_nat_abs", "Int.gcd_linear_combination"]}
|
|
||||||
{"name": "goodman_1_6_13", "id": "goodman-1-6-13", "source_label": "Corollary 1.6.13", "kind": "corollary", "status": "asserted", "proof": null, "statement": "∀ (m n : ℤ), m ≠ 0 → n ≠ 0 → let I := {v : ℤ | ∃ a b : ℤ, v = a * m + b * n}; let d := Int.gcd m n; IsLeast {x : ℤ | 0 < x ∧ x ∈ I} (d : ℤ) ∧ I = {v : ℤ | ∃ k : ℤ, v = k * d}", "natural_language": "gcd(m,n) is the least positive element of I(m,n), and I(m,n) is exactly the set of integer multiples of gcd(m,n).", "identifiers": ["Int.gcd", "IsLeast"], "requires": ["IsLeast", "Int.gcd"]}
|
|
||||||
{"name": "goodman_1_6_15", "id": "goodman-1-6-15", "source_label": "Corollary 1.6.15", "kind": "corollary", "status": "proof_pending", "proof": "intro m n _\nrw [Int.gcd_eq_one_iff_coprime]\nconstructor\n· intro h\n obtain ⟨u, v, huv⟩ := h\n exact ⟨u, v, huv⟩\n· intro h\n obtain ⟨s, t, hst⟩ := h\n exact ⟨s, t, hst⟩", "statement": "∀ (m n : ℤ), m ≠ 0 ∧ n ≠ 0 → Int.gcd m n = 1 ↔ ∃ (s t : ℤ), s * m + t * n = 1", "natural_language": "Two nonzero integers m and n are relatively prime if and only if there exist integers s and t such that sm + tn = 1.", "identifiers": ["Int.gcd"], "requires": ["Int.coprime_iff_gcd_eq_one"]}
|
|
||||||
{"name": "goodman_1_6_17", "id": "goodman-1-6-17", "source_label": "Corollary 1.6.17", "kind": "corollary", "status": "proof_pending", "proof": "intro a b x h\nexact IsCoprime.mul_dvd (Nat.isCoprime_iff_coprime.mpr h.1) h.2.1 h.2.2", "statement": "∀ (a b : ℕ) (x : ℤ), Nat.Coprime a b ∧ (a:ℤ) ∣ x ∧ (b:ℤ) ∣ x → (a:ℤ) * b ∣ x", "natural_language": "If a and b are relatively prime natural numbers and both divide an integer x, then their product ab also divides x.", "identifiers": ["Nat.Coprime"], "requires": ["Nat.coprime_mul_dvd"]}
|
|
||||||
{"name": "goodman_1_6_18", "id": "goodman-1-6-18", "source_label": "Proposition 1.6.18", "kind": "proposition", "status": "asserted", "proof": null, "statement": "∀ (p : ℕ) (a : ℤ), Nat.Prime p ∧ a ≠ 0 → (p:ℤ) ∣ a ∨ IsCoprime (p:ℤ) a", "natural_language": "If p is a prime number and a is any nonzero integer, then either p divides a or p and a are relatively prime.", "identifiers": ["Nat.Prime", "IsCoprime"], "requires": ["Int.prime_or_coprime"]}
|
|
||||||
{"name": "goodman_1_6_19", "id": "goodman-1-6-19", "source_label": "Proposition 1.6.19", "kind": "proposition", "status": "proof_pending", "proof": "intro p a b h hdvd\nexact Int.Prime.dvd_mul' h.1 hdvd", "statement": "∀ (p : ℕ) (a b : ℤ), Nat.Prime p ∧ a ≠ 0 ∧ b ≠ 0 → (p:ℤ) ∣ a * b → (p:ℤ) ∣ a ∨ (p:ℤ) ∣ b", "natural_language": "If p is a prime number and a and b are nonzero integers, and p divides the product ab, then p divides a or p divides b.", "identifiers": ["Nat.Prime"], "requires": ["Int.prime_dvd_mul"]}
|
|
||||||
{"name": "goodman_1_6_20", "id": "goodman-1-6-20", "source_label": "Corollary 1.6.20", "kind": "corollary", "status": "proof_pending", "proof": "intro p as hp _ h\nobtain ⟨a, ha, hpa⟩ := (Nat.Prime.prime_int hp).dvd_prod_iff.mp h\nexact ⟨a, ha, hpa⟩", "statement": "∀ (p : ℕ) (as : List ℤ), Nat.Prime p → (∀ (a : ℤ), a ∈ as → a ≠ 0) → (p:ℤ) ∣ List.prod as → ∃ (a : ℤ), a ∈ as ∧ (p:ℤ) ∣ a", "natural_language": "If a prime number p divides a product of nonzero integers, then p divides one of the factors.", "identifiers": ["as", "List", "Nat.Prime", "List.prod"], "requires": ["Int.prime_dvd_prod"]}
|
|
||||||
{"name": "goodman_1_6_21", "id": "goodman-1-6-21", "source_label": "Theorem 1.6.21", "kind": "theorem", "status": "asserted", "proof": null, "statement": "∀ (n : ℕ), n > 1 → ∃! (ps : List ℕ), (∀ (p : ℕ), p ∈ ps → Nat.Prime p) ∧ n = List.prod ps", "natural_language": "The prime factorization of a natural number is unique.", "identifiers": ["ps", "List", "Nat.Prime", "List.prod"], "requires": ["Nat.unique_prime_factorization"]}
|
|
||||||
{"name": "goodman_1_6_23", "id": "goodman-1-6-23", "source_label": "Lemma 1.6.23", "kind": "lemma", "status": "asserted", "proof": null, "statement": "∀ (n : ℕ) (a : Fin n → ℤ), 2 ≤ n → (∀ i, a i ≠ 0) → ∃ (d : ℕ) (Q : Matrix (Fin n) (Fin n) ℤ), IsUnit Q.det ∧ ∀ j : Fin n, Matrix.vecMul a Q j = if j.val = 0 then (d : ℤ) else 0", "natural_language": "Given n >= 2 nonzero integers a_1..a_n, there is a natural number d and a unimodular (det a unit, i.e. det = +-1) integer n-by-n matrix Q such that the row vector a times Q equals (d, 0, ..., 0).", "identifiers": ["Fin", "Matrix", "IsUnit", "Q.det", "Matrix.vecMul", "j.val"], "requires": ["Matrix.vecMul", "Matrix.det", "IsUnit"]}
|
|
||||||
{"name": "goodman_1_6_24", "id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition", "status": "asserted", "proof": null, "statement": "∀ (as : List ℤ), as.length ≥ 2 → (∀ (a : ℤ), a ∈ as → a ≠ 0) → ∃ (d : ℕ) (cs : List ℤ), IsGCDOfList d as ∧ (d:ℤ) = List.sum (List.zipWith (· * ·) cs as)", "natural_language": "The greatest common divisor of a list of at least two nonzero integers exists and is an integer linear combination of the integers in the list.", "identifiers": ["as", "List", "as.length", "cs", "IsGCDOfList", "List.sum", "List.zipWith"], "requires": ["GCD.gcd_exists", "GCD.gcd_linear_combination"]}
|
|
||||||
{"name": "goodman_1_6_26", "id": "goodman-1-6-26", "source_label": "Lemma 1.6.26", "kind": "lemma", "status": "asserted", "proof": null, "statement": "∀ (as : List ℤ) (z : ℤ), as.length ≥ 2 → (∀ (i j : ℕ), i ≠ j → i < as.length → j < as.length → IsCoprime (as[i]) (as[j])) → (∀ (a : ℤ), a ∈ as → a ≠ 0) → (∀ (a : ℤ), a ∈ as → a ∣ z) → List.prod as ∣ z", "natural_language": "If a list of integers is pairwise relatively prime and each integer in the list divides an integer z, then the product of the integers in the list also divides z.", "identifiers": ["as", "List", "as.length", "IsCoprime", "List.prod"], "requires": ["Int.pairwise_coprime_mul_dvd"]}
|
|
||||||
{"name": "goodman_1_6_2_a", "id": "goodman-1-6-2-a", "source_label": "Proposition 1.6.2", "kind": "proposition", "status": "proof_pending", "proof": "intro u v h\nexact Int.mul_eq_one_iff_eq_one_or_neg_one.mp h", "statement": "∀ (u v : ℤ), u * v = 1 → (u = 1 ∧ v = 1) ∨ (u = -1 ∧ v = -1)", "natural_language": "If two integers multiply to 1, both are 1 or both are -1.", "identifiers": [], "requires": ["Int.eq_one_or_self_of_prime"]}
|
|
||||||
{"name": "goodman_1_6_2_b", "id": "goodman-1-6-2-b", "source_label": "Proposition 1.6.2", "kind": "proposition", "status": "proof_pending", "proof": "intro a b h\nexact Int.associated_iff.mp (associated_of_dvd_dvd h.1 h.2)", "statement": "∀ (a b : ℤ), a ∣ b ∧ b ∣ a → a = b ∨ a = -b", "natural_language": "If two integers divide each other, they are equal or negatives of each other.", "identifiers": [], "requires": ["Int.dvd_antisymm"]}
|
|
||||||
{"name": "goodman_1_6_2_c", "id": "goodman-1-6-2-c", "source_label": "Proposition 1.6.2", "kind": "proposition", "status": "proof_pending", "proof": "intro a b c h\nexact dvd_trans h.1 h.2", "statement": "∀ (a b c : ℤ), a ∣ b ∧ b ∣ c → a ∣ c", "natural_language": "Divisibility of integers is transitive.", "identifiers": [], "requires": ["Int.dvd_trans"]}
|
|
||||||
{"name": "goodman_1_6_2_d", "id": "goodman-1-6-2-d", "source_label": "Proposition 1.6.2", "kind": "proposition", "status": "proof_pending", "proof": "intro a b c s t h\nexact dvd_add (h.1.mul_left s) (h.2.mul_left t)", "statement": "∀ (a b c s t : ℤ), a ∣ b ∧ a ∣ c → a ∣ (s * b + t * c)", "natural_language": "If an integer divides two others, it divides any integer linear combination of them.", "identifiers": [], "requires": ["dvd_add", "Dvd.dvd.mul_left"]}
|
|
||||||
@@ -1,22 +1,24 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from textbook.build_database import build, decl_name, split_prelude, statement_identifiers
|
from textbook.build_database import build, decl_name, resolve_proof, split_prelude, statement_identifiers
|
||||||
|
|
||||||
ENTRIES = [
|
ENTRIES = [
|
||||||
{"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
|
{"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
|
||||||
"statement": "\u2200 (a : \u2124), a + 0 = a", "prelude": "import Mathlib",
|
"statement": "∀ (a : ℤ), a + 0 = a", "prelude": "import Mathlib",
|
||||||
"natural_language": "Zero is an additive identity. [fix note: irrelevant]",
|
"natural_language": "Zero is an additive identity. [fix note: irrelevant]",
|
||||||
"requires": ["Int.add_zero"], "elaborates": True},
|
"requires": ["Int.add_zero"], "elaborates": True},
|
||||||
{"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
|
{"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
|
||||||
"statement": "\u2200 (as : List \u2124), IsGCDOfList 1 as", "prelude": "import Mathlib\ndef IsGCDOfList (d : \u2115) (as : List \u2124) : Prop := True",
|
"statement": "∀ (as : List ℤ), IsGCDOfList 1 as",
|
||||||
|
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
|
||||||
"natural_language": "", "requires": [], "elaborates": True},
|
"natural_language": "", "requires": [], "elaborates": True},
|
||||||
{"id": "goodman-1-6-99", "source_label": "Proposition 1.6.99", "kind": "proposition",
|
{"id": "goodman-1-6-99", "source_label": "Proposition 1.6.99", "kind": "proposition",
|
||||||
"statement": "\u2200 (as : List \u2124), IsGCDOfList 2 as", "prelude": "import Mathlib\ndef IsGCDOfList (d : \u2115) (as : List \u2124) : Prop := True",
|
"statement": "∀ (as : List ℤ), IsGCDOfList 2 as",
|
||||||
|
"prelude": "import Mathlib\ndef IsGCDOfList (d : ℕ) (as : List ℤ) : Prop := True",
|
||||||
"natural_language": "", "requires": [], "elaborates": True},
|
"natural_language": "", "requires": [], "elaborates": True},
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
class BuildDatabaseTests(unittest.TestCase):
|
class HelperTests(unittest.TestCase):
|
||||||
def test_decl_name_is_a_legal_lean_identifier(self):
|
def test_decl_name_is_a_legal_lean_identifier(self):
|
||||||
self.assertEqual(decl_name('goodman-1-6-2-a'), 'goodman_1_6_2_a')
|
self.assertEqual(decl_name('goodman-1-6-2-a'), 'goodman_1_6_2_a')
|
||||||
|
|
||||||
@@ -25,60 +27,83 @@ class BuildDatabaseTests(unittest.TestCase):
|
|||||||
self.assertEqual(imports, ['import Mathlib'])
|
self.assertEqual(imports, ['import Mathlib'])
|
||||||
self.assertEqual(other, 'def Foo := True')
|
self.assertEqual(other, 'def Foo := True')
|
||||||
|
|
||||||
|
def test_statement_identifiers_skips_stopwords(self):
|
||||||
|
self.assertNotIn('let', statement_identifiers('let x := 1'))
|
||||||
|
self.assertIn('Nat.Prime', statement_identifiers('Nat.Prime p'))
|
||||||
|
|
||||||
|
|
||||||
|
class ResolveProofTests(unittest.TestCase):
|
||||||
|
"""A proof only counts once Lean has verified it -- this is the gate that keeps
|
||||||
|
the library from claiming more than was actually checked."""
|
||||||
|
|
||||||
|
def test_no_proof_is_asserted(self):
|
||||||
|
self.assertEqual(resolve_proof({"id": "x"}), (None, 'asserted'))
|
||||||
|
|
||||||
|
def test_unverified_proof_is_pending_not_proved(self):
|
||||||
|
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": False})
|
||||||
|
self.assertEqual(status, 'proof_pending')
|
||||||
|
self.assertIsNone(proof)
|
||||||
|
|
||||||
|
def test_verified_proof_is_proved(self):
|
||||||
|
proof, status = resolve_proof({"id": "x", "proof": "ring", "proof_checked": True})
|
||||||
|
self.assertEqual((proof, status), ('ring', 'proved'))
|
||||||
|
|
||||||
|
def test_successful_bench_run_proves_an_entry_with_no_recorded_proof(self):
|
||||||
|
result = {"success": True, "attempts": [{"proof": "simp"}]}
|
||||||
|
self.assertEqual(resolve_proof({"id": "x"}, result), ('simp', 'proved'))
|
||||||
|
|
||||||
|
def test_failed_bench_run_does_not_prove(self):
|
||||||
|
result = {"success": False, "attempts": []}
|
||||||
|
self.assertEqual(resolve_proof({"id": "x"}, result), (None, 'asserted'))
|
||||||
|
|
||||||
|
|
||||||
|
class BuildDatabaseTests(unittest.TestCase):
|
||||||
def test_unproven_entries_become_axioms(self):
|
def test_unproven_entries_become_axioms(self):
|
||||||
text, index, emitted, skipped = build(ENTRIES)
|
text, emitted, skipped = build(ENTRIES)
|
||||||
self.assertIn('axiom goodman_1_6_1 :', text)
|
self.assertIn('axiom goodman_1_6_1 :', text)
|
||||||
|
self.assertNotIn('theorem goodman_1_6_1 :', text)
|
||||||
self.assertEqual(len(emitted), 3)
|
self.assertEqual(len(emitted), 3)
|
||||||
self.assertEqual(skipped, [])
|
self.assertEqual(skipped, [])
|
||||||
self.assertTrue(all(r['status'] == 'asserted' for r in index))
|
|
||||||
|
|
||||||
def test_proven_entries_become_real_theorems_with_their_proof(self):
|
def test_proven_entries_become_real_theorems_with_their_proof(self):
|
||||||
results = {"goodman-1-6-1": {"id": "goodman-1-6-1", "success": True,
|
results = {"goodman-1-6-1": {"id": "goodman-1-6-1", "success": True,
|
||||||
"attempts": [{"proof": "simp"}]}}
|
"attempts": [{"proof": "simp"}]}}
|
||||||
text, index, _emitted, _skipped = build(ENTRIES, results)
|
text, _emitted, _skipped = build(ENTRIES, results)
|
||||||
self.assertIn('theorem goodman_1_6_1 :', text)
|
self.assertIn('theorem goodman_1_6_1 :', text)
|
||||||
self.assertNotIn('axiom goodman_1_6_1 :', text)
|
self.assertNotIn('axiom goodman_1_6_1 :', text)
|
||||||
self.assertIn(' simp', text)
|
self.assertIn(' simp', text)
|
||||||
statuses = {r['id']: r['status'] for r in index}
|
self.assertIn('Proof verified by Lean.', text)
|
||||||
self.assertEqual(statuses['goodman-1-6-1'], 'proved')
|
|
||||||
self.assertEqual(statuses['goodman-1-6-24'], 'asserted')
|
def test_recorded_proof_is_only_emitted_once_checked(self):
|
||||||
|
pending = [dict(ENTRIES[0], proof='ring', proof_checked=False)]
|
||||||
|
text, _e, _s = build(pending)
|
||||||
|
self.assertIn('axiom goodman_1_6_1 :', text)
|
||||||
|
self.assertIn('not yet verified', text)
|
||||||
|
|
||||||
|
checked = [dict(ENTRIES[0], proof='ring', proof_checked=True)]
|
||||||
|
text, _e, _s = build(checked)
|
||||||
|
self.assertIn('theorem goodman_1_6_1 :', text)
|
||||||
|
self.assertIn(' ring', text)
|
||||||
|
|
||||||
def test_shared_def_in_prelude_is_emitted_once_not_per_entry(self):
|
def test_shared_def_in_prelude_is_emitted_once_not_per_entry(self):
|
||||||
# Two entries share the same `def`; declaring it twice in one Lean file is an
|
# Two entries share the same `def`; declaring it twice in one Lean file is an
|
||||||
# "already declared" error, so it must be hoisted and deduplicated.
|
# "already declared" error, so it must be hoisted and deduplicated.
|
||||||
text, _index, _emitted, _skipped = build(ENTRIES)
|
text, _emitted, _skipped = build(ENTRIES)
|
||||||
self.assertEqual(text.count('def IsGCDOfList'), 1)
|
self.assertEqual(text.count('def IsGCDOfList'), 1)
|
||||||
self.assertEqual(text.count('import Mathlib'), 1)
|
self.assertEqual(text.count('import Mathlib'), 1)
|
||||||
|
|
||||||
def test_require_checked_skips_unverified_entries(self):
|
def test_require_checked_skips_unverified_entries(self):
|
||||||
entries = [dict(ENTRIES[0]), dict(ENTRIES[1], elaborates=False)]
|
entries = [dict(ENTRIES[0]), dict(ENTRIES[1], elaborates=False)]
|
||||||
_text, index, emitted, skipped = build(entries, require_checked=True)
|
text, emitted, skipped = build(entries, require_checked=True)
|
||||||
self.assertEqual(emitted, ['goodman-1-6-1'])
|
self.assertEqual(emitted, ['goodman-1-6-1'])
|
||||||
self.assertEqual(skipped, ['goodman-1-6-24'])
|
self.assertEqual(skipped, ['goodman-1-6-24'])
|
||||||
self.assertEqual(len(index), 1)
|
self.assertNotIn('goodman_1_6_24', text)
|
||||||
|
|
||||||
def test_recorded_proof_counts_only_once_checked(self):
|
def test_docstring_records_provenance_and_axiom_warning(self):
|
||||||
# An unverified proof must NOT be emitted as a theorem -- the library may
|
text, _emitted, _skipped = build(ENTRIES)
|
||||||
# never claim more than Lean has confirmed.
|
self.assertIn('Proposition 1.6.1', text)
|
||||||
pending = [dict(ENTRIES[0], proof='ring', proof_checked=False)]
|
self.assertIn('ASSERTED from the textbook', text)
|
||||||
text, index, _e, _s = build(pending)
|
self.assertNotIn('fix note', text)
|
||||||
self.assertIn('axiom goodman_1_6_1 :', text)
|
|
||||||
self.assertNotIn('theorem goodman_1_6_1 :', text)
|
|
||||||
self.assertEqual(index[0]['status'], 'proof_pending')
|
|
||||||
self.assertEqual(index[0]['proof'], 'ring')
|
|
||||||
|
|
||||||
checked = [dict(ENTRIES[0], proof='ring', proof_checked=True)]
|
|
||||||
text, index, _e, _s = build(checked)
|
|
||||||
self.assertIn('theorem goodman_1_6_1 :', text)
|
|
||||||
self.assertIn(' ring', text)
|
|
||||||
self.assertEqual(index[0]['status'], 'proved')
|
|
||||||
|
|
||||||
def test_index_carries_identifiers_and_strips_fix_notes(self):
|
|
||||||
_text, index, _emitted, _skipped = build(ENTRIES)
|
|
||||||
first = index[0]
|
|
||||||
self.assertIn('Int.add_zero', first['requires'])
|
|
||||||
self.assertNotIn('fix note', first['natural_language'])
|
|
||||||
self.assertNotIn('let', statement_identifiers('let x := 1'))
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from textbook.search_database import format_for_prompt, search, tokenize
|
import json
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from textbook.search_database import format_for_prompt, load_database, search, tokenize
|
||||||
|
|
||||||
INDEX = [
|
INDEX = [
|
||||||
{"name": "goodman_1_6_20", "id": "goodman-1-6-20", "source_label": "Corollary 1.6.20",
|
{"name": "goodman_1_6_20", "id": "goodman-1-6-20", "source_label": "Corollary 1.6.20",
|
||||||
@@ -18,12 +22,63 @@ INDEX = [
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
SOURCE_ENTRIES = [
|
||||||
|
{"id": "goodman-1-6-20", "source_label": "Corollary 1.6.20", "kind": "corollary",
|
||||||
|
"statement": "\u2200 (p : \u2115), Nat.Prime p \u2192 (p:\u2124) \u2223 List.prod as",
|
||||||
|
"prelude": "import Mathlib", "natural_language": "x [fix note: dropped]",
|
||||||
|
"requires": ["Nat.Prime"]},
|
||||||
|
{"id": "goodman-1-6-24", "source_label": "Proposition 1.6.24", "kind": "proposition",
|
||||||
|
"statement": "\u2200 (as : List \u2124), \u2203 d, IsGCDOfList d as",
|
||||||
|
"prelude": "import Mathlib", "natural_language": "", "requires": [],
|
||||||
|
"proof": "exact foo", "proof_checked": True},
|
||||||
|
{"id": "goodman-1-6-1", "source_label": "Proposition 1.6.1", "kind": "proposition",
|
||||||
|
"statement": "\u2200 (a : \u2124), a + 0 = a", "prelude": "import Mathlib",
|
||||||
|
"natural_language": "", "requires": [], "proof": "ring", "proof_checked": False},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class LoadDatabaseTests(unittest.TestCase):
|
||||||
|
"""load_database derives name/status/identifiers from the source file, so there is
|
||||||
|
no separate index that could go stale against it."""
|
||||||
|
|
||||||
|
def _load(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
path = Path(directory) / 'db.jsonl'
|
||||||
|
path.write_text('\n'.join(json.dumps(e) for e in SOURCE_ENTRIES) + '\n',
|
||||||
|
encoding='utf-8')
|
||||||
|
return load_database(path)
|
||||||
|
|
||||||
|
def test_derives_lean_name_and_strips_fix_notes(self):
|
||||||
|
records = {r['id']: r for r in self._load()}
|
||||||
|
self.assertEqual(records['goodman-1-6-20']['name'], 'goodman_1_6_20')
|
||||||
|
self.assertNotIn('fix note', records['goodman-1-6-20']['natural_language'])
|
||||||
|
|
||||||
|
def test_derives_proof_status_from_proof_checked(self):
|
||||||
|
records = {r['id']: r for r in self._load()}
|
||||||
|
self.assertEqual(records['goodman-1-6-24']['status'], 'proved')
|
||||||
|
self.assertEqual(records['goodman-1-6-1']['status'], 'proof_pending')
|
||||||
|
self.assertEqual(records['goodman-1-6-20']['status'], 'asserted')
|
||||||
|
|
||||||
|
def test_search_works_directly_on_loaded_source(self):
|
||||||
|
hits = search(self._load(), '\u22a2 IsGCDOfList d as', limit=2)
|
||||||
|
self.assertEqual(hits[0]['name'], 'goodman_1_6_24')
|
||||||
|
|
||||||
|
|
||||||
class SearchDatabaseTests(unittest.TestCase):
|
class SearchDatabaseTests(unittest.TestCase):
|
||||||
def test_tokenize_keeps_qualified_names_and_drops_stopwords(self):
|
def test_tokenize_keeps_qualified_names_and_drops_stopwords(self):
|
||||||
tokens = tokenize('let x := Nat.Prime p')
|
tokens = tokenize('let x := Nat.Prime p')
|
||||||
self.assertIn('Nat.Prime', tokens)
|
self.assertIn('Nat.Prime', tokens)
|
||||||
self.assertNotIn('let', tokens)
|
self.assertNotIn('let', tokens)
|
||||||
|
|
||||||
|
def test_tokenize_keeps_symbols_but_drops_single_letter_variables(self):
|
||||||
|
# A pure-algebra goal has no identifiers at all, so the symbols have to
|
||||||
|
# survive or it retrieves nothing; matching 'a' against 'a' says nothing.
|
||||||
|
tokens = tokenize('\u2200 (a b : \u2124), a * (b + c) = 0')
|
||||||
|
self.assertIn('\u2124', tokens)
|
||||||
|
self.assertIn('*', tokens)
|
||||||
|
self.assertIn('+', tokens)
|
||||||
|
self.assertNotIn('a', tokens)
|
||||||
|
|
||||||
def test_exact_premise_ranks_first(self):
|
def test_exact_premise_ranks_first(self):
|
||||||
hits = search(INDEX, '\u22a2 IsGCDOfList d as', limit=3)
|
hits = search(INDEX, '\u22a2 IsGCDOfList d as', limit=3)
|
||||||
self.assertEqual(hits[0]['name'], 'goodman_1_6_24')
|
self.assertEqual(hits[0]['name'], 'goodman_1_6_24')
|
||||||
|
|||||||
+12
-7
@@ -6,8 +6,8 @@ distinct things `lean-bench`/`lean-agent` can use, not one:
|
|||||||
- **The bench** (`datasets/goodman-1.6-exercises.jsonl`): real problems the reader
|
- **The bench** (`datasets/goodman-1.6-exercises.jsonl`): real problems the reader
|
||||||
is meant to solve, sourced from the book's own **Exercises** at the end of each
|
is meant to solve, sourced from the book's own **Exercises** at the end of each
|
||||||
section. These are the actual targets `lean-bench` scores an agent against.
|
section. These are the actual targets `lean-bench` scores an agent against.
|
||||||
- **The database** (`database/goodman-1.6.jsonl` -> `lib/Goodman/Chapter01Section06.lean`
|
- **The database** (`database/goodman-1.6.jsonl` -> `lib/Goodman/Chapter01Section06.lean`):
|
||||||
+ `database/goodman-1.6.index.jsonl`): the book's stated Theorem/Proposition/
|
the book's stated Theorem/Proposition/
|
||||||
Lemma/Corollary claims from the main text, turned by `build_database.py` into real
|
Lemma/Corollary claims from the main text, turned by `build_database.py` into real
|
||||||
Lean declarations an agent can **cite by name** while proving a bench target --
|
Lean declarations an agent can **cite by name** while proving a bench target --
|
||||||
the "third party math library" role Mathlib plays, except scoped to this book.
|
the "third party math library" role Mathlib plays, except scoped to this book.
|
||||||
@@ -25,7 +25,13 @@ distinct things `lean-bench`/`lean-agent` can use, not one:
|
|||||||
axiom here still needs a human to have read it against the book.
|
axiom here still needs a human to have read it against the book.
|
||||||
- **Retrieval** (`search_database.py`): given the current goal, ranks database
|
- **Retrieval** (`search_database.py`): given the current goal, ranks database
|
||||||
premises by identifier/symbol overlap with rare identifiers weighted higher, and
|
premises by identifier/symbol overlap with rare identifiers weighted higher, and
|
||||||
formats the top few for the LLM prompt. This is the step the proof loop calls at
|
formats the top few for the LLM prompt. It reads the database JSONL directly and
|
||||||
|
derives each premise's declaration name, identifiers and proof status itself
|
||||||
|
(sharing those functions with `build_database.py`), so the data exists in exactly
|
||||||
|
one place. There is deliberately no generated index: at this scale deriving them
|
||||||
|
costs microseconds, and a second copy that nothing checks for freshness could
|
||||||
|
silently serve statements that no longer match the `.lean` file. An index earns
|
||||||
|
its place once it holds something actually expensive, i.e. embeddings. This is the step the proof loop calls at
|
||||||
each line -- it narrows ~20 premises to the few worth showing, and cannot tell you
|
each line -- it narrows ~20 premises to the few worth showing, and cannot tell you
|
||||||
whether one applies; only Lean can. Swapping in embeddings later means replacing
|
whether one applies; only Lean can. Swapping in embeddings later means replacing
|
||||||
`score_premise()` and nothing else.
|
`score_premise()` and nothing else.
|
||||||
@@ -195,17 +201,16 @@ the next build.
|
|||||||
```sh
|
```sh
|
||||||
python3 textbook/build_database.py database/goodman-1.6.jsonl \
|
python3 textbook/build_database.py database/goodman-1.6.jsonl \
|
||||||
--out-lean lib/Goodman/Chapter01Section06.lean \
|
--out-lean lib/Goodman/Chapter01Section06.lean \
|
||||||
--out-index database/goodman-1.6.index.jsonl \
|
|
||||||
--require-checked
|
--require-checked
|
||||||
```
|
```
|
||||||
|
|
||||||
Add `--results results/goodman-1.6-library-001/results.jsonl` once a `lean-bench`
|
Add `--results results/goodman-1.6-library-001/results.jsonl` once a `lean-bench`
|
||||||
run over `database/goodman-1.6.jsonl` has proven some of them, to upgrade those
|
run over `database/goodman-1.6.jsonl` has proven some of them, to upgrade those
|
||||||
entries from `axiom` to real theorems. Then query it with the current goal:
|
entries from `axiom` to real theorems. Query it with the current goal:
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
python3 textbook/search_database.py database/goodman-1.6.index.jsonl \
|
python3 textbook/search_database.py database/goodman-1.6.jsonl \
|
||||||
"\u22a2 Nat.Prime p \u2192 (p:\u2124) \u2223 List.prod as \u2192 \u2203 a \u2208 as, (p:\u2124) \u2223 a"
|
"⊢ Nat.Prime p → (p:ℤ) ∣ List.prod as → ∃ a ∈ as, (p:ℤ) ∣ a"
|
||||||
```
|
```
|
||||||
|
|
||||||
(`build_library.py` predates this and remains for the narrower job of assembling
|
(`build_library.py` predates this and remains for the narrower job of assembling
|
||||||
|
|||||||
+43
-51
@@ -5,23 +5,21 @@ Theorem/Proposition/Lemma/Corollary claims, turned into real Lean declarations t
|
|||||||
lean-agent can cite by name while proving something else (the bench targets, which
|
lean-agent can cite by name while proving something else (the bench targets, which
|
||||||
come from the exercises instead -- see formalize_exercises.py).
|
come from the exercises instead -- see formalize_exercises.py).
|
||||||
|
|
||||||
Two outputs:
|
Output: an importable Lean file where every database entry is a named declaration,
|
||||||
|
so a proof can cite goodman_1_6_20 by name. An entry is emitted as a real
|
||||||
|
`theorem ... := by <proof>` once its proof has been verified, and otherwise as an
|
||||||
|
`axiom` -- asserted from the textbook, which is what the book itself does for its
|
||||||
|
own body results before an exercise asks you to use them. This is deliberate and
|
||||||
|
has a real cost: anything proven *through* an axiom inherits it, which is exactly
|
||||||
|
why the verifier's `#print axioms` check matters. A statement that only elaborates
|
||||||
|
is not the same as a statement that is true, and a garbled extraction asserted as
|
||||||
|
an axiom can make downstream "proofs" worthless -- so every axiom here needs to have
|
||||||
|
been read by a human against the book, and `--require-checked` refuses to emit any
|
||||||
|
entry not marked as having passed check_statements.py.
|
||||||
|
|
||||||
1. An importable Lean file where every database entry is a named declaration. An
|
There is deliberately no generated search index: search_database.py reads this same
|
||||||
entry is emitted as a real `theorem ... := by <proof>` when a lean-bench run has
|
source file and derives the declaration name, identifiers and status itself, so
|
||||||
actually proven it (status "proved"), and otherwise as an `axiom` -- asserted from
|
there is only ever one copy of the data and nothing to keep in sync.
|
||||||
the textbook, which is what the book itself does for its own body results before
|
|
||||||
an exercise asks you to use them. This is deliberate and has a real cost: anything
|
|
||||||
proven *through* an axiom inherits it, which is exactly why the verifier's
|
|
||||||
`#print axioms` check matters. A statement that only elaborates is not the same as
|
|
||||||
a statement that is true, and a garbled extraction asserted as an axiom can make
|
|
||||||
downstream "proofs" worthless -- so every axiom here needs to have been read by a
|
|
||||||
human against the book, and `--require-checked` refuses to emit any entry not
|
|
||||||
marked as having passed check_statements.py.
|
|
||||||
|
|
||||||
2. A search index (JSONL) carrying each entry's declaration name, statement, the
|
|
||||||
Mathlib identifiers it mentions, and its status -- what search_database.py ranks
|
|
||||||
against the current goals at each step of a proof.
|
|
||||||
|
|
||||||
Pass a lean-bench run's results.jsonl with --results to upgrade the entries it
|
Pass a lean-bench run's results.jsonl with --results to upgrade the entries it
|
||||||
proved from axiom to a real theorem carrying the exact proof that run verified.
|
proved from axiom to a real theorem carrying the exact proof that run verified.
|
||||||
@@ -53,8 +51,26 @@ def split_prelude(prelude):
|
|||||||
return imports, '\n'.join(other).strip()
|
return imports, '\n'.join(other).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_proof(entry, result=None):
|
||||||
|
"""(proof, status) for one entry.
|
||||||
|
|
||||||
|
A proof only counts once Lean has verified it: check_proofs.py sets
|
||||||
|
proof_checked on the entry, or a lean-bench run reports success. An unverified
|
||||||
|
proof leaves the entry asserted, reported as proof_pending so it is visible
|
||||||
|
without being trusted."""
|
||||||
|
run_proof = result['attempts'][-1]['proof'] if (
|
||||||
|
result and result.get('success') and result.get('attempts')) else None
|
||||||
|
own_proof = entry.get('proof') if entry.get('proof_checked') else None
|
||||||
|
proof = run_proof or own_proof
|
||||||
|
if proof:
|
||||||
|
return proof, 'proved'
|
||||||
|
if entry.get('proof'):
|
||||||
|
return None, 'proof_pending'
|
||||||
|
return None, 'asserted'
|
||||||
|
|
||||||
|
|
||||||
def statement_identifiers(statement):
|
def statement_identifiers(statement):
|
||||||
"""Identifiers mentioned in a statement, for the search index."""
|
"""Identifiers mentioned in a statement, for premise retrieval."""
|
||||||
found = []
|
found = []
|
||||||
for match in IDENT_RE.findall(statement or ''):
|
for match in IDENT_RE.findall(statement or ''):
|
||||||
if match in STOPWORDS or match in found or len(match) < 2:
|
if match in STOPWORDS or match in found or len(match) < 2:
|
||||||
@@ -89,7 +105,7 @@ def build(entries, results=None, require_checked=False):
|
|||||||
lines.append(block)
|
lines.append(block)
|
||||||
lines.append('')
|
lines.append('')
|
||||||
|
|
||||||
index, emitted, skipped = [], [], []
|
emitted, skipped = [], []
|
||||||
for entry in entries:
|
for entry in entries:
|
||||||
entry_id = entry['id']
|
entry_id = entry['id']
|
||||||
if require_checked and not entry.get('elaborates'):
|
if require_checked and not entry.get('elaborates'):
|
||||||
@@ -97,17 +113,9 @@ def build(entries, results=None, require_checked=False):
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
name = decl_name(entry_id)
|
name = decl_name(entry_id)
|
||||||
result = results.get(entry_id)
|
proof, status = resolve_proof(entry, results.get(entry_id))
|
||||||
# A proof can come from a lean-bench run, or be recorded on the entry itself
|
proved = status == 'proved'
|
||||||
# (hand-written, or authored by an agent outside a bench run). Either way it
|
proof_pending = status == 'proof_pending'
|
||||||
# only counts once check_proofs.py has actually compiled it: an unverified
|
|
||||||
# proof stays an axiom, so the library never claims more than Lean confirmed.
|
|
||||||
run_proof = result['attempts'][-1]['proof'] if (
|
|
||||||
result and result.get('success') and result.get('attempts')) else None
|
|
||||||
own_proof = entry.get('proof') if entry.get('proof_checked') else None
|
|
||||||
proof = run_proof or own_proof
|
|
||||||
proved = bool(proof)
|
|
||||||
proof_pending = bool(not proved and entry.get('proof'))
|
|
||||||
provenance = entry.get('source_label', entry_id)
|
provenance = entry.get('source_label', entry_id)
|
||||||
|
|
||||||
doc = f'/-- {provenance} (Goodman, Algebra: Abstract and Concrete).'
|
doc = f'/-- {provenance} (Goodman, Algebra: Abstract and Concrete).'
|
||||||
@@ -129,21 +137,9 @@ def build(entries, results=None, require_checked=False):
|
|||||||
lines.append(f'axiom {name} : {entry["statement"]}')
|
lines.append(f'axiom {name} : {entry["statement"]}')
|
||||||
lines.append('')
|
lines.append('')
|
||||||
|
|
||||||
index.append({
|
|
||||||
"name": name,
|
|
||||||
"id": entry_id,
|
|
||||||
"source_label": entry.get('source_label'),
|
|
||||||
"kind": entry.get('kind'),
|
|
||||||
"status": "proved" if proved else ("proof_pending" if proof_pending else "asserted"),
|
|
||||||
"proof": proof if proved else entry.get('proof'),
|
|
||||||
"statement": entry['statement'],
|
|
||||||
"natural_language": entry.get('natural_language', '').split('[fix note:')[0].strip(),
|
|
||||||
"identifiers": statement_identifiers(entry['statement']),
|
|
||||||
"requires": entry.get('requires', []),
|
|
||||||
})
|
|
||||||
emitted.append(entry_id)
|
emitted.append(entry_id)
|
||||||
|
|
||||||
return '\n'.join(lines), index, emitted, skipped
|
return '\n'.join(lines), emitted, skipped
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
@@ -151,27 +147,23 @@ def main():
|
|||||||
parser.add_argument('database', type=Path, help='JSONL of main-content statements, e.g. database/goodman-1.6.jsonl')
|
parser.add_argument('database', type=Path, help='JSONL of main-content statements, e.g. database/goodman-1.6.jsonl')
|
||||||
parser.add_argument('--results', type=Path, help="a lean-bench run's results.jsonl; proven entries become real theorems")
|
parser.add_argument('--results', type=Path, help="a lean-bench run's results.jsonl; proven entries become real theorems")
|
||||||
parser.add_argument('--out-lean', type=Path, required=True, help='importable Lean file of citable premises')
|
parser.add_argument('--out-lean', type=Path, required=True, help='importable Lean file of citable premises')
|
||||||
parser.add_argument('--out-index', type=Path, required=True, help='JSONL search index for search_database.py')
|
|
||||||
parser.add_argument('--require-checked', action='store_true',
|
parser.add_argument('--require-checked', action='store_true',
|
||||||
help='refuse entries not marked "elaborates": true (i.e. not yet verified by check_statements.py)')
|
help='refuse entries not marked "elaborates": true (i.e. not yet verified by check_statements.py)')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
entries = load_jsonl(args.database)
|
entries = load_jsonl(args.database)
|
||||||
results = {r['id']: r for r in load_jsonl(args.results)} if args.results else None
|
results = {r['id']: r for r in load_jsonl(args.results)} if args.results else None
|
||||||
text, index, emitted, skipped = build(entries, results, args.require_checked)
|
text, emitted, skipped = build(entries, results, args.require_checked)
|
||||||
|
|
||||||
args.out_lean.parent.mkdir(parents=True, exist_ok=True)
|
args.out_lean.parent.mkdir(parents=True, exist_ok=True)
|
||||||
args.out_lean.write_text(text + '\n', encoding='utf-8')
|
args.out_lean.write_text(text + '\n', encoding='utf-8')
|
||||||
args.out_index.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
with args.out_index.open('w', encoding='utf-8') as stream:
|
|
||||||
for record in index:
|
|
||||||
stream.write(json.dumps(record, ensure_ascii=False) + '\n')
|
|
||||||
|
|
||||||
proved = sum(1 for r in index if r['status'] == 'proved')
|
by_id = {e['id']: e for e in entries}
|
||||||
pending = sum(1 for r in index if r['status'] == 'proof_pending')
|
statuses = [resolve_proof(by_id[i], (results or {}).get(i))[1] for i in emitted]
|
||||||
|
proved = statuses.count('proved')
|
||||||
|
pending = statuses.count('proof_pending')
|
||||||
print(f'{len(emitted)} premises -> {args.out_lean} ({proved} proven, '
|
print(f'{len(emitted)} premises -> {args.out_lean} ({proved} proven, '
|
||||||
f'{len(emitted) - proved} asserted as axioms, of which {pending} have an unverified proof)')
|
f'{len(emitted) - proved} asserted as axioms, of which {pending} have an unverified proof)')
|
||||||
print(f'search index -> {args.out_index}')
|
|
||||||
if skipped:
|
if skipped:
|
||||||
print(f'{len(skipped)} skipped for not being elaboration-checked: {", ".join(skipped)}')
|
print(f'{len(skipped)} skipped for not being elaboration-checked: {", ".join(skipped)}')
|
||||||
|
|
||||||
|
|||||||
+58
-12
@@ -7,6 +7,13 @@ for entries that might apply to the goal in front of it. The ranked results are
|
|||||||
gets shown to the LLM alongside the goal, so the next line of Lean it writes can
|
gets shown to the LLM alongside the goal, so the next line of Lean it writes can
|
||||||
cite a real premise by name instead of inventing a lemma.
|
cite a real premise by name instead of inventing a lemma.
|
||||||
|
|
||||||
|
It reads the database JSONL directly and derives each premise's declaration name,
|
||||||
|
identifiers and proof status on the fly, sharing those functions with
|
||||||
|
build_database.py. There is no precomputed index: at this scale deriving them costs
|
||||||
|
microseconds, and a second generated copy of the data would only create something
|
||||||
|
that can silently go stale against the source. A precomputed index earns its place
|
||||||
|
when it holds something genuinely expensive, i.e. embeddings.
|
||||||
|
|
||||||
Scoring here is deliberately simple and dependency-free: overlap of identifiers and
|
Scoring here is deliberately simple and dependency-free: overlap of identifiers and
|
||||||
symbols between the goal and each premise, weighted so that a rare identifier
|
symbols between the goal and each premise, weighted so that a rare identifier
|
||||||
matters more than a common one (`IsGCDOfList` appearing in both is a much stronger
|
matters more than a common one (`IsGCDOfList` appearing in both is a much stronger
|
||||||
@@ -23,22 +30,61 @@ import re
|
|||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
TOKEN_RE = re.compile(r'[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+|[A-Za-z_][A-Za-z0-9_]*|[∣∀∃↔→∧∨¬≤≥≠∈]')
|
# Qualified names (Nat.Prime) first so they aren't split, then bare identifiers, then
|
||||||
|
# the type and operator symbols that carry signal. Arithmetic operators are included
|
||||||
|
# deliberately: a goal that is pure algebra ("a * (b + c) = a * b + a * c") has no
|
||||||
|
# identifiers at all, and without these it would tokenize to nothing and retrieve
|
||||||
|
# nothing. They appear in nearly every premise, so inverse-document-frequency
|
||||||
|
# weighting already discounts them to almost zero -- they only decide ties.
|
||||||
|
TOKEN_RE = re.compile(
|
||||||
|
r'[A-Za-z_][A-Za-z0-9_]*(?:\.[A-Za-z_][A-Za-z0-9_]*)+'
|
||||||
|
r'|[A-Za-z_][A-Za-z0-9_]*'
|
||||||
|
r'|[∣∀∃↔→∧∨¬≤≥≠∈ℤℕℚℝ×∑∏]'
|
||||||
|
r'|[+*=/-]')
|
||||||
STOPWORDS = {'let', 'in', 'fun', 'if', 'then', 'else', 'true', 'false', 'Prop', 'Type', 'this'}
|
STOPWORDS = {'let', 'in', 'fun', 'if', 'then', 'else', 'true', 'false', 'Prop', 'Type', 'this'}
|
||||||
|
|
||||||
|
|
||||||
def load_index(path):
|
try:
|
||||||
return [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()]
|
from .build_database import decl_name, resolve_proof, statement_identifiers
|
||||||
|
except ImportError:
|
||||||
|
from build_database import decl_name, resolve_proof, statement_identifiers
|
||||||
|
|
||||||
|
|
||||||
|
def load_database(path, results=None):
|
||||||
|
"""Database entries in the shape search() ranks, derived from the source file."""
|
||||||
|
entries = [json.loads(line) for line in path.read_text(encoding='utf-8').splitlines() if line.strip()]
|
||||||
|
records = []
|
||||||
|
for entry in entries:
|
||||||
|
proof, status = resolve_proof(entry, (results or {}).get(entry['id']))
|
||||||
|
records.append({
|
||||||
|
"name": decl_name(entry['id']),
|
||||||
|
"id": entry['id'],
|
||||||
|
"source_label": entry.get('source_label'),
|
||||||
|
"kind": entry.get('kind'),
|
||||||
|
"status": status,
|
||||||
|
"proof": proof or entry.get('proof'),
|
||||||
|
"statement": entry['statement'],
|
||||||
|
"natural_language": entry.get('natural_language', '').split('[fix note:')[0].strip(),
|
||||||
|
"identifiers": statement_identifiers(entry['statement']),
|
||||||
|
"requires": entry.get('requires', []),
|
||||||
|
})
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
SYMBOLS = set('∣∀∃↔→∧∨¬≤≥≠∈ℤℕℚℝ×∑∏+*=/-')
|
||||||
|
|
||||||
|
|
||||||
def tokenize(text):
|
def tokenize(text):
|
||||||
return [t for t in TOKEN_RE.findall(text or '') if t not in STOPWORDS and len(t) > 1]
|
"""Identifiers and meaningful symbols. Single-letter variable names are dropped
|
||||||
|
(matching `a` against `a` says nothing), but single-character symbols are kept."""
|
||||||
|
return [t for t in TOKEN_RE.findall(text or '')
|
||||||
|
if t not in STOPWORDS and (len(t) > 1 or t in SYMBOLS)]
|
||||||
|
|
||||||
|
|
||||||
def document_frequencies(index):
|
def document_frequencies(records):
|
||||||
"""How many premises each token appears in, for inverse-document-frequency weighting."""
|
"""How many premises each token appears in, for inverse-document-frequency weighting."""
|
||||||
frequencies = Counter()
|
frequencies = Counter()
|
||||||
for record in index:
|
for record in records:
|
||||||
for token in set(tokenize(record['statement']) + record.get('identifiers', [])
|
for token in set(tokenize(record['statement']) + record.get('identifiers', [])
|
||||||
+ record.get('requires', [])):
|
+ record.get('requires', [])):
|
||||||
frequencies[token] += 1
|
frequencies[token] += 1
|
||||||
@@ -57,13 +103,13 @@ def score_premise(goal_tokens, record, frequencies, total):
|
|||||||
return score
|
return score
|
||||||
|
|
||||||
|
|
||||||
def search(index, goal, limit=5):
|
def search(records, goal, limit=5):
|
||||||
"""Premises ranked against one goal, best first. Zero-score entries are dropped."""
|
"""Premises ranked against one goal, best first. Zero-score entries are dropped."""
|
||||||
frequencies = document_frequencies(index)
|
frequencies = document_frequencies(records)
|
||||||
total = max(len(index), 1)
|
total = max(len(records), 1)
|
||||||
goal_tokens = tokenize(goal)
|
goal_tokens = tokenize(goal)
|
||||||
scored = []
|
scored = []
|
||||||
for record in index:
|
for record in records:
|
||||||
score = score_premise(goal_tokens, record, frequencies, total)
|
score = score_premise(goal_tokens, record, frequencies, total)
|
||||||
if score > 0:
|
if score > 0:
|
||||||
scored.append((score, record))
|
scored.append((score, record))
|
||||||
@@ -85,13 +131,13 @@ def format_for_prompt(hits):
|
|||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description='Rank database premises against a proof goal')
|
parser = argparse.ArgumentParser(description='Rank database premises against a proof goal')
|
||||||
parser.add_argument('index', type=Path, help='search index from build_database.py')
|
parser.add_argument('database', type=Path, help='database JSONL, e.g. database/goodman-1.6.jsonl')
|
||||||
parser.add_argument('goal', help='the current goal, as Lean prints it')
|
parser.add_argument('goal', help='the current goal, as Lean prints it')
|
||||||
parser.add_argument('--limit', type=int, default=5)
|
parser.add_argument('--limit', type=int, default=5)
|
||||||
parser.add_argument('--json', action='store_true', help='emit raw JSON instead of prompt text')
|
parser.add_argument('--json', action='store_true', help='emit raw JSON instead of prompt text')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
hits = search(load_index(args.index), args.goal, args.limit)
|
hits = search(load_database(args.database), args.goal, args.limit)
|
||||||
print(json.dumps(hits, ensure_ascii=False, indent=2) if args.json else format_for_prompt(hits))
|
print(json.dumps(hits, ensure_ascii=False, indent=2) if args.json else format_for_prompt(hits))
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user