Bound arXiv tag audit batches by primary key instead of sorting the source
Metadata integrity checks / metadata (push) Successful in 43s

This commit is contained in:
2026-11-06 22:41:43 +08:00
parent bfa7fe2e94
commit c9b0cfa857
3 changed files with 32 additions and 1 deletions
+2
View File
@@ -30,6 +30,8 @@ paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \
审计持有共享维护锁;修复或其他协调写入尚未结束时会拒绝运行。提供修复报告时,还会核对其已完成状态、数据库路径和固定来源范围。审计以一致的只读快照分批读取,记录完整扫描数量、缺失/多余证据、错误论文归属、残留旧键及少量异常样例,不修改主库。
来源记录按整数主键区间读取,避免 SQLite 误选按 external_id 排列的来源索引、每批重新排序整个来源;其他来源交错在编号区间内时仍按 source_id 过滤。类别证据继续使用 source_record_id 索引批量读取。完整核验范围不因查询优化缩小。
只有全量记录核对完毕且无差异时才标记 `passed`;存在差异为 `failed` 并返回非零退出码,解析或执行中断为 `error`。运行中的报告不能作为完成证明。重新核验须使用新输出路径,不能覆盖已有报告;该检查不替代最后的实体去重、外键、SQLite 与全文索引核验。
## 长期维护与更新
+4 -1
View File
@@ -56,7 +56,10 @@ def audit(db, out, repair_report=None, batch_size=500):
save(out, state)
try:
while True:
records = c.execute('''SELECT id,external_id,paper_id,metadata_json FROM source_records
# The (source_id,external_id) index orders by external ID,
# so choosing it sorts the entire source again per batch.
# This fixed ID interval must use the integer primary key.
records = c.execute('''SELECT id,external_id,paper_id,metadata_json FROM source_records NOT INDEXED
WHERE source_id=? AND id>? AND id<=? ORDER BY id LIMIT ?''',
(source[0], state['last_record_id'], maximum, batch_size)).fetchall()
if not records:
+26
View File
@@ -3,6 +3,8 @@ import json
from pathlib import Path
import tempfile
import unittest
import sqlite3
from unittest.mock import patch
from paper_atlas.audits.arxiv_tags import audit, main
from paper_atlas.db.store import Store
@@ -107,6 +109,30 @@ class ArxivTagAuditTests(unittest.TestCase):
audit(self.db, self.out)
self.assertEqual(json.loads(self.out.read_text())['status'], 'error')
def test_interleaved_sources_use_bounded_primary_key_batches_without_sorting(self):
self.store.record('example', 'interleaved', 'https://example.org', 'Other source', 'AB', [])
paper, record = self.store.record('arxiv', 'three', 'https://arxiv.org/abs/three',
'A third record', 'AB', [], metadata={'categories': 'cs.AI'})
refresh_categories(self.store, paper, record, 'cs.AI')
self.store.conn.commit()
queries = []
connect = sqlite3.connect
def traced(*args, **kwargs):
connection = connect(*args, **kwargs)
connection.set_trace_callback(queries.append)
return connection
with patch('paper_atlas.audits.arxiv_tags.sqlite3.connect', side_effect=traced):
result = audit(self.db, self.out, batch_size=1)
self.assertEqual(result['status'], 'passed')
self.assertEqual(result['records_checked'], 3)
self.assertEqual(result['last_record_id'], record)
batches = [sql for sql in queries if sql.startswith('SELECT id,external_id,paper_id,metadata_json')]
self.assertEqual(len(batches), 4) # Includes the bounded final empty read.
for sql in batches:
plan = ' '.join(row[3] for row in self.store.conn.execute('EXPLAIN QUERY PLAN ' + sql))
self.assertIn('INTEGER PRIMARY KEY', plan)
self.assertNotIn('TEMP B-TREE', plan)
if __name__ == '__main__':
unittest.main()