diff --git a/docs/operations.md b/docs/operations.md index f5bd599..88aa83a 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -30,6 +30,8 @@ paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \ 审计持有共享维护锁;修复或其他协调写入尚未结束时会拒绝运行。提供修复报告时,还会核对其已完成状态、数据库路径和固定来源范围。审计以一致的只读快照分批读取,记录完整扫描数量、缺失/多余证据、错误论文归属、残留旧键及少量异常样例,不修改主库。 +来源记录按整数主键区间读取,避免 SQLite 误选按 external_id 排列的来源索引、每批重新排序整个来源;其他来源交错在编号区间内时仍按 source_id 过滤。类别证据继续使用 source_record_id 索引批量读取。完整核验范围不因查询优化缩小。 + 只有全量记录核对完毕且无差异时才标记 `passed`;存在差异为 `failed` 并返回非零退出码,解析或执行中断为 `error`。运行中的报告不能作为完成证明。重新核验须使用新输出路径,不能覆盖已有报告;该检查不替代最后的实体去重、外键、SQLite 与全文索引核验。 ## 长期维护与更新 diff --git a/paper_atlas/audits/arxiv_tags.py b/paper_atlas/audits/arxiv_tags.py index c663bd5..688c651 100644 --- a/paper_atlas/audits/arxiv_tags.py +++ b/paper_atlas/audits/arxiv_tags.py @@ -56,7 +56,10 @@ def audit(db, out, repair_report=None, batch_size=500): save(out, state) try: while True: - records = c.execute('''SELECT id,external_id,paper_id,metadata_json FROM source_records + # The (source_id,external_id) index orders by external ID, + # so choosing it sorts the entire source again per batch. + # This fixed ID interval must use the integer primary key. + records = c.execute('''SELECT id,external_id,paper_id,metadata_json FROM source_records NOT INDEXED WHERE source_id=? AND id>? AND id<=? ORDER BY id LIMIT ?''', (source[0], state['last_record_id'], maximum, batch_size)).fetchall() if not records: diff --git a/tests/test_arxiv_tag_audit.py b/tests/test_arxiv_tag_audit.py index 0fdd779..adb0127 100644 --- a/tests/test_arxiv_tag_audit.py +++ b/tests/test_arxiv_tag_audit.py @@ -3,6 +3,8 @@ import json from pathlib import Path import tempfile import unittest +import sqlite3 +from unittest.mock import patch from paper_atlas.audits.arxiv_tags import audit, main from paper_atlas.db.store import Store @@ -107,6 +109,30 @@ class ArxivTagAuditTests(unittest.TestCase): audit(self.db, self.out) self.assertEqual(json.loads(self.out.read_text())['status'], 'error') + def test_interleaved_sources_use_bounded_primary_key_batches_without_sorting(self): + self.store.record('example', 'interleaved', 'https://example.org', 'Other source', 'AB', []) + paper, record = self.store.record('arxiv', 'three', 'https://arxiv.org/abs/three', + 'A third record', 'AB', [], metadata={'categories': 'cs.AI'}) + refresh_categories(self.store, paper, record, 'cs.AI') + self.store.conn.commit() + queries = [] + connect = sqlite3.connect + def traced(*args, **kwargs): + connection = connect(*args, **kwargs) + connection.set_trace_callback(queries.append) + return connection + with patch('paper_atlas.audits.arxiv_tags.sqlite3.connect', side_effect=traced): + result = audit(self.db, self.out, batch_size=1) + self.assertEqual(result['status'], 'passed') + self.assertEqual(result['records_checked'], 3) + self.assertEqual(result['last_record_id'], record) + batches = [sql for sql in queries if sql.startswith('SELECT id,external_id,paper_id,metadata_json')] + self.assertEqual(len(batches), 4) # Includes the bounded final empty read. + for sql in batches: + plan = ' '.join(row[3] for row in self.store.conn.execute('EXPLAIN QUERY PLAN ' + sql)) + self.assertIn('INTEGER PRIMARY KEY', plan) + self.assertNotIn('TEMP B-TREE', plan) + if __name__ == '__main__': unittest.main()