Add a reproducible PostgreSQL global identity audit
Metadata integrity checks / metadata (push) Successful in 8m22s
Metadata integrity checks / metadata (push) Successful in 8m22s
This commit is contained in:
@@ -6,7 +6,7 @@
|
||||
|
||||
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
|
||||
|
||||
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
||||
历史目录包含 43 个会议。主动维护以核心 AI 会议及其已公开的决定为主,继续跟踪 ACM MM 和 MLSys。此前入库的 CIKM 数据与证据保留,后续主动补漏范围排除 CIKM。实际覆盖以数据库和来源清单为准。
|
||||
|
||||
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
|
||||
|
||||
@@ -108,6 +108,8 @@ AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的
|
||||
|
||||
CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤,不因代码测试通过而自动完成。
|
||||
|
||||
一批数据全部验收后,运行 [`audit global-identity`](docs/operations.md#导入后的全库身份核验) 核对 PostgreSQL 的稳定标识、同题与近似标题候选、摘要完整性和全文检索。审计报告和候选文件对应一次快照,后续导入完成后重新生成。使用 `audit histories` 检查跨会议投稿决定及其来源关系。
|
||||
|
||||
开发通过工作分支和 PR 交付,由用户手动合并。
|
||||
|
||||
早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。
|
||||
|
||||
@@ -1,5 +1,21 @@
|
||||
# 运行维护与数据核验
|
||||
|
||||
## 导入后的全库身份核验
|
||||
|
||||
在服务器准备一个至少有 2 GiB 可用空间的审计目录。使用生产读取角色,在计划中的导入全部验收后运行以下命令。命令在 PostgreSQL 的同一个可重复读快照内检查库内记录。SQLite 临时文件包含用来比较的标题键与作者指纹,扫描结束后清理。
|
||||
|
||||
```bash
|
||||
paper-atlas audit global-identity \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--scratch-root /data/paper-index/validation \
|
||||
--candidate-output /data/paper-index/validation/identity-candidates-2026-11-22.jsonl \
|
||||
> /data/paper-index/validation/identity-report-2026-11-22.json
|
||||
```
|
||||
|
||||
JSON 报告记录快照时间、论文与缺摘要计数、稳定标识冲突、约束状态、会议年度与轨道的来源覆盖,以及一次实际的全文搜索和执行计划。JSONL 保存同题且规范化作者集合相同的论文组,以及相同作者集合中标题相似度至少为 0.90 的论文对。超过 100 条记录的作者组会列入报告,留作后续逐组核对。缺作者、作者增减或换名的版本需要结合 DOI、论坛或作者说明,使用 `audit source-identities` 和原始来源复核。
|
||||
|
||||
将报告、候选文件、运行版本与 SHA-256 一起保留。使用 `audit histories` 核对拒稿、撤稿、后续录用和来源链接。需要合并候选时先确认论文版本关系并保存投稿、决定、标签及来源证据,再按已授权的流程操作。新一批导入完成后更新快照和审计报告。
|
||||
|
||||
## PostgreSQL 迁移
|
||||
|
||||
共享实例的容器名为 `foundry-postgres`,底层镜像为 `postgres:17-alpine`;Compose 服务名仍为 `postgres`。数据库和 schema 是可配置的逻辑边界,表按研究数据域组织,不按调用项目复制。其他项目可获授 `research` 的 USAGE 及必要表权限,复用相同论文 ID 和表。未来换库只更新 libpq service;换 schema 可更新角色 search_path 或查询的 `--pg-schema`。
|
||||
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Read-only PostgreSQL identity and integrity audit for an existing snapshot."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from contextlib import closing
|
||||
from difflib import SequenceMatcher
|
||||
from hashlib import sha256
|
||||
from itertools import combinations, groupby
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sqlite3
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from paper_atlas.db.store import author_keys
|
||||
from paper_atlas.postgres.client import ReadConnection, SEARCH_VECTOR
|
||||
|
||||
|
||||
ROSTER_GROUP_LIMIT = 100
|
||||
TITLE_SCORE = .90
|
||||
|
||||
|
||||
def _scalar(connection, query, params=()):
|
||||
return connection.execute(query, params).fetchone()[0]
|
||||
|
||||
|
||||
def _checks(connection):
|
||||
counts = {}
|
||||
for table, columns in (
|
||||
('identifiers', 'scheme,value'),
|
||||
('source_records', 'source_id,external_id'),
|
||||
('submission_identifiers', 'edition_id,scheme,value'),
|
||||
):
|
||||
counts[f'duplicate_{table}_keys'] = _scalar(connection, f'''
|
||||
SELECT count(*) FROM (
|
||||
SELECT {columns} FROM {table} GROUP BY {columns} HAVING count(*)>1
|
||||
) duplicates''')
|
||||
counts['normalized_doi_collisions'] = _scalar(connection, '''
|
||||
SELECT count(*) FROM (
|
||||
SELECT lower(regexp_replace(btrim(value),
|
||||
?, '', 'i')) AS doi
|
||||
FROM identifiers WHERE lower(scheme)='doi' GROUP BY 1
|
||||
HAVING count(DISTINCT paper_id)>1
|
||||
) duplicates''', (r'^(https?://(dx\.)?doi\.org/|doi:)[[:space:]]*',))
|
||||
counts['multiple_papers_per_base_arxiv_id'] = _scalar(connection, r'''
|
||||
SELECT count(*) FROM (
|
||||
SELECT regexp_replace(btrim(value),'v[0-9]+$','','i') AS arxiv_id
|
||||
FROM identifiers WHERE lower(scheme)='arxiv' GROUP BY 1
|
||||
HAVING count(DISTINCT paper_id)>1
|
||||
) duplicates''')
|
||||
counts['submission_identifier_wrong_edition'] = _scalar(connection, '''
|
||||
SELECT count(*) FROM submission_identifiers i
|
||||
JOIN submissions s ON s.id=i.submission_id WHERE s.edition_id<>i.edition_id''')
|
||||
counts['unvalidated_foreign_key_or_check_constraints'] = _scalar(connection, '''
|
||||
SELECT count(*) FROM pg_constraint k JOIN pg_namespace n ON n.oid=k.connamespace
|
||||
WHERE n.nspname=current_schema() AND k.contype IN ('f','c') AND NOT k.convalidated''')
|
||||
counts['tag_wrong_source_paper'] = _scalar(connection, '''
|
||||
SELECT count(*) FROM paper_tags t JOIN source_records r ON r.id=t.source_record_id
|
||||
WHERE t.paper_id<>r.paper_id''')
|
||||
return counts
|
||||
|
||||
|
||||
def _coverage(connection):
|
||||
return [dict(row) for row in connection.execute('''
|
||||
SELECT v.acronym AS venue,e.year,e.track,e.event_key,o.name AS source,
|
||||
l.list_url,l.status,l.expected_count,l.imported_count
|
||||
FROM edition_sources l JOIN editions e ON e.id=l.edition_id
|
||||
JOIN venues v ON v.id=e.venue_id JOIN sources o ON o.id=l.source_id
|
||||
ORDER BY v.acronym,e.year,e.track,e.event_key,o.name,l.list_url''')]
|
||||
|
||||
|
||||
def _fts(connection):
|
||||
indexed = bool(_scalar(connection, '''
|
||||
SELECT count(*) FROM pg_index i JOIN pg_class t ON t.oid=i.indrelid
|
||||
JOIN pg_class x ON x.oid=i.indexrelid JOIN pg_am a ON a.oid=x.relam
|
||||
JOIN pg_namespace n ON n.oid=t.relnamespace
|
||||
WHERE n.nspname=current_schema() AND t.relname='papers' AND a.amname='gin'
|
||||
AND i.indisvalid AND i.indisready AND x.relname='papers_search_gin'
|
||||
AND strpos(pg_get_indexdef(i.indexrelid),'to_tsvector')>0'''))
|
||||
query = 'SELECT id FROM papers WHERE ' + SEARCH_VECTOR + " @@ websearch_to_tsquery('english',?) LIMIT 5"
|
||||
phrase = 'convolutional vision transformer'
|
||||
plan = [row[0] for row in connection.execute('EXPLAIN (COSTS OFF) '+query, (phrase,))]
|
||||
ids = [row[0] for row in connection.execute(query, (phrase,))]
|
||||
return {'gin_index_ready': indexed, 'plan': plan, 'matched_paper_ids': ids, 'query': phrase}
|
||||
|
||||
|
||||
def _author_fingerprint(raw):
|
||||
authors = sorted(author_keys(json.loads(raw)))
|
||||
if not authors:
|
||||
return None
|
||||
return sha256(json.dumps(authors, ensure_ascii=False).encode()).digest()
|
||||
|
||||
|
||||
def _stage_titles(connection, scratch):
|
||||
scratch.execute('CREATE TABLE titles (id INTEGER PRIMARY KEY, title_key TEXT NOT NULL, authors BLOB)')
|
||||
counts = Counter(papers=0, missing_abstract=0, missing_author_keys=0, invalid_authors_json=0)
|
||||
with connection.connection.cursor(name='paper_atlas_global_identity_scan') as titles:
|
||||
titles.itersize = 5000
|
||||
titles.execute('SELECT id,title_key,authors_json,abstract IS NULL OR btrim(abstract)=\'\' FROM papers ORDER BY id')
|
||||
while rows := titles.fetchmany(5000):
|
||||
batch = []
|
||||
for row in rows:
|
||||
paper_id,title_key,authors,missing = (row[index] for index in range(4))
|
||||
try:
|
||||
fingerprint = _author_fingerprint(authors)
|
||||
except (TypeError, ValueError):
|
||||
fingerprint = None
|
||||
counts['invalid_authors_json'] += 1
|
||||
counts['papers'] += 1
|
||||
counts['missing_abstract'] += int(missing)
|
||||
counts['missing_author_keys'] += int(fingerprint is None)
|
||||
batch.append((paper_id,title_key,fingerprint))
|
||||
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', batch)
|
||||
scratch.commit()
|
||||
scratch.execute('CREATE INDEX keyed_titles ON titles (title_key,authors,id)')
|
||||
scratch.execute('CREATE INDEX title_rosters ON titles (authors,title_key,id) WHERE authors IS NOT NULL')
|
||||
return dict(counts)
|
||||
|
||||
|
||||
def _candidates(scratch, output):
|
||||
counts = Counter(exact_title_groups=0, exact_title_and_roster_groups=0,
|
||||
exact_title_and_roster_papers=0, near_title_and_roster_pairs=0,
|
||||
oversized_rosters=0, papers_in_oversized_rosters=0)
|
||||
def emit(data):
|
||||
output.write(json.dumps(data, ensure_ascii=False, separators=(',',':'))+'\n')
|
||||
counts['exact_title_groups'] = scratch.execute('''
|
||||
SELECT count(*) FROM (SELECT title_key FROM titles WHERE title_key<>''
|
||||
GROUP BY title_key HAVING count(*)>1)''').fetchone()[0]
|
||||
for title,roster,size in scratch.execute('''
|
||||
SELECT title_key,authors,count(*) FROM titles WHERE title_key<>'' AND authors IS NOT NULL
|
||||
GROUP BY title_key,authors HAVING count(*)>1 ORDER BY title_key,authors'''):
|
||||
counts['exact_title_and_roster_groups'] += 1
|
||||
counts['exact_title_and_roster_papers'] += size
|
||||
ids = [row[0] for row in scratch.execute('''
|
||||
SELECT id FROM titles WHERE title_key=? AND authors=? ORDER BY id''', (title,roster))]
|
||||
emit({'kind':'exact_title_and_roster','title_key':title,'paper_ids':ids})
|
||||
cursor = scratch.execute('''
|
||||
SELECT authors,title_key,id FROM titles WHERE authors IS NOT NULL AND length(title_key)>=25
|
||||
ORDER BY authors,title_key,id''')
|
||||
for _, group in groupby(cursor, key=lambda row:row[0]):
|
||||
entries = []
|
||||
size = 0
|
||||
for _, title, paper_id in group:
|
||||
size += 1
|
||||
if size <= ROSTER_GROUP_LIMIT:
|
||||
entries.append((title,paper_id))
|
||||
if size > ROSTER_GROUP_LIMIT:
|
||||
counts['oversized_rosters'] += 1
|
||||
counts['papers_in_oversized_rosters'] += size
|
||||
emit({'kind':'oversized_roster','size':size,'sample_paper_ids':[p for _,p in entries[:5]]})
|
||||
continue
|
||||
for (left,left_id),(right,right_id) in combinations(entries,2):
|
||||
if left==right or 2*min(len(left),len(right))/(len(left)+len(right)) < TITLE_SCORE:
|
||||
continue
|
||||
matcher = SequenceMatcher(None,left,right,autojunk=False)
|
||||
if matcher.quick_ratio()>=TITLE_SCORE and (score:=matcher.ratio())>=TITLE_SCORE:
|
||||
counts['near_title_and_roster_pairs'] += 1
|
||||
emit({'kind':'near_title_and_roster','paper_ids':[left_id,right_id],'score':round(score,5)})
|
||||
return dict(counts)
|
||||
|
||||
|
||||
def report(connection, scratch_root, candidates):
|
||||
transaction = dict(connection.execute('''
|
||||
SELECT current_database() AS database,current_schema() AS schema,
|
||||
current_setting('transaction_read_only') AS read_only,
|
||||
current_setting('transaction_isolation') AS isolation,
|
||||
transaction_timestamp()::text AS snapshot_at''').fetchone())
|
||||
if transaction['read_only']!='on' or transaction['isolation']!='repeatable read':
|
||||
raise ValueError('A read-only repeatable-read snapshot is required')
|
||||
if shutil.disk_usage(scratch_root).free < 2*1024**3:
|
||||
raise ValueError('The scratch filesystem must have at least 2 GiB free')
|
||||
connection.execute("SET LOCAL statement_timeout='180s'")
|
||||
integrity = _checks(connection)
|
||||
coverage = _coverage(connection)
|
||||
search = _fts(connection)
|
||||
with TemporaryDirectory(prefix='paper-atlas-audit-',dir=scratch_root) as directory:
|
||||
with closing(sqlite3.connect(Path(directory)/'titles.sqlite')) as scratch:
|
||||
scratch.execute('PRAGMA temp_store=FILE')
|
||||
counts = _stage_titles(connection,scratch)
|
||||
candidate_counts = _candidates(scratch,candidates)
|
||||
return {'snapshot':transaction,'papers':counts,'integrity':integrity,
|
||||
'candidate_counts':candidate_counts,'edition_sources':coverage,'search':search,
|
||||
'candidate_rules':{'minimum_normalized_title_length':25,'near_title_score':TITLE_SCORE,
|
||||
'max_papers_per_near_title_author_group':ROSTER_GROUP_LIMIT,
|
||||
'requires_exact_normalized_author_set':True},
|
||||
'canonical_data_changed':False}
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog,description='只读核验 PostgreSQL 全库身份候选、约束、来源覆盖和全文检索')
|
||||
parser.add_argument('--pg-service',required=True)
|
||||
parser.add_argument('--pg-schema')
|
||||
parser.add_argument('--scratch-root',type=Path,required=True)
|
||||
parser.add_argument('--candidate-output',type=Path,required=True)
|
||||
args = parser.parse_args(argv)
|
||||
with closing(ReadConnection(args.pg_service,args.pg_schema)) as connection:
|
||||
with args.candidate_output.open('x',encoding='utf-8') as candidates:
|
||||
result = report(connection,args.scratch_root,candidates)
|
||||
print(json.dumps(result,ensure_ascii=False))
|
||||
@@ -66,6 +66,7 @@ COMMANDS = {
|
||||
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
|
||||
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
|
||||
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
|
||||
('audit','global-identity'): ('paper_atlas.audits.global_identity', {}, '只读核对 PostgreSQL 身份候选与完整性'),
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
import io
|
||||
import json
|
||||
import sqlite3
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from paper_atlas.audits import global_identity
|
||||
|
||||
|
||||
class GlobalIdentityCandidatesTest(unittest.TestCase):
|
||||
def test_exact_same_title_and_near_title_require_author_evidence(self):
|
||||
output = io.StringIO()
|
||||
with sqlite3.connect(':memory:') as scratch:
|
||||
scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)')
|
||||
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [
|
||||
(1, 'astronomicaltransientcatalogue', b'first'),
|
||||
(2, 'astronomicaltransientcatalogue', b'first'),
|
||||
(3, 'astronomicaltransientcatalogues', b'first'),
|
||||
(4, 'astronomicaltransientcatalogue', b'second'),
|
||||
(5, 'astronomicaltransientcatalogue', None),
|
||||
])
|
||||
counts = global_identity._candidates(scratch, output)
|
||||
|
||||
records = [json.loads(row) for row in output.getvalue().splitlines()]
|
||||
self.assertEqual(counts['exact_title_groups'], 1)
|
||||
self.assertEqual(counts['exact_title_and_roster_papers'], 2)
|
||||
self.assertEqual(counts['near_title_and_roster_pairs'], 2)
|
||||
self.assertEqual(records[0]['paper_ids'], [1, 2])
|
||||
self.assertEqual({tuple(r['paper_ids']) for r in records[1:]}, {(1, 3), (2, 3)})
|
||||
|
||||
def test_author_roster_limit_is_visible_in_receipt(self):
|
||||
output = io.StringIO()
|
||||
with sqlite3.connect(':memory:') as scratch:
|
||||
scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)')
|
||||
scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [
|
||||
(1, 'astronomicaltransientcatalogue', b'first'),
|
||||
(2, 'astronomicaltransientcatalogues', b'first'),
|
||||
(3, 'astronomicaltransientcatalogued', b'first'),
|
||||
])
|
||||
with patch.object(global_identity, 'ROSTER_GROUP_LIMIT', 2):
|
||||
counts = global_identity._candidates(scratch, output)
|
||||
|
||||
self.assertEqual(counts['oversized_rosters'], 1)
|
||||
self.assertEqual(counts['papers_in_oversized_rosters'], 3)
|
||||
self.assertEqual(counts['near_title_and_roster_pairs'], 0)
|
||||
self.assertEqual(json.loads(output.getvalue())['kind'], 'oversized_roster')
|
||||
|
||||
def test_fingerprint_normalizes_supported_author_encodings(self):
|
||||
strings = json.dumps(['Alice Doe', 'Chloë Li'])
|
||||
records = json.dumps([['Li', 'Chloë'], {'given':'Alice','family':'Doe'}])
|
||||
self.assertEqual(global_identity._author_fingerprint(strings),
|
||||
global_identity._author_fingerprint(records))
|
||||
self.assertIsNone(global_identity._author_fingerprint('[]'))
|
||||
|
||||
def test_doi_pattern_is_bound_as_a_parameter(self):
|
||||
connection = Mock()
|
||||
|
||||
def execute(query, params):
|
||||
self.assertEqual(query.count('?'), len(params))
|
||||
if params:
|
||||
pattern, = params
|
||||
self.assertIn('https?', pattern)
|
||||
result = Mock()
|
||||
result.fetchone.return_value = (0,)
|
||||
return result
|
||||
|
||||
connection.execute.side_effect = execute
|
||||
self.assertEqual(global_identity._checks(connection)['normalized_doi_collisions'], 0)
|
||||
Reference in New Issue
Block a user