From 4c834dc10e043473daba0a6ae567438e7dd7605e Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 22 Nov 2026 15:02:25 +0800 Subject: [PATCH] Add a reproducible PostgreSQL global identity audit --- README.md | 4 +- docs/operations.md | 16 +++ paper_atlas/audits/global_identity.py | 199 ++++++++++++++++++++++++++ paper_atlas/cli.py | 1 + tests/test_global_identity_audit.py | 68 +++++++++ 5 files changed, 287 insertions(+), 1 deletion(-) create mode 100644 paper_atlas/audits/global_identity.py create mode 100644 tests/test_global_identity_audit.py diff --git a/README.md b/README.md index ce72192..271d23c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 -项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 +历史目录包含 43 个会议。主动维护以核心 AI 会议及其已公开的决定为主,继续跟踪 ACM MM 和 MLSys。此前入库的 CIKM 数据与证据保留,后续主动补漏范围排除 CIKM。实际覆盖以数据库和来源清单为准。 当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。 @@ -108,6 +108,8 @@ AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的 CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤,不因代码测试通过而自动完成。 +一批数据全部验收后,运行 [`audit global-identity`](docs/operations.md#导入后的全库身份核验) 核对 PostgreSQL 的稳定标识、同题与近似标题候选、摘要完整性和全文检索。审计报告和候选文件对应一次快照,后续导入完成后重新生成。使用 `audit histories` 检查跨会议投稿决定及其来源关系。 + 开发通过工作分支和 PR 交付,由用户手动合并。 早期WWW档案的可重放元数据采集、论文/海报分轨与待核查条目见 [WWW档案采集](docs/www-archive.md)。 diff --git a/docs/operations.md b/docs/operations.md index ae875f2..96e7c24 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,5 +1,21 @@ # 运行维护与数据核验 +## 导入后的全库身份核验 + +在服务器准备一个至少有 2 GiB 可用空间的审计目录。使用生产读取角色,在计划中的导入全部验收后运行以下命令。命令在 PostgreSQL 的同一个可重复读快照内检查库内记录。SQLite 临时文件包含用来比较的标题键与作者指纹,扫描结束后清理。 + +```bash +paper-atlas audit global-identity \ + --pg-service foundry-research --pg-schema research \ + --scratch-root /data/paper-index/validation \ + --candidate-output /data/paper-index/validation/identity-candidates-2026-11-22.jsonl \ + > /data/paper-index/validation/identity-report-2026-11-22.json +``` + +JSON 报告记录快照时间、论文与缺摘要计数、稳定标识冲突、约束状态、会议年度与轨道的来源覆盖,以及一次实际的全文搜索和执行计划。JSONL 保存同题且规范化作者集合相同的论文组,以及相同作者集合中标题相似度至少为 0.90 的论文对。超过 100 条记录的作者组会列入报告,留作后续逐组核对。缺作者、作者增减或换名的版本需要结合 DOI、论坛或作者说明,使用 `audit source-identities` 和原始来源复核。 + +将报告、候选文件、运行版本与 SHA-256 一起保留。使用 `audit histories` 核对拒稿、撤稿、后续录用和来源链接。需要合并候选时先确认论文版本关系并保存投稿、决定、标签及来源证据,再按已授权的流程操作。新一批导入完成后更新快照和审计报告。 + ## PostgreSQL 迁移 共享实例的容器名为 `foundry-postgres`,底层镜像为 `postgres:17-alpine`;Compose 服务名仍为 `postgres`。数据库和 schema 是可配置的逻辑边界,表按研究数据域组织,不按调用项目复制。其他项目可获授 `research` 的 USAGE 及必要表权限,复用相同论文 ID 和表。未来换库只更新 libpq service;换 schema 可更新角色 search_path 或查询的 `--pg-schema`。 diff --git a/paper_atlas/audits/global_identity.py b/paper_atlas/audits/global_identity.py new file mode 100644 index 0000000..645aa1c --- /dev/null +++ b/paper_atlas/audits/global_identity.py @@ -0,0 +1,199 @@ +"""Read-only PostgreSQL identity and integrity audit for an existing snapshot.""" + +import argparse +from collections import Counter +from contextlib import closing +from difflib import SequenceMatcher +from hashlib import sha256 +from itertools import combinations, groupby +import json +from pathlib import Path +import shutil +import sqlite3 +from tempfile import TemporaryDirectory + +from paper_atlas.db.store import author_keys +from paper_atlas.postgres.client import ReadConnection, SEARCH_VECTOR + + +ROSTER_GROUP_LIMIT = 100 +TITLE_SCORE = .90 + + +def _scalar(connection, query, params=()): + return connection.execute(query, params).fetchone()[0] + + +def _checks(connection): + counts = {} + for table, columns in ( + ('identifiers', 'scheme,value'), + ('source_records', 'source_id,external_id'), + ('submission_identifiers', 'edition_id,scheme,value'), + ): + counts[f'duplicate_{table}_keys'] = _scalar(connection, f''' + SELECT count(*) FROM ( + SELECT {columns} FROM {table} GROUP BY {columns} HAVING count(*)>1 + ) duplicates''') + counts['normalized_doi_collisions'] = _scalar(connection, ''' + SELECT count(*) FROM ( + SELECT lower(regexp_replace(btrim(value), + ?, '', 'i')) AS doi + FROM identifiers WHERE lower(scheme)='doi' GROUP BY 1 + HAVING count(DISTINCT paper_id)>1 + ) duplicates''', (r'^(https?://(dx\.)?doi\.org/|doi:)[[:space:]]*',)) + counts['multiple_papers_per_base_arxiv_id'] = _scalar(connection, r''' + SELECT count(*) FROM ( + SELECT regexp_replace(btrim(value),'v[0-9]+$','','i') AS arxiv_id + FROM identifiers WHERE lower(scheme)='arxiv' GROUP BY 1 + HAVING count(DISTINCT paper_id)>1 + ) duplicates''') + counts['submission_identifier_wrong_edition'] = _scalar(connection, ''' + SELECT count(*) FROM submission_identifiers i + JOIN submissions s ON s.id=i.submission_id WHERE s.edition_id<>i.edition_id''') + counts['unvalidated_foreign_key_or_check_constraints'] = _scalar(connection, ''' + SELECT count(*) FROM pg_constraint k JOIN pg_namespace n ON n.oid=k.connamespace + WHERE n.nspname=current_schema() AND k.contype IN ('f','c') AND NOT k.convalidated''') + counts['tag_wrong_source_paper'] = _scalar(connection, ''' + SELECT count(*) FROM paper_tags t JOIN source_records r ON r.id=t.source_record_id + WHERE t.paper_id<>r.paper_id''') + return counts + + +def _coverage(connection): + return [dict(row) for row in connection.execute(''' + SELECT v.acronym AS venue,e.year,e.track,e.event_key,o.name AS source, + l.list_url,l.status,l.expected_count,l.imported_count + FROM edition_sources l JOIN editions e ON e.id=l.edition_id + JOIN venues v ON v.id=e.venue_id JOIN sources o ON o.id=l.source_id + ORDER BY v.acronym,e.year,e.track,e.event_key,o.name,l.list_url''')] + + +def _fts(connection): + indexed = bool(_scalar(connection, ''' + SELECT count(*) FROM pg_index i JOIN pg_class t ON t.oid=i.indrelid + JOIN pg_class x ON x.oid=i.indexrelid JOIN pg_am a ON a.oid=x.relam + JOIN pg_namespace n ON n.oid=t.relnamespace + WHERE n.nspname=current_schema() AND t.relname='papers' AND a.amname='gin' + AND i.indisvalid AND i.indisready AND x.relname='papers_search_gin' + AND strpos(pg_get_indexdef(i.indexrelid),'to_tsvector')>0''')) + query = 'SELECT id FROM papers WHERE ' + SEARCH_VECTOR + " @@ websearch_to_tsquery('english',?) LIMIT 5" + phrase = 'convolutional vision transformer' + plan = [row[0] for row in connection.execute('EXPLAIN (COSTS OFF) '+query, (phrase,))] + ids = [row[0] for row in connection.execute(query, (phrase,))] + return {'gin_index_ready': indexed, 'plan': plan, 'matched_paper_ids': ids, 'query': phrase} + + +def _author_fingerprint(raw): + authors = sorted(author_keys(json.loads(raw))) + if not authors: + return None + return sha256(json.dumps(authors, ensure_ascii=False).encode()).digest() + + +def _stage_titles(connection, scratch): + scratch.execute('CREATE TABLE titles (id INTEGER PRIMARY KEY, title_key TEXT NOT NULL, authors BLOB)') + counts = Counter(papers=0, missing_abstract=0, missing_author_keys=0, invalid_authors_json=0) + with connection.connection.cursor(name='paper_atlas_global_identity_scan') as titles: + titles.itersize = 5000 + titles.execute('SELECT id,title_key,authors_json,abstract IS NULL OR btrim(abstract)=\'\' FROM papers ORDER BY id') + while rows := titles.fetchmany(5000): + batch = [] + for row in rows: + paper_id,title_key,authors,missing = (row[index] for index in range(4)) + try: + fingerprint = _author_fingerprint(authors) + except (TypeError, ValueError): + fingerprint = None + counts['invalid_authors_json'] += 1 + counts['papers'] += 1 + counts['missing_abstract'] += int(missing) + counts['missing_author_keys'] += int(fingerprint is None) + batch.append((paper_id,title_key,fingerprint)) + scratch.executemany('INSERT INTO titles VALUES (?,?,?)', batch) + scratch.commit() + scratch.execute('CREATE INDEX keyed_titles ON titles (title_key,authors,id)') + scratch.execute('CREATE INDEX title_rosters ON titles (authors,title_key,id) WHERE authors IS NOT NULL') + return dict(counts) + + +def _candidates(scratch, output): + counts = Counter(exact_title_groups=0, exact_title_and_roster_groups=0, + exact_title_and_roster_papers=0, near_title_and_roster_pairs=0, + oversized_rosters=0, papers_in_oversized_rosters=0) + def emit(data): + output.write(json.dumps(data, ensure_ascii=False, separators=(',',':'))+'\n') + counts['exact_title_groups'] = scratch.execute(''' + SELECT count(*) FROM (SELECT title_key FROM titles WHERE title_key<>'' + GROUP BY title_key HAVING count(*)>1)''').fetchone()[0] + for title,roster,size in scratch.execute(''' + SELECT title_key,authors,count(*) FROM titles WHERE title_key<>'' AND authors IS NOT NULL + GROUP BY title_key,authors HAVING count(*)>1 ORDER BY title_key,authors'''): + counts['exact_title_and_roster_groups'] += 1 + counts['exact_title_and_roster_papers'] += size + ids = [row[0] for row in scratch.execute(''' + SELECT id FROM titles WHERE title_key=? AND authors=? ORDER BY id''', (title,roster))] + emit({'kind':'exact_title_and_roster','title_key':title,'paper_ids':ids}) + cursor = scratch.execute(''' + SELECT authors,title_key,id FROM titles WHERE authors IS NOT NULL AND length(title_key)>=25 + ORDER BY authors,title_key,id''') + for _, group in groupby(cursor, key=lambda row:row[0]): + entries = [] + size = 0 + for _, title, paper_id in group: + size += 1 + if size <= ROSTER_GROUP_LIMIT: + entries.append((title,paper_id)) + if size > ROSTER_GROUP_LIMIT: + counts['oversized_rosters'] += 1 + counts['papers_in_oversized_rosters'] += size + emit({'kind':'oversized_roster','size':size,'sample_paper_ids':[p for _,p in entries[:5]]}) + continue + for (left,left_id),(right,right_id) in combinations(entries,2): + if left==right or 2*min(len(left),len(right))/(len(left)+len(right)) < TITLE_SCORE: + continue + matcher = SequenceMatcher(None,left,right,autojunk=False) + if matcher.quick_ratio()>=TITLE_SCORE and (score:=matcher.ratio())>=TITLE_SCORE: + counts['near_title_and_roster_pairs'] += 1 + emit({'kind':'near_title_and_roster','paper_ids':[left_id,right_id],'score':round(score,5)}) + return dict(counts) + + +def report(connection, scratch_root, candidates): + transaction = dict(connection.execute(''' + SELECT current_database() AS database,current_schema() AS schema, + current_setting('transaction_read_only') AS read_only, + current_setting('transaction_isolation') AS isolation, + transaction_timestamp()::text AS snapshot_at''').fetchone()) + if transaction['read_only']!='on' or transaction['isolation']!='repeatable read': + raise ValueError('A read-only repeatable-read snapshot is required') + if shutil.disk_usage(scratch_root).free < 2*1024**3: + raise ValueError('The scratch filesystem must have at least 2 GiB free') + connection.execute("SET LOCAL statement_timeout='180s'") + integrity = _checks(connection) + coverage = _coverage(connection) + search = _fts(connection) + with TemporaryDirectory(prefix='paper-atlas-audit-',dir=scratch_root) as directory: + with closing(sqlite3.connect(Path(directory)/'titles.sqlite')) as scratch: + scratch.execute('PRAGMA temp_store=FILE') + counts = _stage_titles(connection,scratch) + candidate_counts = _candidates(scratch,candidates) + return {'snapshot':transaction,'papers':counts,'integrity':integrity, + 'candidate_counts':candidate_counts,'edition_sources':coverage,'search':search, + 'candidate_rules':{'minimum_normalized_title_length':25,'near_title_score':TITLE_SCORE, + 'max_papers_per_near_title_author_group':ROSTER_GROUP_LIMIT, + 'requires_exact_normalized_author_set':True}, + 'canonical_data_changed':False} + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog,description='只读核验 PostgreSQL 全库身份候选、约束、来源覆盖和全文检索') + parser.add_argument('--pg-service',required=True) + parser.add_argument('--pg-schema') + parser.add_argument('--scratch-root',type=Path,required=True) + parser.add_argument('--candidate-output',type=Path,required=True) + args = parser.parse_args(argv) + with closing(ReadConnection(args.pg_service,args.pg_schema)) as connection: + with args.candidate_output.open('x',encoding='utf-8') as candidates: + result = report(connection,args.scratch_root,candidates) + print(json.dumps(result,ensure_ascii=False)) diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 1bb2f24..1c771f1 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -66,6 +66,7 @@ COMMANDS = { ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'), ('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'), ('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'), + ('audit','global-identity'): ('paper_atlas.audits.global_identity', {}, '只读核对 PostgreSQL 身份候选与完整性'), } diff --git a/tests/test_global_identity_audit.py b/tests/test_global_identity_audit.py new file mode 100644 index 0000000..1f90625 --- /dev/null +++ b/tests/test_global_identity_audit.py @@ -0,0 +1,68 @@ +import io +import json +import sqlite3 +import unittest +from unittest.mock import Mock, patch + +from paper_atlas.audits import global_identity + + +class GlobalIdentityCandidatesTest(unittest.TestCase): + def test_exact_same_title_and_near_title_require_author_evidence(self): + output = io.StringIO() + with sqlite3.connect(':memory:') as scratch: + scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)') + scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [ + (1, 'astronomicaltransientcatalogue', b'first'), + (2, 'astronomicaltransientcatalogue', b'first'), + (3, 'astronomicaltransientcatalogues', b'first'), + (4, 'astronomicaltransientcatalogue', b'second'), + (5, 'astronomicaltransientcatalogue', None), + ]) + counts = global_identity._candidates(scratch, output) + + records = [json.loads(row) for row in output.getvalue().splitlines()] + self.assertEqual(counts['exact_title_groups'], 1) + self.assertEqual(counts['exact_title_and_roster_papers'], 2) + self.assertEqual(counts['near_title_and_roster_pairs'], 2) + self.assertEqual(records[0]['paper_ids'], [1, 2]) + self.assertEqual({tuple(r['paper_ids']) for r in records[1:]}, {(1, 3), (2, 3)}) + + def test_author_roster_limit_is_visible_in_receipt(self): + output = io.StringIO() + with sqlite3.connect(':memory:') as scratch: + scratch.execute('CREATE TABLE titles (id INTEGER, title_key TEXT, authors BLOB)') + scratch.executemany('INSERT INTO titles VALUES (?,?,?)', [ + (1, 'astronomicaltransientcatalogue', b'first'), + (2, 'astronomicaltransientcatalogues', b'first'), + (3, 'astronomicaltransientcatalogued', b'first'), + ]) + with patch.object(global_identity, 'ROSTER_GROUP_LIMIT', 2): + counts = global_identity._candidates(scratch, output) + + self.assertEqual(counts['oversized_rosters'], 1) + self.assertEqual(counts['papers_in_oversized_rosters'], 3) + self.assertEqual(counts['near_title_and_roster_pairs'], 0) + self.assertEqual(json.loads(output.getvalue())['kind'], 'oversized_roster') + + def test_fingerprint_normalizes_supported_author_encodings(self): + strings = json.dumps(['Alice Doe', 'Chloë Li']) + records = json.dumps([['Li', 'Chloë'], {'given':'Alice','family':'Doe'}]) + self.assertEqual(global_identity._author_fingerprint(strings), + global_identity._author_fingerprint(records)) + self.assertIsNone(global_identity._author_fingerprint('[]')) + + def test_doi_pattern_is_bound_as_a_parameter(self): + connection = Mock() + + def execute(query, params): + self.assertEqual(query.count('?'), len(params)) + if params: + pattern, = params + self.assertIn('https?', pattern) + result = Mock() + result.fetchone.return_value = (0,) + return result + + connection.execute.side_effect = execute + self.assertEqual(global_identity._checks(connection)['normalized_doi_collisions'], 0)