Compare commits

..

1 Commits

Author SHA1 Message Date
codex-bot 3f3dc3f7ad Distinguish historical primary editions from affiliated workshops
Metadata integrity checks / metadata (push) Successful in 1m9s
2026-11-08 15:12:18 +08:00
13 changed files with 117 additions and 539 deletions
-1
View File
@@ -54,7 +54,6 @@ jobs:
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
+4
View File
@@ -178,6 +178,10 @@ paper-atlas collect dblp \
输出按会议/年份/track 分文件,附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记,不默认算作主会。DBLP 不含摘要,因此输出摘要为空、清单为 partial;需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留,不依据书目缺席推断拒稿。已存在其他出版来源的届次,还须先回填其权威身份并核验镜像记录,不能直接累加投稿数。
早期 AISTATS 1995/1997/1999/2001/2003/2005 和 COLT 1988–1991 的主系列名称本身含 Workshop,不能当作附属 workshop 漏掉。识别要求已核对的完整 proceedings key、会议书目名、原始标题前缀、届数和事件年一致;轨道名为 `main` 表示主系列,保留原始 Workshop 名称及官方目录证据,不改造其历史名称或出版形式。参见 [AISTATS 官方历史目录](https://aistats.org/other.html)、[COLT 协会目录](https://www.learningtheory.org/)。1995 AISTATS 的会议年与1996图书出版年分开;[PMLR R0](https://proceedings.mlr.press/r0/) 是后来重发的 pre-proceedings,不与 DBLP 图书条目盲目逐篇等同。
真正附属的 AISTATS 2010 Active Learning and Experimental Design workshop 仍单列。COLT 2003 与 Kernel Workshop 的 [混合论文集](https://doi.org/10.1007/b12006) 缺少逐篇章节信息,使用 `proceedings_unspecified`,不能整卷改成 main 或附属 workshop。输出保存 `track_scope_note` 与 `track_scope_evidence_url`,导入后仍可追查分类理由;这些出版证据不是真实拒稿或原始决定签名,也不自动修正既有主库记录。
部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键;只有论文集标题明确给出唯一完整年份且与短键相符时才采用,不推测世纪。WWW 特殊兴趣、Alternate track 单列;同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`,不把整卷归为某一个具体 track。
书目中的前后页、目录组合和明确会务标题不产生论文录用记录;缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料,原始快照仍保留。范围报告分别统计这些原因;单纯缺作者并不排除真实研究论文,也不补造作者。
-51
View File
@@ -2,55 +2,6 @@
先在已确认的远程环境安装项目,使用 `paper-atlas`。从源码运行时,将命令前缀替换为 `python3 -B -m paper_atlas`,其余参数相同。数据目录位于仓库外。
## 按证据补充稳定标识
`paper-atlas curate identifiers --plan /data/review/plan.json --output /data/review/prepared`
只生成新的 JSONL、manifest、证据副本及 `curation-report.json`,不会连接数据库。
输出目录必须不存在,父目录必须已存在;任何输入或断言失败,都不发布本批输出。
原始文件不变,未选中的记录逐字节保留。题名、作者、年份、页码、投稿决定、标签和既有 metadata
均不重写,仅补充 `identifiers` 并追加 `metadata.identifier_curation` 证据链。
计划格式如下(哈希占位符须替换为真实 SHA256,所有路径按命令当前目录解析):
```json
{
"format_version": 1,
"inputs": [{
"file": "/data/review/uai2001.jsonl",
"sha256": "<JSONL 文件 SHA256>",
"manifest_sha256": "<同名 .manifest.json 文件 SHA256>"
}],
"assertions": [{
"source": "dblp",
"external_id": "conf/uai/example",
"record_sha256": "<原始物理行字节 SHA256,包含该行原有换行符>",
"identifiers": {"arxiv": "1408.1485"},
"evidence": [{
"file": "/data/review/primary-metadata.json",
"sha256": "<证据 JSON 文件 SHA256>",
"url": "https://example.org/primary-record",
"checked_at": "2026-11-08T08:00:00Z",
"note": "说明人工核对的题名、作者、出版版本和稳定标识证据"
}]
}]
}
```
输入 manifest 必须包含 `source/venue/year/track/count/list_url/file_sha256`,且与逐条记录一致。
断言必须唯一、准确命中一条记录。仅支持无版本后缀的 arXiv 编号与小写规范 DOI;
已有同类标识不允许覆盖,相同值可以补记证据。证据只接受不超过 16 MiB 的 JSON 元数据文件,
来源 URL 要求 HTTPS 且不含凭据,核查时间必须带时区;不下载论文 PDF。
`verification_scope=input_and_evidence_integrity_only` 和 `semantic_identity_verified=false`
明确表示哈希校验不能证明两条记录属于同一作品。请先独立核对版本关系;仅年份、页码或相同题名
不能作为合并依据。存在来源冲突的行应继续暂缓。新 manifest 保留原来的 partial 状态及摘要缺失数,
不会把补充编号伪装成来源覆盖或摘要补全。证据副本按 SHA256 保存在输出目录 `evidence/`,
原计划也保存在 `plan.json`,供之后复查。
文件生成后仍需另做当前 PostgreSQL 身份、投稿和决定冲突预检;导入运行时必须包含
“稳定标识优先、多个标题候选则整文件回滚”的身份保护,再以 `import records` 独立导入并验收。
该命令不会自动导入或合并实体,也不代表最终全库去重通过。
## PostgreSQL 查询
在远程环境安装 `.[postgres]`,通过权限为 `0600` 的 libpq service 文件连接,不把密码放入 URL、命令行或仓库。
@@ -148,8 +99,6 @@ paper-atlas query --pg-service foundry-research \
`--stats` 中 papers 是当前规则下的论文实体数;source_records 是来源记录数;decisions 是决定观察数。一个投稿可能有多个来源和多条观察,不能相加当作论文数。
已有稳定标识或原来源记录匹配时,该身份优先;同标题、同作者的另一 arXiv 存档或扩展稿不会仅因相似标题被并入。没有身份锚点时,作者佐证的精确标题只有一个候选才可匹配;多个候选会报 `Ambiguous paper identity`,整个采集文件回滚并记录失败运行。应补有证据的标识或先核查版本关系,不删除重复候选或任取第一个。一个新来源明确同时携带两个既有稳定标识时,仍可按既有规则归并并保留投稿、标签和合并日志;未归并的同标题关系继续保留为待核查候选。这些规则不证明全库去重完成。
## 5. 投稿历史关系审计
```sh
+1 -2
View File
@@ -6,7 +6,6 @@ from paper_atlas import __version__
COMMANDS = {
('curate','identifiers'): ('paper_atlas.curation.identifiers', {}, '按证据补充稳定标识并生成独立待导入文件'),
('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'),
('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'),
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
@@ -49,7 +48,7 @@ def main(argv=None):
leaf=commands.add_parser(route[0],help=help_text,add_help=False)
else:
if route[0] not in groups:
group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据','curate':'按证据整理元数据'}[route[0]])
group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据'}[route[0]])
groups[route[0]]=group.add_subparsers(dest='operation',required=True)
leaf=groups[route[0]].add_parser(route[1],help=help_text,add_help=False)
leaf.set_defaults(handler=(route,module,kwargs))
+2
View File
@@ -90,6 +90,8 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
if edition.get('track_scope_note'):
metadata['track_scope_note']=edition['track_scope_note']
if edition.get('track_scope_evidence_url'):
metadata['track_scope_evidence_url']=edition['track_scope_evidence_url']
flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I))
result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html',
'title':title,'abstract':None,'authors':authors,'identifiers':own,
+39 -1
View File
@@ -38,6 +38,39 @@ BOOKS = {
}
# These are the historical primary series themselves, not co-located workshops.
# Exact proceedings keys, original names, ordinals and event years must agree.
HISTORICAL_WORKSHOP_TITLES = {
**{('aistats', year): prefix + ordinal + ' International Workshop on Artificial Intelligence and Statistics'
for year, ordinal, prefix in [
(1995, 'Fifth', 'Learning from Data - '),
(1997, 'Sixth', 'Proceedings of the '),
(1999, 'Seventh', 'Proceedings of the '),
(2001, 'Eighth', 'Proceedings of the '),
(2003, 'Ninth', 'Proceedings of the '),
(2005, 'Tenth', 'Proceedings of the ')]},
**{('colt', year): 'Proceedings of the ' + ordinal + ' Annual Workshop on Computational Learning Theory'
for year, ordinal in [(1988, 'First'), (1989, 'Second'), (1990, 'Third'), (1991, 'Fourth')]},
}
def historical_track(family, base, suffix, volume_key, year, title):
if suffix or base.casefold() != family or volume_key != f'conf/{family}/{year}':
return None
prefix = HISTORICAL_WORKSHOP_TITLES.get((family, year))
if (prefix and title.casefold().startswith(prefix.casefold() + ',')
and re.search(r'\b' + str(year) + r'\b', title)):
url = 'https://aistats.org/other.html' if family == 'aistats' else 'https://www.learningtheory.org/'
return ('main', 'Historical primary series named Workshop; exact proceedings key, ordinal, title and event year verified.', url)
if (family == 'colt' and year == 2003
and title.startswith('Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003,')
and title.endswith('Proceedings')):
return ('proceedings_unspecified',
'Combined COLT 2003 and Kernel Workshop volume; per-paper section absent from DBLP XML. Do not infer a main or workshop track.',
'https://doi.org/10.1007/b12006')
return None
def scope(row,volume,through_year=2026):
fields=row['fields'];books=fields.get('booktitle',[])
if len(books)!=1:return None,'ambiguous_booktitle'
@@ -75,7 +108,9 @@ def scope(row,volume,through_year=2026):
if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
text=(suffix+' '+title).casefold();track='main'
if 'findings' in text:track='findings'
historical = historical_track(family, base, suffix, volume['key'], year, title)
if historical:track=historical[0]
elif 'findings' in text:track='findings'
elif 'shared task' in text:track='shared_task'
elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop'
elif ('demo track' in text and ('industrial track' in text or 'applied data science track' in text)):track='proceedings_unspecified'
@@ -98,6 +133,9 @@ def scope(row,volume,through_year=2026):
if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas'
result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
if historical:
result['track_scope_note'] = historical[1]
result['track_scope_evidence_url'] = historical[2]
if venue=='MM' and track=='main':
# A main MM volume also includes doctoral, tutorials, companion papers
# and other sections. DBLP XML omits the per-paper TOC section.
-1
View File
@@ -1 +0,0 @@
"""Offline, evidence-bound preparation of metadata for separate ingestion."""
-212
View File
@@ -1,212 +0,0 @@
"""Add reviewed stable identifiers without rewriting bibliographic source fields.
Hashes establish file integrity, not the truth of an identity assertion. Identity
review and a fresh database preflight remain separate requirements.
"""
import argparse
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import urlsplit
SCOPE = 'input_and_evidence_integrity_only'
MAX_METADATA_BYTES = 16 * 1024 * 1024
def digest(data):
return hashlib.sha256(data).hexdigest()
def object_keys(value, required, optional=()):
if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional):
raise ValueError('Missing or unsupported plan fields')
def json_bytes(value):
return (json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode('utf-8')
def decode(raw):
def unique(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError('Duplicate JSON key: ' + key)
result[key] = value
return result
return json.loads(raw, object_pairs_hook=unique,
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Non-finite JSON value')))
def metadata(path, expected=None):
with Path(path).open('rb') as stream:
raw = stream.read(MAX_METADATA_BYTES + 1)
if len(raw) > MAX_METADATA_BYTES:
raise ValueError('Metadata exceeds size limit')
if expected is not None:
check_hash(raw, expected)
return raw, decode(raw)
def check_hash(raw, expected):
if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected) or digest(raw) != expected:
raise ValueError('SHA256 mismatch')
def identifier(scheme, value):
if not isinstance(value, str) or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value):
raise ValueError('Invalid identifier')
if scheme == 'arxiv':
pattern = r'(?:\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}|[a-zA-Z][a-zA-Z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3})'
valid = re.fullmatch(pattern, value)
elif scheme == 'doi':
valid = re.fullmatch(r'10\.\d{4,9}/\S+', value) and value == value.lower()
else:
valid = False
if not valid:
raise ValueError('Unsupported or noncanonical identifier: ' + scheme)
def prepare(plan_path, output):
plan_raw, plan = metadata(plan_path)
object_keys(plan, ('format_version', 'inputs', 'assertions'))
if type(plan['format_version']) is not int or plan['format_version'] != 1:
raise ValueError('Unsupported format version')
if not isinstance(plan['inputs'], list) or not plan['inputs'] or not isinstance(plan['assertions'], list) or not plan['assertions']:
raise ValueError('Inputs and assertions must be nonempty lists')
output = Path(output).absolute()
if output.exists():
raise ValueError('Output directory must not already exist')
if not output.parent.is_dir():
raise ValueError('Output parent directory must exist')
assertions = {}
evidence_files = {}
checked_evidence = {}
for assertion in plan['assertions']:
object_keys(assertion, ('source', 'external_id', 'record_sha256', 'identifiers', 'evidence'))
if any(not isinstance(assertion[key], str) or not assertion[key] for key in ('source', 'external_id')):
raise ValueError('Invalid source identity')
key = (assertion['source'], assertion['external_id'])
if key in assertions:
raise ValueError('Duplicate identity assertion')
ids = assertion['identifiers']
if not isinstance(ids, dict) or not ids:
raise ValueError('Identifiers must be a nonempty object')
for scheme, value in ids.items():
identifier(scheme, value)
if not isinstance(assertion['evidence'], list) or not assertion['evidence']:
raise ValueError('Evidence is required')
references = []
for evidence in assertion['evidence']:
object_keys(evidence, ('file', 'sha256', 'url', 'checked_at', 'note'))
if not all(isinstance(value, str) and value for value in evidence.values()):
raise ValueError('Invalid evidence fields')
url = urlsplit(evidence['url'])
if url.scheme != 'https' or not url.hostname or url.username or url.password:
raise ValueError('Evidence requires an HTTPS URL without credentials')
if datetime.fromisoformat(evidence['checked_at'].replace('Z', '+00:00')).utcoffset() is None:
raise ValueError('Evidence timestamp requires a timezone')
evidence_key = (str(Path(evidence['file']).absolute()), evidence['sha256'])
if evidence_key not in checked_evidence:
checked_evidence[evidence_key] = metadata(evidence['file'], evidence['sha256'])[0]
raw = checked_evidence[evidence_key]
evidence_files[evidence['sha256']] = raw
references.append({**evidence, 'file': 'evidence/' + evidence['sha256'] + '.json'})
assertions[key] = (assertion, references)
seen = set()
applied = set()
names = set()
report = {'format_version': 1, 'plan_sha256': digest(plan_raw),
'verification_scope': SCOPE, 'semantic_identity_verified': False,
'database_written': False, 'files': [], 'assertions_applied': 0, 'identifiers_added': 0}
with tempfile.TemporaryDirectory(prefix='.identifier-curation-', dir=output.parent) as temporary:
staging = Path(temporary) / 'prepared'
staging.mkdir()
(staging / 'evidence').mkdir()
(staging / 'plan.json').write_bytes(plan_raw)
for sha, raw in evidence_files.items():
(staging / 'evidence' / (sha + '.json')).write_bytes(raw)
for entry in plan['inputs']:
object_keys(entry, ('file', 'sha256', 'manifest_sha256'))
path = Path(entry['file'])
if path.suffix != '.jsonl' or path.name in names:
raise ValueError('Inputs require distinct JSONL basenames')
names.add(path.name)
_, manifest = metadata(path.with_suffix('.manifest.json'), entry['manifest_sha256'])
if not isinstance(manifest, dict) or not {'source','venue','year','track','count','list_url','file_sha256'} <= manifest.keys():
raise ValueError('Complete source manifest required')
if manifest['file_sha256'] != entry['sha256']:
raise ValueError('Manifest and plan hashes differ')
before = hashlib.sha256()
after = hashlib.sha256()
count = 0
with path.open('rb') as source, (staging / path.name).open('wb') as target:
for raw in source:
before.update(raw)
row = decode(raw)
if any(row[field] != manifest[field] for field in ('source', 'venue', 'year', 'track')):
raise ValueError('Record differs from source manifest')
key = (row['source'], row['external_id'])
if key in seen:
raise ValueError('Duplicate source identity in input batch')
seen.add(key)
if key in assertions:
assertion, references = assertions[key]
check_hash(raw, assertion['record_sha256'])
existing = row.get('identifiers', {})
if existing is None:
existing = {}
if not isinstance(existing, dict):
raise ValueError('Invalid existing identifiers')
for scheme, value in assertion['identifiers'].items():
if scheme in existing and existing[scheme] != value:
raise ValueError('Existing identifier conflict')
details = row.setdefault('metadata', {})
if not isinstance(details, dict):
raise ValueError('Invalid record metadata')
history = details.setdefault('identifier_curation', [])
if not isinstance(history, list):
raise ValueError('Invalid curation history')
history.append({'plan_sha256': digest(plan_raw), 'input_file_sha256': entry['sha256'],
'record_sha256': digest(raw), 'previous_identifiers': dict(existing),
'identifiers_asserted': assertion['identifiers'], 'evidence': references,
'verification_scope': SCOPE})
report['identifiers_added'] += len(assertion['identifiers'].keys() - existing.keys())
row['identifiers'] = {**existing, **assertion['identifiers']}
raw = json_bytes(row)
applied.add(key)
target.write(raw)
after.update(raw)
count += 1
if before.hexdigest() != entry['sha256'] or not re.fullmatch('[0-9a-f]{64}', entry['sha256']):
raise ValueError('Input SHA256 mismatch')
if not count or type(manifest['count']) is not int or count != manifest['count']:
raise ValueError('Manifest count mismatch')
provenance = {'plan_sha256': digest(plan_raw), 'input_file_sha256': before.hexdigest(),
'input_manifest_sha256': entry['manifest_sha256'], 'verification_scope': SCOPE}
prepared = {**manifest, 'file': str(output / path.name), 'file_sha256': after.hexdigest(),
'identifier_curation': provenance}
(staging / path.with_suffix('.manifest.json').name).write_bytes(json_bytes(prepared))
report['files'].append({'file': str(output / path.name), 'count': count,
'input_sha256': before.hexdigest(), 'sha256': after.hexdigest()})
if applied != assertions.keys():
raise ValueError('Assertion target absent from input batch')
report['assertions_applied'] = len(applied)
(staging / 'curation-report.json').write_bytes(json_bytes(report))
# All files are on the same filesystem. No database or input mutation.
if output.exists():
raise ValueError('Output directory appeared during preparation')
staging.rename(output)
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--plan', required=True, type=Path)
parser.add_argument('--output', required=True, type=Path)
args = parser.parse_args(argv)
print(json.dumps(prepare(args.plan, args.output), ensure_ascii=False))
+7 -13
View File
@@ -95,19 +95,13 @@ class Store:
matches.add(old[0])
key = normalized(title)
title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall()
# An existing identifier/source record anchors the entity. Title
# similarity must not pull in another deposit or an extended version.
if not matches:
title_only_matches = set()
authors_set = author_keys(authors)
for paper,old_authors,old_abstract in title_matches:
overlap = authors_set & author_keys(json.loads(old_authors))
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
title_only_matches.add(paper)
if len(title_only_matches)>1:
raise ValueError('Ambiguous paper identity: multiple author-corroborated title matches; provide verified identifiers')
matches = title_only_matches
authors_set = author_keys(authors)
for paper,old_authors,old_abstract in title_matches:
overlap = authors_set & author_keys(json.loads(old_authors))
# Cross-platform title matching needs independent author corroboration.
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
matches.add(paper)
if matches:
paper = min(matches)
for other in sorted(matches-{paper}):
+64 -1
View File
@@ -10,7 +10,7 @@ from unittest.mock import patch
from paper_atlas.collectors.dblp import XML_SHA
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
from paper_atlas.collectors.dblp_scope import scope
from paper_atlas.collectors.dblp_scope import scope, HISTORICAL_WORKSHOP_TITLES
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
@@ -70,6 +70,69 @@ class DblpRecordTests(unittest.TestCase):
self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict')
self.assertEqual(scope(self.row(year=2027),self.volume('aaai',2027))[1],'future_edition')
def test_historical_primary_series_workshop_names_require_exact_editions(self):
self.assertEqual(set(HISTORICAL_WORKSHOP_TITLES),
{('aistats',year) for year in [1995,1997,1999,2001,2003,2005]} |
{('colt',year) for year in [1988,1989,1990,1991]})
for (family,year),prefix in HISTORICAL_WORKSHOP_TITLES.items():
with self.subTest(family=family,year=year):
title=prefix+f', {family.upper()} {year}, Proceedings.'
volume=self.volume(family,year,title)
row=self.row(family,family.upper(),year)
if year==1995:
row['fields']['year']=['1996'];volume['fields']['year']=['1996']
edition,reason=scope(row,volume)
self.assertIsNone(reason);self.assertEqual(edition['track'],'main')
self.assertEqual(edition['year'],year)
self.assertIn('Historical primary series',edition['track_scope_note'])
record,reason=normalize_row(row,{volume['key']:volume},{family.upper()})
self.assertIsNone(reason);self.assertEqual(record['metadata']['track_scope_evidence_url'],edition['track_scope_evidence_url'])
self.assertEqual(record['metadata']['proceedings_title'],title)
if year==1995:self.assertEqual(record['metadata']['bibliographic_year'],1996)
def test_historical_exception_does_not_promote_affiliated_or_wrong_scope_workshops(self):
title=HISTORICAL_WORKSHOP_TITLES[('aistats',1995)]+', AISTATS 1995, Proceedings.'
for book,key,changed in [
('AISTATS','conf/aistats/1995w',title),
('AISTATS (Workshops)','conf/aistats/1995',title),
('AISTATS','conf/aistats/1995',title.replace('Fifth','Sixth')),
('AISTATS','conf/aistats/1995',title.replace('1995','1997'))]:
volume=self.volume('aistats',1995,changed);volume['key']=key
row=self.row('aistats',book,1995);row['fields']['crossref']=[key]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
volume=self.volume('aistats',2010,'Active Learning and Experimental Design workshop, In conjunction with AISTATS 2010')
volume['key']='conf/aistats/2010w'
row=self.row('aistats','AISTATS',2011);row['fields']['crossref']=[volume['key']]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
def test_combined_colt_kernel_volume_keeps_unknown_per_paper_track(self):
title='Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003, Washington, DC, USA, August 24-27, 2003, Proceedings'
volume=self.volume('colt',2003,title);row=self.row('colt','COLT',2003)
record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
self.assertIsNone(reason);self.assertEqual(record['track'],'proceedings_unspecified')
self.assertEqual(record['status'],'accepted')
self.assertIn('per-paper section absent',record['metadata']['track_scope_note'])
self.assertEqual(record['metadata']['track_scope_evidence_url'],'https://doi.org/10.1007/b12006')
volume['key']='conf/colt/2003w';row['fields']['crossref']=[volume['key']]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
def test_historical_scope_provenance_survives_file_ingestion(self):
title=HISTORICAL_WORKSHOP_TITLES[('colt',1988)]+', COLT 1988, Proceedings.'
row=self.row('colt','COLT',1988);volume=self.volume('colt',1988,title)
record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
self.assertIsNone(reason)
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);path=root/'history.jsonl'
path.write_text(json.dumps(record)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()}))
ingest_file(root/'sample.sqlite',path)
with closing(sqlite3.connect(root/'sample.sqlite')) as connection:
track=connection.execute('SELECT track FROM editions').fetchone()[0]
metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0])
self.assertEqual(track,'main')
self.assertEqual(metadata['track_scope_evidence_url'],'https://www.learningtheory.org/')
self.assertEqual(metadata['proceedings_title'],title)
def test_only_own_identifier_links_are_used_and_ambiguity_is_preserved(self):
own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/published','https://doi.org/10.48550/arXiv.2401.00001','https://openreview.net/forum?id=realForum'],
'cite':['https://doi.org/10.9999/reference']})
-185
View File
@@ -1,185 +0,0 @@
import copy
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.curation.identifiers import SCOPE, digest, json_bytes, prepare
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
class IdentifierCurationTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.output = self.root / 'prepared'
self.row = {'source':'dblp','external_id':'conf/uai/example','url':'https://example.org/paper',
'title':'A sufficiently long example research paper title','abstract':None,
'authors':['Jane Doe','John Smith'],'venue':'UAI','year':2001,'track':'main',
'status':'accepted','tags':[{'namespace':'dblp:topic','value':'version\u2029study\x00'}],
'metadata':{'pages':'203-210','raw_title':'Original\u2029title'}}
self.evidence = self.root / 'evidence.json'
self.evidence.write_bytes(json_bytes({'url':'https://example.org/archive','comment':'Conference version'}))
self.path = self.root / 'records.jsonl'
self.raw = json_bytes(self.row)
self.path.write_bytes(self.raw)
self.manifest = {'source':'dblp','venue':'UAI','year':2001,'track':'main','count':1,
'list_url':'https://example.org/list','file_sha256':digest(self.raw),
'status':'partial','missing_abstract_count':1,'parser_version':2}
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
self.plan = {'format_version':1,'inputs':[self.input(self.path)],'assertions':[
{'source':'dblp','external_id':self.row['external_id'],'record_sha256':digest(self.raw),
'identifiers':{'arxiv':'1408.1485'},'evidence':[{
'file':str(self.evidence),'sha256':digest(self.evidence.read_bytes()),
'url':'https://example.org/archive','checked_at':'2026-11-08T08:00:00Z',
'note':'Reviewed title, authors, publication and version evidence'}]}]}
def input(self, path):
return {'file':str(path),'sha256':digest(path.read_bytes()),
'manifest_sha256':digest(path.with_suffix('.manifest.json').read_bytes())}
def run_plan(self, plan=None):
path = self.root / 'plan.json'
path.write_bytes(json_bytes(self.plan if plan is None else plan))
return prepare(path, self.output)
def test_only_identifiers_and_provenance_change_and_raw_inputs_survive(self):
result = self.run_plan()
self.assertEqual(self.path.read_bytes(), self.raw)
prepared = json.loads((self.output / self.path.name).read_bytes())
provenance = prepared['metadata'].pop('identifier_curation')
self.assertEqual(prepared.pop('identifiers'), {'arxiv':'1408.1485'})
self.assertEqual(prepared, self.row)
self.assertEqual(provenance[0]['record_sha256'], digest(self.raw))
self.assertEqual((self.output / provenance[0]['evidence'][0]['file']).read_bytes(), self.evidence.read_bytes())
manifest = json.loads((self.output / 'records.manifest.json').read_bytes())
for key, value in self.manifest.items():
if key != 'file_sha256': self.assertEqual(manifest[key], value)
self.assertEqual(manifest['file_sha256'], digest((self.output / self.path.name).read_bytes()))
self.assertEqual(result['assertions_applied'], 1)
self.assertEqual(result['identifiers_added'], 1)
self.assertEqual(result['verification_scope'], SCOPE)
self.assertFalse(result['semantic_identity_verified'])
self.assertFalse(result['database_written'])
self.assertEqual(list(self.output.glob('*.jsonl')), [self.output / self.path.name])
with self.assertRaisesRegex(ValueError, 'already exist'): self.run_plan()
def test_late_invalid_file_publishes_nothing(self):
second = self.root / 'second.jsonl'
second.write_bytes(json_bytes({**self.row,'external_id':'second'}))
manifest = {**self.manifest,'file_sha256':digest(second.read_bytes()),'count':2}
second.with_suffix('.manifest.json').write_bytes(json_bytes(manifest))
self.plan['inputs'].append(self.input(second))
with self.assertRaisesRegex(ValueError, 'count mismatch'): self.run_plan()
self.assertFalse(self.output.exists())
self.assertFalse(list(self.root.glob('.identifier-curation-*')))
self.assertEqual(self.path.read_bytes(), self.raw)
def test_unasserted_rows_keep_exact_physical_bytes_and_doi_is_additive(self):
untouched = (json.dumps({**self.row,'external_id':'unselected'},ensure_ascii=False,indent=None) + ' ').encode()
self.path.write_bytes(self.raw + untouched)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,
'file_sha256':digest(self.path.read_bytes()),'count':2,'missing_abstract_count':2}))
self.plan['inputs'] = [self.input(self.path)]
self.plan['assertions'][0]['identifiers']['doi'] = '10.1234/example'
result = self.run_plan()
with (self.output / self.path.name).open('rb') as stream:
first = json.loads(next(stream))
self.assertEqual(next(stream), untouched)
self.assertEqual(first['identifiers'], {'arxiv':'1408.1485','doi':'10.1234/example'})
self.assertEqual(result['identifiers_added'], 2)
def test_missing_duplicate_and_mismatched_assertions_fail_closed(self):
for mode in ('missing','duplicate','line_hash','duplicate_input'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
assertion = plan['assertions'][0]
if mode == 'missing': assertion['external_id'] = 'absent'
elif mode == 'duplicate': plan['assertions'].append(copy.deepcopy(assertion))
elif mode == 'line_hash': assertion['record_sha256'] = '0' * 64
else:
other = self.root / 'duplicate.jsonl'
other.write_bytes(self.raw)
other.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
plan['inputs'].append(self.input(other))
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
def test_identifier_conflict_does_not_overwrite_and_same_value_is_auditable(self):
self.row['identifiers'] = {'arxiv':'cs/0307069'}
self.path.write_bytes(json_bytes(self.row))
self.manifest['file_sha256'] = digest(self.path.read_bytes())
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
self.plan['inputs'] = [self.input(self.path)]
self.plan['assertions'][0]['record_sha256'] = digest(self.path.read_bytes())
with self.assertRaisesRegex(ValueError, 'identifier conflict'): self.run_plan()
self.assertFalse(self.output.exists())
self.plan['assertions'][0]['identifiers']['arxiv'] = 'cs/0307069'
self.assertEqual(self.run_plan()['identifiers_added'], 0)
def test_unsupported_changes_identifiers_and_unbound_evidence_are_rejected(self):
for mode in ('title','bad_month','version','unknown_scheme','doi_case','evidence_hash','credentials','timestamp'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
assertion = plan['assertions'][0]
evidence = assertion['evidence'][0]
if mode == 'title': assertion['title'] = 'Rewrite forbidden'
elif mode == 'bad_month': assertion['identifiers'] = {'arxiv':'1413.12345'}
elif mode == 'version': assertion['identifiers'] = {'arxiv':'1408.1485v2'}
elif mode == 'unknown_scheme': assertion['identifiers'] = {'url':'https://example.org'}
elif mode == 'doi_case': assertion['identifiers'] = {'doi':'10.1234/UPPER'}
elif mode == 'evidence_hash': evidence['sha256'] = '0' * 64
elif mode == 'credentials': evidence['url'] = 'https://user:secret@example.org'
else: evidence['checked_at'] = '2026-11-08T08:00:00'
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
def test_input_and_manifest_hashes_and_duplicate_json_keys_are_checked(self):
for mode in ('input','manifest','duplicate_json','invalid_evidence_json'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
if mode == 'input':
plan['inputs'][0]['sha256'] = '0' * 64
elif mode == 'manifest':
plan['inputs'][0]['manifest_sha256'] = '0' * 64
elif mode == 'duplicate_json':
raw = self.raw[:-2] + b',"source":"dblp"}\n'
self.path.write_bytes(raw)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,'file_sha256':digest(raw)}))
plan['inputs'] = [self.input(self.path)]
else:
self.evidence.write_bytes(b'not json')
plan['assertions'][0]['evidence'][0]['sha256'] = digest(self.evidence.read_bytes())
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
self.path.write_bytes(self.raw)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
def test_curated_anchor_import_does_not_merge_extended_version_or_its_history(self):
db = self.root / 'papers.sqlite'
store = Store(db)
try:
original, _ = store.record('arxiv','1408.1485','https://example.org/one',self.row['title'],None,self.row['authors'],{'arxiv':'1408.1485'})
extended, record = store.record('arxiv','cs/0307069','https://example.org/two',self.row['title'],None,self.row['authors'],{'arxiv':'cs/0307069'})
store.decision(extended,record,'ICLR',2025,'main','another-attempt','https://example.org/reject','rejected','Reject')
store.conn.commit()
finally:
store.conn.close()
self.run_plan()
ingest_file(db,self.output / self.path.name)
store = Store(db)
try:
c = store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
self.assertEqual(c.execute('SELECT paper_id FROM source_records WHERE external_id=?',(self.row['external_id'],)).fetchone()[0], original)
self.assertEqual(c.execute('SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE v.acronym=?',('ICLR',)).fetchone()[0], extended)
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0], 0)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
finally:
store.conn.close()
if __name__ == '__main__':
unittest.main()
-44
View File
@@ -53,50 +53,6 @@ class StoreTests(unittest.TestCase):
self.assertNotEqual(other,paper)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
def test_identifier_anchor_does_not_absorb_a_same_title_arxiv_version(self):
conference,first=self.record('arxiv','1408.1485')
extended,second=self.record('arxiv','cs/0307069')
self.store.decision(extended,second,'ICLR',2025,'main','separate','https://example.org/reject','rejected','Reject')
resolved,_=self.record('dblp','conf/uai/example',{'arxiv':'1408.1485'})
self.assertEqual(resolved,conference);self.assertNotEqual(conference,extended)
c=self.store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],extended)
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
self.assertEqual(c.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
# A refresh of the anchored original source is equally unambiguous.
self.assertEqual(self.record('arxiv','1408.1485')[0],conference)
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
def test_ambiguous_title_aborts_whole_file_without_merging_existing_entities(self):
self.record('arxiv','1408.1485');self.record('arxiv','cs/0307069')
self.store.conn.commit()
rows=[{'source':'dblp','external_id':'conf/uai/new','url':'https://example.org/new',
'title':'An unrelated earlier record in this file','abstract':None,'authors':[],
'venue':'UAI','year':2001,'track':'main','status':'accepted'},
{'source':'dblp','external_id':'conf/uai/ambiguous','url':'https://example.org/ambiguous',
'title':'A sufficiently long example research paper title','abstract':None,
'authors':['Jane Doe','John Smith'],'identifiers':{'doi':'10.0000/not-yet-known'},
'venue':'UAI','year':2001,'track':'main','status':'accepted'}]
path=Path(self.tmp.name)/'ambiguous.jsonl'
path.write_text(''.join(json.dumps(row)+'\n' for row in rows))
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','list_url':'https://example.org/list','count':2}))
with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):ingest_file(self.db,path)
c=self.store.conn
for table,count in [('papers',2),('source_records',2),('submissions',0),('decisions',0),('paper_merges',0)]:
self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],count,table)
self.assertEqual(c.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
def test_explicit_bridge_merges_only_its_identifier_matches(self):
first,_=self.record('arxiv','2601.00001',authors=['First Author'])
second,_=self.record('arxiv','2601.00002',{'doi':'10.0000/bridge'},authors=['Second Author'])
unrelated,_=self.record('arxiv','2601.00003')
resolved,_=self.record('official','bridge',{'arxiv':'2601.00001','doi':'10.0000/bridge'})
self.assertEqual(resolved,min(first,second))
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2)
self.assertIsNotNone(self.store.conn.execute('SELECT 1 FROM papers WHERE id=?',(unrelated,)).fetchone())
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
def test_manifest_count_mismatch_rolls_back_records(self):
path=Path(self.tmp.name)/'source.jsonl'
path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n')
-28
View File
@@ -110,34 +110,6 @@ class PostgreSQLIngestionTests(unittest.TestCase):
finally:
reader.close()
def test_identity_anchor_keeps_other_arxiv_version_and_ambiguous_file_rolls_back(self):
title='An identical long research title shared by conference and extended versions'
authors=['Jane Doe','John Smith']
store=self.store()
try:
first,_=store.record('arxiv','1408.1485','https://arxiv.org/abs/1408.1485',title,'Conference abstract',authors)
second,_=store.record('arxiv','cs/0307069','https://arxiv.org/abs/cs/0307069',title,'Extended abstract',authors)
resolved,_=store.record('dblp','known','https://example.org/known',title,None,authors,{'arxiv':'1408.1485'})
self.assertEqual(resolved,first);self.assertNotEqual(first,second)
store.conn.commit()
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) # migration fixture only
self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
finally:store.conn.close()
rows=[self.row('official','earlier',status='accepted',venue='UAI',year=2001),
dict(self.row('official','ambiguous',status='accepted',venue='UAI',year=2001),
title=title,abstract=None,authors=authors,identifiers={})]
with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):self.ingest(self.file('ambiguous-identity',rows))
store=self.store()
try:
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],4)
self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
self.assertEqual(store.conn.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
self.assertEqual(store.conn.execute("SELECT count(*) FROM source_records WHERE external_id IN (?,?)",('earlier','ambiguous')).fetchone()[0],0)
finally:store.conn.close()
def test_file_import_is_idempotent_and_refreshes_only_own_tags(self):
row = self.row(); row['tags'] = [{'namespace': 'openreview_snapshot:area', 'value': 'theory'}]
path = self.file('reject', [row]); self.ingest(path)