Compare commits

...

6 Commits

Author SHA1 Message Date
codex-bot 9093a8ce25 Preserve separate ICAPS registry publication versions and abstracts
Metadata integrity checks / metadata (push) Successful in 2m51s
2026-11-17 00:30:14 +08:00
codex-bot e19cb83cd6 Align ICAPS collector entry point with shared CLI
Metadata integrity checks / metadata (push) Successful in 2m47s
2026-11-17 00:13:30 +08:00
codex-bot 4d2df2a826 Preserve ICAPS 2026 official acceptance and presentation boundaries
Metadata integrity checks / metadata (push) Failing after 2m30s
2026-11-17 00:08:49 +08:00
Zayden-Jung 1cfd44d8b6 Merge pull request '为RSS发表记录保留跨来源的同次投稿身份' (#53) from codex/rss-publication-identity into main
Metadata integrity checks / metadata (push) Successful in 2m48s
Reviewed-on: #53
2026-11-16 08:13:30 +00:00
Zayden-Jung 5de7f6ff22 Merge pull request '封装RSS2026官方论文集采集与来源差异' (#52) from codex-bot/rss2026 into main
Metadata integrity checks / metadata (push) Successful in 3m1s
Reviewed-on: #52
2026-11-16 08:13:16 +00:00
codex-bot b4e249e675 Recognize accepted RSS publisher DOI as an edition-scoped publication identity
Metadata integrity checks / metadata (push) Successful in 2m55s
2026-11-15 23:07:11 +08:00
13 changed files with 607 additions and 1 deletions
+18
View File
@@ -49,6 +49,24 @@ jobs:
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
icaps_snapshots = root/'icaps2026-snapshots'
icaps_snapshots.mkdir()
for name, html in icaps_fixture['pages'].items():
(icaps_snapshots/name).write_text(html)
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
registry_snapshots = root/'icaps2026-registry-snapshots'
registry_snapshots.mkdir()
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
rss_snapshots = root/'rss2026-snapshots'
rss_snapshots.mkdir()
+4
View File
@@ -120,3 +120,7 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
+6
View File
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
### RSS 发表身份
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
+40
View File
@@ -0,0 +1,40 @@
# ICAPS 2026 官方名单
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
```sh
paper-atlas collect icaps2026-program \
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
--out /data/paper-index/collected/icaps2026-normalized-v1
```
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
## 独立注册元数据版本
```sh
paper-atlas collect icaps2026-registry \
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
```
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
+2
View File
@@ -23,6 +23,8 @@ COMMANDS = {
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
+155
View File
@@ -0,0 +1,155 @@
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
from pathlib import Path
from paper_atlas.io.html import Document, Node, clean
SOURCE = 'icaps_official_program'
ROOT = 'https://icaps26.icaps-conference.org/'
PAGES = {
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
}
MAIN = 'Papers accepted to the Main Tracks'
PREVIOUS = 'Previously Published Papers'
TRACK = 'main_tracks_unclassified'
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def authors_from_literal(literal):
# This frozen listing uses commas between authors and a final "and".
# Preserve the complete literal too; never infer person-name aliases.
if literal.count(' and ') != 1 or ', and ' in literal:
raise ValueError('Unreviewed author-list punctuation')
first, last = literal.rsplit(' and ', 1)
authors = first.split(', ') + [last]
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
raise ValueError('Incomplete or ambiguous author list')
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
raise ValueError('Author literal cannot be reconstructed')
return authors
def parse_listing(raw):
nodes = list(Document(raw).root.walk())
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
if headings != ['List of Papers Accepted to ICAPS 2026']:
raise ValueError('Unexpected official conference edition')
section = None; sections = []; rows = []; seen = set()
for n in nodes:
if n.tag == 'h2':
section = clean(n.text()); sections.append(section)
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
continue
if section not in (MAIN, PREVIOUS):
raise ValueError('Paper belongs to an unreviewed section')
# Require the reviewed title and complete-author blocks; fail on layout drift.
blocks = [b for b in n.children if isinstance(b, Node)]
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
raise ValueError('Changed paper metadata layout')
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
raise ValueError('Changed paper title/link layout')
title = clean(titles[0].text()); literal = clean(blocks[1].text())
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
raise ValueError('Empty or repeated paper title')
seen.add(title.casefold())
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
'authors_literal': literal})
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
raise ValueError('Incomplete reviewed official listing')
return rows
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
root = Path(snapshots)
manifest_path = root/'manifest.json'
if manifest_path.is_symlink():
raise ValueError('Snapshot symlinks rejected')
receipt = json.loads(manifest_path.read_bytes())
observed = receipt['at']
if (receipt.get('status') != 'collected_not_parsed_not_imported'
or receipt.get('database_written') is not False
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
raise ValueError('Invalid reviewed snapshot receipt')
receipts = receipt['sources']
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
raise ValueError('Unexpected snapshot source set')
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
for name, (url, expected) in PAGES.items():
matches = [r for r in receipts if r['file'] == name]
if len(matches) != 1:
raise ValueError('Repeated snapshot receipt')
r = matches[0]; p = root/name
if p.is_symlink() or p.stat().st_size > 2*1024**2:
raise ValueError('Unsafe or oversized snapshot')
raw = p.read_bytes()
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
raise ValueError('Reviewed snapshot hash changed')
if r['url'] != url or r['final_url'] != url:
raise ValueError('Unexpected snapshot URL or redirect')
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
bodies[name] = raw.decode('utf-8')
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
for entry in entries:
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
if entry['section'] == PREVIOUS:
previous.append({**entry, 'entry_key': entry_key,
'presentation_kind': 'previously_published_research',
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
continue
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
'metadata': {'official_listing': entry, 'source_evidence': evidence,
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
'abstract_availability': 'not_on_official_listing',
'original_decision_signature_verified': False,
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
'status': 'partial', 'missing_abstract_count': 90}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
sidecar = out/'previously-published-presentations.json'
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
@@ -0,0 +1,131 @@
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
import argparse
from datetime import datetime, date
import hashlib
from html import unescape
import json
from pathlib import Path
import re
from paper_atlas.io.html import clean
SOURCE = 'crossref_icaps_registry'
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
TRACK = 'main_tracks_unclassified'
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def abstract_text(raw):
if not isinstance(raw, str):
raise ValueError('Missing registry abstract')
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
if not match or re.search(r'<[^>]*>', match[1]):
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
text = clean(unescape(match[1]))
if not text:
raise ValueError('Empty registry abstract')
return text
def registry_papers(snapshot):
observed = snapshot['at']
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
or snapshot.get('database_written') is not False
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
raise ValueError('Unreviewed registry snapshot receipt')
papers = []; excluded = []; seen = set()
for row in snapshot['records']:
doi = row.get('DOI')
if not isinstance(doi, str) or doi in seen:
raise ValueError('Missing or repeated registry DOI')
seen.add(doi)
if row.get('volume') == '35':
excluded.append(doi)
continue
if (row.get('volume') != '36' or row.get('issue') != '1'
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
or row.get('URL') != 'https://doi.org/'+doi
or not isinstance(row.get('title'), list) or len(row['title']) != 1
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
raise ValueError('Unreviewed publication volume, DOI, type or title')
day, = row['published']['date-parts']
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
raise ValueError('Unreviewed publication date')
publication_date = date(*day).isoformat()
authors = row.get('authors')
if (not isinstance(authors, list) or not authors
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
or not a.get('given') or not a.get('family') for a in authors)):
raise ValueError('Incomplete registry authors')
names = [clean(a['given']+' '+a['family']) for a in authors]
if (len(set(n.casefold() for n in names)) != len(names)
or any('@' in n or 'anonymous' in n.casefold() for n in names)
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
raise ValueError('Unreviewed registry author order or identity')
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
if len(papers) != 90 or len(excluded) != 1:
raise ValueError('Incomplete reviewed volume census')
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
path = Path(snapshots)/'crossref-filtered.json'
if path.is_symlink() or path.stat().st_size > 4*1024**2:
raise ValueError('Unsafe or oversized registry snapshot')
raw = path.read_bytes()
if digest(raw) != SNAPSHOT_SHA:
raise ValueError('Reviewed registry snapshot hash changed')
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
for paper in papers:
original = paper['original']; doi = original['DOI']
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
'metadata': {'registry_fields': original, 'source_evidence': evidence,
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
+1 -1
View File
@@ -4,7 +4,7 @@ import re
from paper_atlas.io.publication_ids import mlsys_id
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
def timestamp(value=None):
Binary file not shown.
Binary file not shown.
+95
View File
@@ -0,0 +1,95 @@
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import icaps2026 as c
class ICAPS2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
def snapshots(self, root):
root.mkdir()
for name, html in self.bundle['pages'].items():
(root/name).write_text(html)
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
return root
def test_full_listing_matches_independently_recorded_literals(self):
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
def test_incomplete_and_duplicate_listing_rejected(self):
html = self.bundle['pages']['accepted.html']
with self.assertRaises(ValueError):
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
def test_changed_edition_or_section_rejected(self):
html = self.bundle['pages']['accepted.html']
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
html.replace(c.MAIN, 'Workshop Papers')]:
with self.assertRaises(ValueError):
c.parse_listing(changed)
def test_ambiguous_author_lists_not_guessed(self):
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
with self.assertRaises(ValueError):
c.authors_from_literal(literal)
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
def test_collection_separates_presentations_and_unresolved_tracks(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
report = c.collect(out, snapshots=src)
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
self.assertEqual(report['public_rejected_availability'], 'unknown')
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
previous = json.loads((out/report['previously_published_file']).read_text())
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
self.assertEqual(len({r['external_id'] for r in rows}), 90)
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
self.assertEqual(report['files'][0]['status'], 'partial')
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(out, snapshots=src)
def test_tampering_retains_failure_without_success_receipt(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src')
(src/'accepted.html').write_text('tampered')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'out', snapshots=src)
self.assertTrue((root/'out/failed.json').exists())
self.assertFalse((root/'out/collection-report.json').exists())
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
for kind in ('redirect', 'time', 'duplicate'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
p.write_text(json.dumps(r))
with self.assertRaises(ValueError):
c.collect(root/'out', snapshots=src)
self.assertFalse((root/'out/collection-report.json').exists())
def test_snapshot_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
with self.assertRaisesRegex(ValueError, 'Unsafe'):
c.collect(root/'out', snapshots=src)
+96
View File
@@ -0,0 +1,96 @@
import copy
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import icaps_registry2026 as c
class ICAPSRegistry2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
cls.snapshot = json.loads(cls.raw)
def test_exact_edition_not_publication_year_alone(self):
papers, excluded = c.registry_papers(self.snapshot)
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
report = c.collect(root/'out', snapshots=src)
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
self.assertEqual(report['public_rejected_availability'], 'unknown')
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
self.assertEqual({r['external_id'] for r in rows}, set(original))
for r in rows:
doi = r['external_id']; o = original[doi]
self.assertEqual(r['metadata']['registry_fields'], o)
self.assertEqual(r['title'], o['title'][0])
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
self.assertEqual(r['identifiers'], {'doi': doi})
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
self.assertEqual(r['track'], 'main_tracks_unclassified')
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
self.assertIn('Stephen F. Smith', changed['authors'])
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(root/'out', snapshots=src)
def test_math_markup_is_not_flattened_as_plain_numbers(self):
self.assertEqual(c.abstract_text('<jats:p>A &lt; B and C &amp; D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
with self.assertRaises(ValueError):
c.abstract_text(raw)
def test_invalid_or_repeated_doi_and_edition_rejected(self):
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
with self.subTest(kind=kind):
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
if kind == 'doi': r['DOI'] = '10.9999/foreign'
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
if kind == 'volume': r['volume'] = '37'
if kind == 'container': r['container-title'] = ['Other Conference']
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_missing_authors_and_changed_order_evidence_rejected(self):
for kind in ('missing', 'sequence', 'profile'):
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
if kind == 'missing': r['authors'][0].pop('family')
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_receipt_scope_and_timezone_rejected(self):
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
s = copy.deepcopy(self.snapshot); s[key] = value
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_tamper_failure_retained_and_success_not_published(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'out', snapshots=src)
self.assertTrue((root/'out/failed.json').exists())
self.assertFalse((root/'out/collection-report.json').exists())
def test_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
with self.assertRaisesRegex(ValueError, 'Unsafe'):
c.collect(root/'out', snapshots=src)
+59
View File
@@ -0,0 +1,59 @@
"""RSS publisher/DBLP observations share one event without losing old decisions."""
from pathlib import Path
import json
import sqlite3
import tempfile
import unittest
from paper_atlas.db.submissions import identities_for_row
from tests.test_aamas_publication_identity import row, ingest
DOI='10.15607/rss.2026.xxii.041'
def rss(source,external,**kwargs):
return row(source,external,venue='RSS',year=2026,track='main',
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
class RssPublicationIdentityTests(unittest.TestCase):
def test_only_verified_accepted_source_adds_publication_identity(self):
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
self.assertEqual(identities_for_row({**r,'status':status}),[])
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
for pair in ((a,b),(b,a)):
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
for r in pair:ingest(db,root,r)
ingest(db,root,a)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
ingest(db,root,old)
with sqlite3.connect(db) as c:
before=c.execute('SELECT * FROM decisions').fetchall()
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
ingest(db,root,new)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])