Compare commits

..

6 Commits

Author SHA1 Message Date
codex-bot 28255743ef Collect SDM 2026 registry records and separate program observations
Metadata integrity checks / metadata (push) Successful in 3m3s
2026-11-17 15:23:54 +08:00
Zayden-Jung 59053fd4fb Merge pull request '封装CoRL2026官方名单及分字段档案来源' (#54) from codex/corl2026 into main
Metadata integrity checks / metadata (push) Successful in 2m54s
Reviewed-on: #54
2026-11-16 16:50:20 +00:00
codex-bot b50136d5bc Merge main into CoRL branch and preserve both conference guides
Metadata integrity checks / metadata (push) Successful in 2m52s
2026-11-17 00:46:21 +08:00
Zayden-Jung 87a0689fcc Merge pull request '封装ICAPS2026官方名单与独立注册元数据' (#55) from codex/icaps2026 into main
Metadata integrity checks / metadata (push) Successful in 2m49s
Reviewed-on: #55
2026-11-16 16:41:57 +00:00
codex-bot 19d03ced54 Verify installed CoRL collector against all 687 source records
Metadata integrity checks / metadata (push) Successful in 2m52s
2026-11-16 22:59:19 +08:00
codex-bot c33f5422de Collect CoRL 2026 program with versioned public metadata provenance
Metadata integrity checks / metadata (push) Successful in 2m51s
2026-11-16 22:54:52 +08:00
11 changed files with 819 additions and 0 deletions
+30
View File
@@ -49,6 +49,23 @@ jobs:
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
sdm_snapshots = root/'sdm2026-snapshots'
for name, text in sdm_fixture['files'].items():
target = sdm_snapshots/name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(text.encode('utf-8'))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
assert sdm_report['program_registry_author_sequence_differences'] == 20
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes())) icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
icaps_snapshots = root/'icaps2026-snapshots' icaps_snapshots = root/'icaps2026-snapshots'
icaps_snapshots.mkdir() icaps_snapshots.mkdir()
@@ -75,6 +92,19 @@ jobs:
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts'])) (rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210 assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
corl_official = root/'corl2026-official'
corl_archive = root/'corl2026-archive'
corl_official.mkdir()
corl_archive.mkdir()
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
+6
View File
@@ -124,3 +124,9 @@ RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 de
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。 ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。 ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
### CoRL 2026 官方程序快照
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
+24
View File
@@ -0,0 +1,24 @@
# CoRL 2026:官方程序与公开档案
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
```sh
paper-atlas collect corl2026-program \
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
--out /data/paper-index/collected/corl2026-normalized-v1
```
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
+44
View File
@@ -0,0 +1,44 @@
# SDM 2026:论文集与日程版本
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
```sh
paper-atlas collect sdm2026-proceedings \
--snapshots /data/paper-index/collected \
--out /data/paper-index/collected/sdm2026-normalized-pass01
```
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
## 两种输出
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
## 公开拒稿与访问限制
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
## 来源恢复与后续核查
| 冻结目录 | manifest SHA-256 |
| --- | --- |
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
+2
View File
@@ -24,11 +24,13 @@ COMMANDS = {
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'), ('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
+183
View File
@@ -0,0 +1,183 @@
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import unicodedata
from urllib.parse import parse_qs, urlsplit
from paper_atlas.io.html import Document, clean
SOURCE = 'corl_program_archive'
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def comparable(text):
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
def forum_id(url):
if not isinstance(url, str):
raise ValueError('Forum URL must be text')
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
raise ValueError('Invalid original forum identity')
return q['id'][0]
def parse_program(raw):
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
if len(embeds) != 1:
raise ValueError('Expected one official embedded program')
doc = Document(embeds[0]); nodes = list(doc.root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
raise ValueError('Unexpected program edition')
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
raise ValueError('Reviewed program census changed')
papers = {}; sessions = {}; occurrences = []
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
sid = section.attrs.get('data-sess', section.attrs.get('id'))
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
raise ValueError('Unexpected or repeated presentation session')
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
if len(headings) != 1:
raise ValueError('Ambiguous session heading')
sessions[sid] = {'kind': kind, 'heading': headings[0]}
seen = set()
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
fid = forum_id(link.attrs['href']); title = clean(link.text())
if fid in seen or not title:
raise ValueError('Repeated or empty paper in session')
seen.add(fid)
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
if paper['title'] != title:
raise ValueError('Conflicting official titles share a forum')
event = {'session_id': sid, **sessions[sid]}
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
counts = Counter(kind for _, _, kind in occurrences)
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
raise ValueError('Incomplete official program')
for paper in papers.values():
kinds = Counter(p['kind'] for p in paper['presentations'])
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
raise ValueError('Invalid poster/talk multiplicity')
return papers
def join_metadata(program, archive):
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
or not isinstance(archive.get('papers'), list)):
raise ValueError('Unpinned public metadata provenance')
indexed = {}
for row in archive['papers']:
if not isinstance(row, dict) or set(row) - FIELDS:
raise ValueError('Unexpected metadata/profile fields')
fid = forum_id(row.get('openreview'))
if fid != row.get('id') or fid in indexed:
raise ValueError('Repeated or mismatched archive forum')
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
raise ValueError('Field source belongs to another submission')
if (not isinstance(row.get('title'), str) or not row['title'].strip()
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
or not isinstance(row.get('authors'), list) or not row['authors']
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
raise ValueError('Incomplete scholarly metadata')
indexed[fid] = row
if set(indexed) != set(program):
raise ValueError('Official and archive forum sets differ')
return indexed
def collect(out, *, official_snapshots, archive_snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
if official.is_symlink() or filtered.is_symlink():
raise ValueError('Snapshot symlinks rejected')
raw = official.read_bytes(); data = filtered.read_bytes()
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
raise ValueError('Reviewed snapshot hash changed')
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
matches = [r for r in receipts if r.get('url') == OFFICIAL]
if len(matches) != 1:
raise ValueError('Missing official source receipt')
receipt = matches[0]; observed = receipt['at']
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
raise ValueError('Invalid official receipt')
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
evidence = out/'evidence'; evidence.mkdir()
sources = {}
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
rows = []; variants = []
for fid in sorted(program):
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
'observed_at': observed, 'tags': [], 'metadata': {
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
'source_evidence': sources, 'archive_revision': REVISION,
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
'original_decision_signature_verified': False,
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
if len(variants) != 34:
raise ValueError('Reviewed title-variant census changed')
path = out/'corl-program-archive-2026-main.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--out', type=Path, required=True)
parser.add_argument('--official-snapshots', type=Path, required=True)
parser.add_argument('--archive-snapshots', type=Path, required=True)
args = parser.parse_args(argv)
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
+313
View File
@@ -0,0 +1,313 @@
"""Frozen SDM 2026 publication records and separate public program observations."""
import argparse
from collections import Counter
from datetime import date, datetime
import hashlib
from html.parser import HTMLParser
import json
from pathlib import Path
import re
from urllib.parse import parse_qs, urljoin, urlsplit
from paper_atlas.io.html import Document, clean
SOURCE = 'crossref_sdm_registry'
TRACK = 'proceedings_unclassified'
BOOK = '10.1137/1.9781611979329'
ISBN = '9781611979329'
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
SNAPSHOTS = {
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def one(items):
if len(items) != 1:
raise ValueError('Expected exactly one reviewed source field')
return items[0]
def load_snapshot(root, name):
expected, key = SNAPSHOTS[name]
directory = root/name; path = directory/'manifest.json'
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
raise ValueError('Unsafe snapshot manifest')
raw = path.read_bytes()
if digest(raw) != expected:
raise ValueError('Reviewed snapshot manifest hash changed')
manifest = json.loads(raw)
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
raise ValueError('Snapshot timestamp requires timezone')
files = {'manifest.json': raw}
for receipt in manifest[key]:
if 'file' not in receipt:
# A recorded 403 is an access observation, not an empty data set.
continue
filename = receipt['file']; path = directory/filename
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
raise ValueError('Unsafe snapshot source file')
body = path.read_bytes()
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
raise ValueError('Snapshot receipt/body disagree')
if filename in files:
raise ValueError('Repeated snapshot source file')
files[filename] = body
return manifest, files
def registry_papers(message):
items = message.get('items', [])
if message.get('total-results') != 55 or len(items) != 55:
raise ValueError('Incomplete exact-ISBN registry census')
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
raise ValueError('Unreviewed publication item types')
papers = []; excluded = []; seen = set()
for original in items:
doi = original.get('DOI')
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
raise ValueError('Missing/repeated DOI or wrong ISBN')
seen.add(doi)
if doi in (BOOK, BOOK+'.fm'):
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
if (original['type'], original.get('title')) != expected:
raise ValueError('Unreviewed excluded volume/front matter')
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
continue
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
or original.get('URL') != 'https://doi.org/'+doi
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
or original.get('relation', {}).get('is-part-of') != [
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
raise ValueError('Wrong publication edition, publisher or parent DOI')
title = one(original.get('title', []))
if not isinstance(title, str) or not clean(title):
raise ValueError('Missing registry title')
flags = []
if '<' in title or '>' in title:
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
if not match:
raise ValueError('Unreviewed title markup')
comparison_title = clean(match[1])
flags.append('title_sup_asterisk_footnote_requires_review')
else:
comparison_title = clean(Document(title).root.text())
displayed_title = clean(Document(title).root.text())
authors = original.get('author')
if (not isinstance(authors, list) or not authors
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
or not clean(a['given']) or not clean(a['family']) for a in authors)
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
raise ValueError('Incomplete registry authors or order evidence')
names = [clean(a['given']+' '+a['family']) for a in authors]
if any('@' in n or 'anonymous' in n.casefold() for n in names):
raise ValueError('Unreviewed registry author identity')
dates = {}
for field in ('published', 'published-print', 'published-online', 'issued'):
if field in original:
parts = one(original[field]['date-parts'])
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
raise ValueError('Unreviewed registry publication date')
dates[field] = date(*parts).isoformat()
if not dates or original.get('abstract'):
raise ValueError('Reviewed registry date/abstract availability changed')
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
'authors': names, 'review_flags': flags, 'publication_dates': dates})
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
raise ValueError('Incomplete reviewed 53-chapter DOI set')
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
class SessionParser(HTMLParser):
"""The source omits closing dt/dd tags; each next dt starts a new event."""
def __init__(self):
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
def finish(self):
if self.current is not None:
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
if not match or not clean(self.current['authors_literal']):
raise ValueError('Incomplete contributed program event')
self.current['time_literal'], self.current['title'] = match.groups()
self.current['authors_literal'] = clean(self.current['authors_literal'])
self.rows.append(self.current)
self.current = None; self.strong = False; self.author = False
def handle_starttag(self, tag, attrs):
attrs = dict(attrs)
if tag == 'dt':
self.finish()
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
if self.current is not None:
if tag == 'strong': self.strong = True
elif tag == 'dd': self.author = True
elif tag == 's': self.current['cancelled_presentation'] = True
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
self.current['abstract_links'].append(attrs['href'])
def handle_endtag(self, tag):
if tag == 'strong': self.strong = False
elif tag == 'dl': self.finish()
def handle_data(self, text):
if self.current is not None:
if self.strong: self.current['timed_title'] += text
elif self.author: self.current['authors_literal'] += text
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
def talk_fields(raw, session_url):
nodes = list(Document(raw.decode('utf-8')).root.walk())
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
raise ValueError('Wrong program page edition')
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
abstract = clean(abstract_node.text())
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
raise ValueError('Missing public program title/abstract')
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
def session_key(url):
u = urlsplit(url)
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
raise ValueError('Foreign program session link')
return query['sessioncode']
if session_key(link) != session_key(session_url):
raise ValueError('Program page belongs to a different session')
authors = []
for n in nodes:
if n.tag == 'li':
# Source rows contain name, affiliations, country and contact email.
# Only the independently checked leading name is a metadata field.
literal = clean(n.text()); name, separator, _ = literal.partition(',')
if not separator or not name or '@' in name:
raise ValueError('Unreviewed program author row')
authors.append(name)
if not authors:
raise ValueError('Missing source-reported program authors')
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
def program_observations(snapshots, papers):
official, official_files = snapshots['sdm2026-official-research-v1']
sessions, session_files = snapshots['sdm2026-program-research-v1']
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
raise ValueError('Contributed session membership disagrees with main program')
receipts = {r['url']: r for r in pages['records']}
if len(receipts) != len(pages['records']) or len(receipts) != 52:
raise ValueError('Incomplete or repeated public talk receipts')
rows = []; used = set()
for session in sessions['sessions']:
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
for position, event in enumerate(parser.rows, 1):
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
event.update(record_kind='source_observation_not_import_record', session=session['title'],
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
if not urls and event['cancelled_presentation']:
event.update(program_page=None, public_page_fields=None)
else:
url = one(urls)
if url not in receipts or url in used:
raise ValueError('Missing or repeated public program page')
used.add(url); receipt = receipts[url]
if receipt['session_url'] != session['url']:
raise ValueError('Public page receipt session disagrees')
fields = talk_fields(page_files[receipt['file']], session['url'])
if fields['title'] != event['title']:
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
candidates = []
for paper in papers:
if paper['comparison_title'].casefold() == event['title'].casefold():
fields = event['public_page_fields']
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
'comparison': 'title_candidate_only_not_identity_approval',
'registry_title_review_flags': paper['review_flags'],
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
event['registry_candidates'] = candidates
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
rows.append(event)
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
raise ValueError('Incomplete reviewed program event census')
return rows
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
message = json.loads(registry_files['isbn-works.json'])['message']
papers, excluded = registry_papers(message)
observations = program_observations(loaded, papers)
evidence = []; directory = out/'evidence'; directory.mkdir()
for name, (manifest, files) in loaded.items():
for filename, raw in files.items():
rel = 'evidence/'+digest(raw)+Path(filename).suffix
(out/rel).write_bytes(raw)
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
rows = []
for p in papers:
original = p['original']; doi = original['DOI']
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
'observed_at': receipt['at'], 'evidence_url': QUERY,
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
'original_decision_signature_verified': False,
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
sidecar = out/'program-observations.jsonl'
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
'cancelled_presentations': 1, 'not_an_import_manifest': True},
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
Binary file not shown.
Binary file not shown.
+75
View File
@@ -0,0 +1,75 @@
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import corl2026 as c
class CoRL2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
def test_official_census_and_presentation_dedup(self):
program = c.parse_program(self.bundle['official_html'])
self.assertEqual(len(program), 687)
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
def test_conflicting_official_title_is_not_silently_folded(self):
html = self.bundle['official_html']
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
self.assertIn(title, html)
with self.assertRaises(ValueError):
c.parse_program(html.replace(title, 'Different paper', 1))
def test_foreign_forum_source_is_rejected(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
with self.assertRaisesRegex(ValueError, 'another submission'):
c.join_metadata(program, archive)
def test_unknown_profile_fields_are_rejected(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'][0]['authorids'] = ['profile']
with self.assertRaisesRegex(ValueError, 'profile'):
c.join_metadata(program, archive)
def test_source_sets_must_agree(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'].pop()
with self.assertRaisesRegex(ValueError, 'forum sets'):
c.join_metadata(program, archive)
def test_complete_snapshot_and_explicit_version_provenance(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); official = root/'official'; archive = root/'archive'
official.mkdir(); archive.mkdir()
(official/'accepted.html').write_text(self.bundle['official_html'])
(official/'research.json').write_text(json.dumps(self.bundle['research']))
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
self.assertEqual(report['public_rejected_availability'], 'unknown')
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
self.assertEqual(len({r['external_id'] for r in rows}), 687)
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(out, official_snapshots=official, archive_snapshots=archive)
(official/'accepted.html').write_text('tampered')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
self.assertTrue((root/'failed/failed.json').exists())
self.assertFalse((root/'failed/collection-report.json').exists())
def test_forum_identity_validation(self):
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
with self.assertRaises(ValueError):
c.forum_id(bad)
+142
View File
@@ -0,0 +1,142 @@
import copy
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import sdm2026 as c
class SDM2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
def snapshots(self, root):
for name, text in self.fixture['files'].items():
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
papers, excluded = c.registry_papers(self.message)
self.assertEqual(len(papers), 53)
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
marked = [p for p in papers if p['review_flags']]
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
for p in marked:
self.assertTrue(p['title'].endswith('*'))
self.assertIn('<sup>*</sup>', p['original']['title'][0])
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); self.snapshots(root/'src')
report = c.collect(root/'out', snapshots=root/'src')
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
for row in rows:
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
sidecar = report['program_observations']
self.assertTrue(sidecar['not_an_import_manifest'])
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
self.assertEqual(len(observations), 53)
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
for observation in observations:
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
cancelled, = [x for x in observations if x['cancelled_presentation']]
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
self.assertIsNone(cancelled['public_page_fields'])
# The real source has conflicting author sets under the same title.
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
# Every copied source and manifest remains byte-identical, including CRLF.
for evidence in report['evidence']:
body = (root/'out'/evidence['output_file']).read_bytes()
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(root/'out', snapshots=root/'src')
def test_all_program_events_match_independent_inventory(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); self.snapshots(root)
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
papers, _ = c.registry_papers(self.message)
observations = c.program_observations(loaded, papers)
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
for item in observations:
original = gold[(item['session_url'], item['title'])]
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
self.assertEqual(item[key], original[key])
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
files = self.fixture['files']
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
receipts = {r['url']: r for r in manifest['records']}
self.assertEqual(len(self.fixture['page_gold']), 52)
for gold in self.fixture['page_gold']:
receipt = receipts[gold['program_url']]
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
fields = c.talk_fields(raw, receipt['session_url'])
self.assertEqual(fields['title'], gold['title'])
self.assertEqual(fields['source_reported_authors'], gold['authors'])
self.assertEqual(fields['abstract'], gold['abstract'])
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
with self.subTest(kind=kind):
message = copy.deepcopy(self.message)
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
if kind == 'isbn': row['ISBN'] = ['9780000000000']
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
if kind == 'repeat': row['DOI'] = c.BOOK
if kind == 'type': row['type'] = 'other'
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
if kind == 'author': row['author'][0]['sequence'] = 'additional'
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
if kind == 'missing': message['items'].pop()
with self.assertRaises(ValueError): c.registry_papers(message)
def test_foreign_session_and_changed_edition_rejected(self):
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
with self.assertRaises(ValueError): c.talk_fields(changed, url)
def test_corruption_preserves_failure_without_success_report(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); self.snapshots(root/'src')
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
with self.assertRaisesRegex(ValueError, 'receipt/body'):
c.collect(root/'out', snapshots=root/'src')
self.assertTrue((root/'out/failed.json').exists())
self.assertFalse((root/'out/collection-report.json').exists())
def test_source_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); self.snapshots(root/'src')
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
with self.assertRaisesRegex(ValueError, 'Unsafe'):
c.collect(root/'out', snapshots=root/'src')