Compare commits

...

13 Commits

Author SHA1 Message Date
codex-bot 19d03ced54 Verify installed CoRL collector against all 687 source records
Metadata integrity checks / metadata (push) Successful in 2m52s
2026-11-16 22:59:19 +08:00
codex-bot c33f5422de Collect CoRL 2026 program with versioned public metadata provenance
Metadata integrity checks / metadata (push) Successful in 2m51s
2026-11-16 22:54:52 +08:00
Zayden-Jung 1cfd44d8b6 Merge pull request '为RSS发表记录保留跨来源的同次投稿身份' (#53) from codex/rss-publication-identity into main
Metadata integrity checks / metadata (push) Successful in 2m48s
Reviewed-on: #53
2026-11-16 08:13:30 +00:00
Zayden-Jung 5de7f6ff22 Merge pull request '封装RSS2026官方论文集采集与来源差异' (#52) from codex-bot/rss2026 into main
Metadata integrity checks / metadata (push) Successful in 3m1s
Reviewed-on: #52
2026-11-16 08:13:16 +00:00
codex-bot b4e249e675 Recognize accepted RSS publisher DOI as an edition-scoped publication identity
Metadata integrity checks / metadata (push) Successful in 2m55s
2026-11-15 23:07:11 +08:00
codex-bot 93f00d1a64 Preserve failed-run checkpoint when snapshot receipts are invalid
Metadata integrity checks / metadata (push) Successful in 2m44s
2026-11-15 22:47:27 +08:00
codex-bot 8b7bc6d871 Include frozen RSS 2026 source fixture and independent 210-record inventory
Metadata integrity checks / metadata (push) Successful in 2m54s
2026-11-15 22:43:21 +08:00
codex-bot 7588a4e904 Collect reviewed RSS 2026 publisher metadata with separate program variants 2026-11-15 22:43:05 +08:00
Zayden-Jung 891a63d982 Merge pull request '封装来源身份只读预检与投稿历史证据' (#51) from codex/source-identity-audit into main
Metadata integrity checks / metadata (push) Successful in 2m49s
Reviewed-on: #51
2026-11-10 06:55:25 +00:00
codex-bot fe3e2502a5 Add read-only source identity candidate audit with evidence shards
Metadata integrity checks / metadata (push) Successful in 2m42s
2026-11-10 02:55:27 +08:00
Zayden-Jung b675a38b97 Merge pull request '适配AAMAS2026官方目录并保留撤稿与撤回证据' (#49) from codex/aamas2026 into main
Metadata integrity checks / metadata (push) Successful in 2m45s
Reviewed-on: #49
2026-11-09 18:49:37 +00:00
Zayden-Jung cb875b8eef Merge pull request '封装NeurIPS2026节目采集,分离投稿与重复展示' (#50) from codex/neurips-program into main
Metadata integrity checks / metadata (push) Successful in 2m34s
Reviewed-on: #50
2026-11-09 18:49:29 +00:00
codex-bot 6df904e6d8 Collect official NeurIPS 2026 program with forum-level presentation deduplication
Metadata integrity checks / metadata (push) Successful in 2m40s
2026-11-10 02:18:38 +08:00
22 changed files with 1622 additions and 1 deletions
+33
View File
@@ -48,8 +48,40 @@ jobs:
outside.mkdir()
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
rss_snapshots = root/'rss2026-snapshots'
rss_snapshots.mkdir()
for name, html in rss_fixture['pages'].items():
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
corl_official = root/'corl2026-official'
corl_archive = root/'corl2026-archive'
corl_official.mkdir()
corl_archive.mkdir()
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
neurips_snapshots = root/'neurips2026-program-snapshots'
neurips_snapshots.mkdir()
(neurips_snapshots/'program-json.response').write_bytes(gzip.decompress((src/'tests/fixtures/neurips2026-program.json.gz').read_bytes()))
(neurips_snapshots/'receipts.json').write_bytes((src/'tests/fixtures/neurips2026-receipts.json').read_bytes())
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--snapshots',str(neurips_snapshots),'--out',str(root/'installed-neurips2026-program')],cwd=outside,check=True)
neurips_report = json.loads((root/'installed-neurips2026-program/neurips-program-2026.json').read_text())
assert neurips_report['records'] == 9004 and len(neurips_report['held_presentations']) == 139
assert len(neurips_report['exact_duplicate_event_rows']) == 1 and neurips_report['folded_extra_presentation_rows'] == 132
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
@@ -86,6 +118,7 @@ jobs:
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
+10
View File
@@ -70,6 +70,8 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
--out /data/paper-index/collected
```
NeurIPS 2026 的[官方节目采集器](docs/neurips-program.md)按 main、evaluations/datasets、position 分开保存,合并同一 forum 的口头与海报展示,保留期刊等未决关系;运行 `paper-atlas collect neurips-program`。它只生成元数据文件,身份核验和生产导入另行执行。
早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别,保留原名与范围证据;代码支持不代表已补齐生产数据。
EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验,保留原始系列及事件/出版年份;旧候选文件缺少 ECP/EWSP 时会明确提示,避免静默遗漏。
@@ -114,3 +116,11 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
### CoRL 2026 官方程序快照
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
+24
View File
@@ -0,0 +1,24 @@
# CoRL 2026:官方程序与公开档案
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
```sh
paper-atlas collect corl2026-program \
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
--out /data/paper-index/collected/corl2026-normalized-v1
```
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
+6
View File
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
### RSS 发表身份
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
+56
View File
@@ -0,0 +1,56 @@
# NeurIPS 官方节目元数据
`paper-atlas collect neurips-program` 是按需运行的采集器,当前核验范围是 2026 年的[官方 orals/posters JSON](https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json)。它只请求这个固定公开元数据文件,不连接数据库,不请求 OpenReview API,不下载 PDF、视频或个人资料页。模块位于 `paper_atlas.collectors.neurips_program`,通过统一 CLI 使用。
## 已核验快照
2026-11-09T17:57:40Z 冻结的原始响应共 20,447,021 字节,SHA256 `879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997`。原始 `count=9276` 与结果数组长度一致,`next`、`previous` 均为空;这只证明此份公开文件已完整取得,不是全部会议出版物或公开拒稿全集。
按官方逐条 `sourceurl` 中的明确投稿来源分开处理:
| 原始 OpenReview group | 数据库 track | 不同 forum 数 |
| --- | --- | ---: |
| `NeurIPS.cc/2026/Conference` | `main` | 7896 |
| `NeurIPS.cc/2026/Evaluations_and_Datasets_Track` | `evaluations_datasets` | 971 |
| `NeurIPS.cc/2026/Position_Paper_Track` | `position` | 137 |
| 合计 | | 9004 |
只接受已核验的显式 `Accept`、`Accept (poster/spotlight/oral)` 标签,保留原始大小写和文本。未知来源、决定、展示类型、可见性或无法验证的 forum 链接进入 `held_presentations`,不推成 accepted 或 rejected。空标题、缺少作者、相同 event ID 不同内容、同一 forum 的题名/作者/来源/决定冲突会停止采集。
原 9276 行中,event 154866 有一条内容完全相同的重复行,单独保留其原始行号;另外 132 个口头展示与同 forum 的海报合并为一条投稿候选,完整保留两种展示及时间、场次、原始带符号 `sourceid`。负数 `sourceid` 不能被解释为拒稿或另一次投稿,也不转换为全局论文标识。
其余 139 行单独留档:87 个期刊展示、36 个 Bridging the Future 展示、16 个未提供明确投稿来源的条目。这些不因缺少链接而被推为拒稿,也不能被冒充为独立主会审稿录用。后续须核实原期刊发表、认证、复现或其他展示关系。
9004 条是不同 forum 的来源记录,不是已经新增 9004 篇独立科学论文。不同 forum 即使题名和作者相同也保留各自投稿;跨会及版本关系另行核验。题名、完整作者顺序和机构取自原始 JSON,口头与海报不是研究类型;展示标签独立使用 `neurips_program:presentation_format` namespace。节目中的未来展示时间不冒充决定时间。
## 运行与恢复
在线模式只请求一个固定元数据 URL,并复用受限大小的响应缓存:
```sh
paper-atlas collect neurips-program \
--cache /data/paper-index/cache/neurips2026-program \
--out /data/paper-index/collected/neurips2026-program-YYYYMMDD
```
冻结模式只读 `receipts.json` 对应的原始文件:
```sh
paper-atlas collect neurips-program \
--snapshots /data/paper-index/collected/neurips2026-program-probe-v1 \
--out /data/paper-index/collected/neurips2026-program-normalized-v1
```
快照回执包含精确 `url`、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区的 `finished_at`。拒绝越界路径/符号链接、重复回执、失败响应、大小或哈希不符。在线模式沿用项目 HTTP 缓存和限速,验证回执时间后保存原始 JSON。输出目录有互斥锁,完成报告最后发布;已有完成报告时要求新建输出目录,避免覆盖后误重放。
输出三个 track 的 JSONL/manifest、原始 JSON 证据和 `neurips-program-2026.json`。manifest 始终为 `partial`;原始节目 JSON 没有摘要,9004 条 `abstract=null` 和缺失原因如实保存,不能用别篇摘要填充。命令终端只输出数量与路径,完整待处理展示与原始行号在报告中。
恢复时先校验文件和回执,使用同一完整快照生成新输出;若已有完成报告或导入回执,先核实实际来源绑定和 checkpoint,不重放已消费队列。生成文件后还需 PostgreSQL 身份预检、关联审核、增量队列、原生导入和独立验收;源文件成功、CI 通过均不能替代生产导入或最终全库去重。
## 目录差异与公开拒稿限制
同期[官网 Downloads 页面](https://neurips.cc/Downloads/2026)的活动总数不是本 JSON 的论文总数。逐个 poster ID/题名对照后,网页另有 event 155165(Lemon)未出现在本 JSON 中,须单独取得并审核元数据,不能补造其 track 或 forum。16 个未提供来源的条目在 JSON 中使用通用 session URL,而网页有各自 poster URL;原值保留,不能因 URL 映射差异制造新论文。
[2026 Main Track Handbook](https://neurips.cc/Conferences/2026/MainTrackHandbook)明确允许 rejected 作者选择公开。当前无账号公开 notes API 请求返回 `ChallengeRequiredError`、HTTP 403,失败原文和回执保留,不绕过验证;因此不能把本 accepted 节目文件当作拒稿数据源或据此推断拒稿是否存在。官网下载表单的 JSON 导出也曾超时,未取得导出内容,已改用独立公开静态节目文件并保留失败证据,不重试超时导出。
当前主动补漏范围从 2020 年起。期刊/展示关系、网页独有条目、摘要和公开负面决定仍有待办;冻结资料和代码适配不代表完整会议覆盖。
+21
View File
@@ -0,0 +1,21 @@
# RSS 2026 论文集来源
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
```sh
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
--out /data/paper-index/collected/rss2026-native-v1
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
--out /data/paper-index/collected/rss2026-native-replay-v1
```
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
+21
View File
@@ -0,0 +1,21 @@
# 来源身份只读预检
`audit source-identities` 在 PostgreSQL 的同一 `REPEATABLE READ READ ONLY` 快照中,检查规范化 JSONL 的来源绑定、稳定标识及精确标题候选,并导出候选论文的原始来源和投稿决定历史。用于入库前评审,不执行导入、合并或标识修改。
```sh
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit source-identities \
--pg-service foundry-research --pg-schema research \
--files /data/paper-index/collected/example/main.jsonl \
/data/paper-index/collected/example/position.jsonl \
--out /data/paper-index/collected/example-identity-preflight-v1
```
每个 JSONL 必须有同名 `.manifest.json`,包含 `source`、`venue`、`year`、`track`、`count` 和 `file_sha256`。逐行校验届次、显式决定状态、标识类型和跨文件来源主键唯一性;文件哈希和数量必须相符。只读取调用者指定的文件,不跟随 manifest 中的绝对 `file` 路径;原始行哈希包含换行符。原始 JSONL 和 manifest 应与报告一起保留。
输出目录必须不存在,防止复用旧报告或覆盖证据。失败会保留 `failed.json` 及已完成分片,不能把部分输出当成功;排查后使用新的输出目录重新核查。成功报告 `postgresql-preflight.json` 最后原子写入,记录输入哈希、数据库快照和每条候选。实体详情存为 `entities/<分片>/<paper_id>.json.gz`,报告中列出压缩文件的 SHA-256;包含完整 paper、标识和限量原始来源、投稿、决定及出处。每篇最多保留 100 条来源、200 次投稿,每次投稿最多 200 条决定,每组决定或投稿出处最多 100 条,所有截断均显式标记。精确标题最多取 21 个候选;达到上限即标记,不能据此推算总数。这里没有计算全表精确数量。
候选分类用于安排审核:既有来源记录需要核对并避免重放,多个绑定冲突必须暂缓;单个 DOI、arXiv 或 OpenReview 绑定也可能错误,必须核对原文、标题、作者和历史。作者比较保留完整规范化姓名的重复次数,但姓名相同不能证明是同一个人。未找到精确标题或标识不能证明是新作品,仍需近似标题、版本和跨会历史核查。
所有条目固定为 `not_approved_for_import`,批准数量为 0。不要直接把此报告当成 curation 计划或导入队列。后续获批导入前应重新检查实时来源绑定;来源摘要缺失不得覆盖已有摘要,旧拒稿与新录用应作为不同投稿经历保留。本命令不能代替所有导入结束后的全库去重、覆盖和约束验收。
构建与测试仅在已确认的 `ces-control:host` 远程 runner 执行。集成测试使用已有 PostgreSQL 服务做真实只读候选查询,不往生产库添加测试记录。
+252
View File
@@ -0,0 +1,252 @@
"""Read-only, evidence-preserving identity candidates for normalized source files.
A candidate is never permission to import or merge. Absence of an exact match is
never proof of a new work. Run again against a fresh snapshot before curation.
"""
import argparse
from collections import Counter
import gzip
import hashlib
import json
from pathlib import Path
from paper_atlas.db.store import author_keys, normalized
from paper_atlas.db.submissions import timestamp
from paper_atlas.postgres.client import ReadConnection
STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'}
def digest(data):
return hashlib.sha256(data).hexdigest()
def authors_multiset(authors):
# author_keys intentionally removes duplicates; identity review must retain them.
return Counter(key for author in authors for key in author_keys([author]))
def load_inputs(files):
records, inputs, seen = [], [], set()
for filename in files:
path = Path(filename).resolve()
manifest_path = path.with_suffix('.manifest.json')
raw_manifest = manifest_path.read_bytes()
manifest = json.loads(raw_manifest)
hasher, count = hashlib.sha256(), 0
with path.open('rb') as stream:
for line_number, line in enumerate(stream, 1):
hasher.update(line)
row = json.loads(line)
for key in ('source', 'external_id', 'title', 'venue', 'track'):
if not isinstance(row.get(key), str) or not row[key].strip():
raise ValueError(f'Invalid {key}: {path.name}:{line_number}')
if type(row.get('year')) is not int or row.get('status') not in STATUSES:
raise ValueError('Invalid year or explicit source status')
for key in ('source', 'venue', 'year', 'track'):
if row[key] != manifest.get(key):
raise ValueError(f'Manifest scope mismatch: {key}')
authors = row.get('authors')
if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors):
raise ValueError('Invalid authors')
identifiers = row.get('identifiers', {})
if not isinstance(identifiers, dict) or any(
not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip()
for k, v in identifiers.items()):
raise ValueError('Identifiers must contain singular nonempty strings')
key = (row['source'], row['external_id'])
if key in seen:
raise ValueError('Duplicate source/external_id across inputs')
seen.add(key)
record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors',
'venue', 'year', 'track', 'status')}
record.update(identifiers=identifiers, file=str(path), line=line_number,
raw_line_sha256=digest(line), source_record_hits=[],
identifier_hits=[], title_candidate_ids=[])
records.append(record)
count += 1
if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'):
raise ValueError('Input count/hash mismatch or empty input')
inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count,
'manifest_sha256': digest(raw_manifest), 'manifest': manifest})
if not records:
raise ValueError('At least one nonempty input is required')
return records, inputs
def _lookup(connection, values, columns, lateral):
"""Bound scalar VALUES allow the existing text codec to encode each key."""
width = len(columns)
for start in range(0, len(values), 100):
batch = values[start:start + 100]
sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join(
'(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true'
for row in connection.execute(sql, tuple(value for item in batch for value in item)):
yield dict(row)
def find_candidates(connection, records):
source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)]
for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), '''
SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id
WHERE s.name=keys.source AND r.external_id=keys.external_id'''):
records[hit.pop('n')]['source_record_hits'].append(hit)
identifiers = []
for i, row in enumerate(records):
pairs = {(row['source'], row['external_id']), *row['identifiers'].items()}
identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs))
for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), '''
SELECT scheme,value,paper_id FROM identifiers
WHERE scheme=keys.scheme AND value=keys.value'''):
records[hit.pop('n')]['identifier_hits'].append(hit)
title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)]
for hit in _lookup(connection, title_keys, ('n', 'title_key'), '''
SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''):
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
def entity_packet(connection, paper_id):
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
if paper is None:
raise RuntimeError('Candidate disappeared from read-only snapshot')
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
FROM source_records r JOIN sources s ON s.id=r.source_id
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
for attempt in packet['attempts']:
observations = [dict(r) for r in connection.execute(
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
for decision in attempt['decisions']:
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
return packet
def classify(record, entities):
anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']}
titles = record['title_candidate_ids']
flags = []
own_authors = authors_multiset(record['authors'])
if not own_authors:
flags.append('missing_source_authors')
if any(n > 1 for n in own_authors.values()):
flags.append('duplicate_normalized_author_names')
if len(normalized(record['title'])) < 25:
flags.append('short_title')
comparisons = []
for pid in sorted(anchors | set(titles)):
entity = entities[pid]
other = authors_multiset(entity['authors'])
comparisons.append({'paper_id': pid,
'normalized_title_equal': normalized(record['title']) == normalized(entity['title']),
'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other,
'author_name_overlap': sorted(own_authors.keys() & other.keys())})
if len(anchors) > 1:
kind = 'conflicting_existing_bindings_hold'
elif record['source_record_hits']:
kind = 'existing_source_record_reconcile_no_replay'
elif anchors:
kind = 'existing_identifier_binding_requires_review'
elif len(titles) >= 21:
kind = 'title_candidate_limit_reached_hold'
elif not titles:
kind = 'no_exact_match_not_proof_of_novelty'
elif len(titles) > 1:
kind = 'multiple_exact_title_entities_hold'
elif comparisons[0]['nonempty_author_multiset_equal']:
kind = 'one_exact_title_author_multiset_requires_review'
elif comparisons[0]['author_name_overlap']:
kind = 'partial_author_overlap_requires_review'
else:
kind = 'no_author_agreement_hold'
return {'kind': kind, 'flags': flags, 'comparisons': comparisons,
'title_candidates_truncated': len(titles) >= 21,
'decision': 'not_approved_for_import'}
def _atomic_json(path, value):
temporary = path.with_suffix(path.suffix + '.part')
with temporary.open('w', encoding='utf-8') as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write('\n')
temporary.replace(path)
def audit(files, out, *, pg_service, pg_schema):
# Reject input corruption before opening a database connection or reserving output.
records, inputs = load_inputs(files)
out = Path(out)
out.mkdir(parents=True, exist_ok=False)
(out / 'audit.lock').write_text(timestamp() + '\n')
connection = None
try:
connection = ReadConnection(pg_service, pg_schema)
connection.execute("SET LOCAL statement_timeout='60s'")
if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on':
raise RuntimeError('A read-only transaction is required')
snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema,
transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone())
find_candidates(connection, records)
ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} |
{pid for row in records for pid in row['title_candidate_ids']})
entities, index = {}, {}
for number, pid in enumerate(ids, 1):
packet = entity_packet(connection, pid)
relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz'
target = out / relative
target.parent.mkdir(parents=True, exist_ok=True)
raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0)
target.write_bytes(raw)
index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)}
entities[pid] = {'title': packet['paper']['title'],
'authors': json.loads(packet['paper']['authors_json'])}
if number % 250 == 0 or number == len(ids):
print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True)
kinds = Counter()
for row in records:
row.update(classify(row, entities))
kinds[row['kind']] += 1
report = {'status': 'completed_read_only_candidates_not_import_approval',
'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True,
'database_written': False, 'approved_for_import': 0, 'records': len(records),
'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records,
'limitations': ['Exact keys only; no match does not establish a new work.',
'Name agreement does not prove person or scientific-work identity.',
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
'Source/history limits are explicit in compressed entity packets.',
'Not a final whole-database deduplication or coverage audit.']}
_atomic_json(out / 'postgresql-preflight.json', report)
return report
except Exception as error:
_atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval',
'error_type': type(error).__name__, 'database_written': False})
raise
finally:
if connection is not None:
connection.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--files', nargs='+', required=True, type=Path)
parser.add_argument('--pg-service', required=True)
parser.add_argument('--pg-schema', required=True)
parser.add_argument('--out', required=True, type=Path)
args = parser.parse_args(argv)
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
+4
View File
@@ -23,9 +23,12 @@ COMMANDS = {
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
('collect','dblp-family'): ('paper_atlas.collectors.dblp_recovery', {}, '从固定快照恢复指定 DBLP family 的原始候选'),
@@ -45,6 +48,7 @@ COMMANDS = {
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
}
+183
View File
@@ -0,0 +1,183 @@
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import unicodedata
from urllib.parse import parse_qs, urlsplit
from paper_atlas.io.html import Document, clean
SOURCE = 'corl_program_archive'
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def comparable(text):
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
def forum_id(url):
if not isinstance(url, str):
raise ValueError('Forum URL must be text')
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
raise ValueError('Invalid original forum identity')
return q['id'][0]
def parse_program(raw):
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
if len(embeds) != 1:
raise ValueError('Expected one official embedded program')
doc = Document(embeds[0]); nodes = list(doc.root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
raise ValueError('Unexpected program edition')
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
raise ValueError('Reviewed program census changed')
papers = {}; sessions = {}; occurrences = []
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
sid = section.attrs.get('data-sess', section.attrs.get('id'))
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
raise ValueError('Unexpected or repeated presentation session')
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
if len(headings) != 1:
raise ValueError('Ambiguous session heading')
sessions[sid] = {'kind': kind, 'heading': headings[0]}
seen = set()
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
fid = forum_id(link.attrs['href']); title = clean(link.text())
if fid in seen or not title:
raise ValueError('Repeated or empty paper in session')
seen.add(fid)
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
if paper['title'] != title:
raise ValueError('Conflicting official titles share a forum')
event = {'session_id': sid, **sessions[sid]}
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
counts = Counter(kind for _, _, kind in occurrences)
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
raise ValueError('Incomplete official program')
for paper in papers.values():
kinds = Counter(p['kind'] for p in paper['presentations'])
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
raise ValueError('Invalid poster/talk multiplicity')
return papers
def join_metadata(program, archive):
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
or not isinstance(archive.get('papers'), list)):
raise ValueError('Unpinned public metadata provenance')
indexed = {}
for row in archive['papers']:
if not isinstance(row, dict) or set(row) - FIELDS:
raise ValueError('Unexpected metadata/profile fields')
fid = forum_id(row.get('openreview'))
if fid != row.get('id') or fid in indexed:
raise ValueError('Repeated or mismatched archive forum')
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
raise ValueError('Field source belongs to another submission')
if (not isinstance(row.get('title'), str) or not row['title'].strip()
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
or not isinstance(row.get('authors'), list) or not row['authors']
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
raise ValueError('Incomplete scholarly metadata')
indexed[fid] = row
if set(indexed) != set(program):
raise ValueError('Official and archive forum sets differ')
return indexed
def collect(out, *, official_snapshots, archive_snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
if official.is_symlink() or filtered.is_symlink():
raise ValueError('Snapshot symlinks rejected')
raw = official.read_bytes(); data = filtered.read_bytes()
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
raise ValueError('Reviewed snapshot hash changed')
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
matches = [r for r in receipts if r.get('url') == OFFICIAL]
if len(matches) != 1:
raise ValueError('Missing official source receipt')
receipt = matches[0]; observed = receipt['at']
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
raise ValueError('Invalid official receipt')
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
evidence = out/'evidence'; evidence.mkdir()
sources = {}
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
rows = []; variants = []
for fid in sorted(program):
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
'observed_at': observed, 'tags': [], 'metadata': {
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
'source_evidence': sources, 'archive_revision': REVISION,
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
'original_decision_signature_verified': False,
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
if len(variants) != 34:
raise ValueError('Reviewed title-variant census changed')
path = out/'corl-program-archive-2026-main.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--out', type=Path, required=True)
parser.add_argument('--official-snapshots', type=Path, required=True)
parser.add_argument('--archive-snapshots', type=Path, required=True)
args = parser.parse_args(argv)
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
+218
View File
@@ -0,0 +1,218 @@
"""Official NeurIPS program metadata, with presentations separate from submissions."""
import argparse
from collections import Counter, defaultdict
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import parse_qs, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.http import fetch
URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json'
GROUPS = {
'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main',
'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets',
'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position',
}
ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)',
'Accept (Spotlight)', 'Accept (Oral)'}
LIMIT = 64*1024**2
def forum_id(url):
if not isinstance(url, str):
return None
try:
parsed = urlsplit(url)
except ValueError:
return None
query = parse_qs(parsed.query, keep_blank_values=True)
if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or
parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or
len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])):
return None
return query['id'][0]
def parse_program(data, evidence):
root = json.loads(data)
if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or
not isinstance(root.get('results'), list) or
type(root.get('count')) is not int or root['count'] != len(root['results']) or
root.get('next') is not None or root.get('previous') is not None):
raise ValueError('Expected one complete, nonpaginated public program snapshot')
if not root['results']:
raise ValueError('Empty program snapshot')
events, grouped, held, duplicates = {}, defaultdict(list), [], []
for index, event in enumerate(root['results']):
if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0:
raise ValueError('Missing stable event ID')
eid = event['id']
if eid in events:
first_index, first = events[eid]
if first != event:
raise ValueError('Conflicting payloads share one publisher event ID')
duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index})
continue
events[eid] = index, event
origin = event.get('sourceurl')
reason = None
if not isinstance(origin, str) or origin not in GROUPS:
reason = ('journal_presentation_requires_original_publication_relation'
if isinstance(origin, str) and origin.startswith('journal-track/') else
'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else
'unreviewed_or_missing_submission_origin')
elif event.get('visible') is not True:
reason = 'program_visibility_requires_review'
elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED:
reason = 'unreviewed_decision_label'
elif event.get('event_type') not in ('Poster', 'Oral'):
reason = 'unreviewed_presentation_type'
forum = forum_id(event.get('paper_url'))
if reason is None and forum is None:
reason = 'missing_or_invalid_original_submission_link'
if reason:
held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event})
continue
if (not isinstance(event.get('name'), str) or not event['name'].strip() or
not isinstance(event.get('authors'), list) or not event['authors'] or
any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or
not a['fullname'].strip() for a in event['authors'])):
raise ValueError('Empty title or incomplete ordered author list')
if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch(
r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']):
raise ValueError('Publisher event URL differs from year/type/ID')
grouped[forum].append((index, event))
records = []
folded = 0
for forum, observations in sorted(grouped.items()):
# Signed sourceid values are preserved verbatim in the presentation
# evidence; they are not independent submissions or scientific works.
signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']],
ensure_ascii=False, sort_keys=True) for _, e in observations}
if len(signatures) != 1:
raise ValueError('Title, authors, origin or decision disagree for one forum')
posters = [e for _, e in observations if e['event_type'] == 'Poster']
event = posters[0] if posters else observations[0][1]
track = GROUPS[event['sourceurl']]
presentations = []
for index, e in observations:
presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'),
'sourceid': e.get('sourceid'), 'event_type': e['event_type'],
'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'],
'session': e.get('session'), 'starttime': e.get('starttime'),
'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'),
'raw_decision': e['decision']})
formats = sorted({e['event_type'] for _, e in observations})
records.append({
'source': 'neurips_program', 'external_id': '2026/'+forum,
'url': event['paper_url'], 'title': event['name'], 'abstract': None,
'authors': [a['fullname'] for a in event['authors']],
'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [],
'year': 2026, 'track': track, 'status': 'accepted',
'observed_at': evidence['observed_at'], 'raw_decision': event['decision'],
'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum,
'author_affiliations': [a.get('institution') for a in event['authors']],
'presentations': presentations, 'source_evidence': evidence,
'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.',
'decision_date_precision': 'not_supplied',
'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.',
'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'},
'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value}
for value in formats]})
folded += len(observations)-1
if not records:
raise ValueError('No reviewed submission origins found')
if root['count'] != len(records)+len(held)+len(duplicates)+folded:
raise ValueError('Program partition failed')
return records, held, duplicates, folded
def read_program(cache, snapshots):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one cache or frozen snapshot directory')
if snapshots:
base = Path(snapshots).resolve()
matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL]
if len(matches) != 1:
raise ValueError('Expected one official program receipt')
r = matches[0]; filename = Path(r['file'])
if (filename.is_absolute() or '..' in filename.parts or
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
raise ValueError('Unsafe or unsuccessful frozen program response')
data = (base/filename).read_bytes()
if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
raise ValueError('Frozen program checksum/size mismatch')
observed = r['finished_at']
else:
data = fetch(URL, cache, LIMIT, delay=1)
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
observed = r['retrieved_at']
timestamp(observed)
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json',
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
def collect(out, *, cache=None, snapshots=None):
out = Path(out); out.mkdir(parents=True, exist_ok=True)
report_name = 'neurips-program-2026.json'
with (out/'neurips-program.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if (out/report_name).exists():
raise ValueError('Completed collection exists; use a new directory and reconcile before refresh')
data, evidence = read_program(cache, snapshots)
records, held, duplicates, folded = parse_program(data, evidence)
groups = defaultdict(list)
for r in records:
groups[r['track']].append(r)
files = []
with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary:
stage = Path(temporary); (stage/'evidence').mkdir()
(stage/evidence['file']).write_bytes(data)
for track, group in sorted(groups.items()):
p = stage/f'neurips-program-2026-{track}.jsonl'
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS',
'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name),
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
'status': 'partial', 'missing_abstract_count': len(group),
'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'}
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(),
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
'held_presentations': held, 'exact_duplicate_event_rows': duplicates,
'folded_extra_presentation_rows': folded, 'source_receipts': [evidence],
'database_written': False, 'pdfs_downloaded': False,
'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'}
(stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
(out/'evidence').mkdir(exist_ok=True)
for p in (stage/'evidence').iterdir():
p.replace(out/'evidence'/p.name)
for p in stage.iterdir():
if p.is_file() and p.name != report_name:
p.replace(out/p.name)
(stage/report_name).replace(out/report_name)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--out', required=True)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument('--cache')
source.add_argument('--snapshots')
args = parser.parse_args(argv)
result = collect(args.out, cache=args.cache, snapshots=args.snapshots)
print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'],
'records': result['records'], 'files': result['files'],
'held_presentations': len(result['held_presentations']),
'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']),
'folded_extra_presentation_rows': result['folded_extra_presentation_rows'],
'report_file': str(Path(args.out)/'neurips-program-2026.json'),
'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False))
+222
View File
@@ -0,0 +1,222 @@
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
import argparse
from datetime import datetime, date
import hashlib
import json
from pathlib import Path
import re
import unicodedata
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'rss_proceedings'
ROOT = 'https://www.roboticsproceedings.org/rss22/'
HEADING = 'Robotics: Science and Systems XXII'
PAGES = {'index.html': ROOT,
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def one(values):
if len(values) != 1:
raise ValueError('Expected one reviewed HTML field')
return values[0]
def comparable(value):
return unicodedata.normalize('NFC', clean(value))
def parse_lists(index_html, program_html, policy_html, sponsors_html):
policy = clean(Document(policy_html).root.text())
for text in ('RSS 2026 will not have a separate category for demo papers.',
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
if text not in policy:
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
raise ValueError('Sponsor activity page changed; never classify these as papers')
nodes = list(Document(index_html).root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
raise ValueError('Unexpected publisher edition')
index = {}
for row in (n for n in nodes if n.tag == 'tr'):
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
if not links:
continue
link = one(links); pid = int(link.attrs['href'][1:4])
if pid in index:
raise ValueError('Repeated publisher ID')
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
program = {}
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
raise ValueError('Unexpected accepted program')
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
if not cells:
continue
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
raise ValueError('Unexpected program row')
pid = int(clean(cells[0].text()))
if pid in program:
raise ValueError('Repeated program ID')
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
if not preview or not full.startswith(preview):
raise ValueError('Author preview is not a prefix of complete author evidence')
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
if links != [f'/2026/program/papers/{pid}/']:
raise ValueError('Program display link changed')
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
'authors_preview': preview, 'session': clean(cells[1].text()),
'program_url': 'https://roboticsconference.org'+links[0]}
if set(index) != set(program) or set(index) != set(range(1, 211)):
raise ValueError('Reviewed 210-entry source census changed')
return index, program
def parse_paper(html, pid, index, program):
nodes = list(Document(html).root.walk()); meta = {}
for n in nodes:
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
day = one(meta.get('citation_publication_date', []))
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
raise ValueError('Unexpected publication date')
date.fromisoformat(day.replace('/', '-'))
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
or not authors or any(not clean(a) for a in authors)
or comparable(title) != comparable(index['title'])
or comparable(', '.join(authors)) != comparable(index['authors'])
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
raise ValueError('Publisher citation/index/visible metadata disagree')
pdf = one(meta.get('citation_pdf_url', []))
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
bib = one([n.text() for n in nodes if n.tag == 'pre'])
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
raise ValueError('BibTeX DOI/year mismatch')
variants = []
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
raise ValueError('Unreviewed complete author variant')
variants.append('reviewed_program_author_name_variant_not_person_alias')
if comparable(title) != comparable(program['title']):
if (pid, title, program['title']) != (89,
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
raise ValueError('Unreviewed program title variant')
variants.append('reviewed_program_title_variant_not_erratum')
inside = False; parts = []; markers = []
for n in (n for n in nodes if n.tag == 'p'):
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
if 'Abstract:' in labels:
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
if value: parts.append(value)
elif 'Download:' in labels:
inside = False; markers.append('end')
elif inside:
parts.append(clean(n.text()))
if markers != ['start', 'end'] or not any(parts):
raise ValueError('Missing or changed publisher abstract layout')
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
def collect(out, *, snapshots=None, cache=None):
if (snapshots is None) == (cache is None):
raise ValueError('Specify snapshots or cache')
out = Path(out)
if out.exists():
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
out.parent.mkdir(parents=True, exist_ok=True)
out.mkdir()
evidence = {}; receipts = {}
if snapshots is not None:
root = Path(snapshots)
try:
for r in json.loads((root/'receipts.json').read_bytes()):
name = r['file']
if not isinstance(name, str) or Path(name).name != name or name in receipts:
raise ValueError('Unsafe or repeated snapshot receipt')
receipts[name] = r
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def page(name, url):
if snapshots is not None:
path = root/name
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
raw = path.read_bytes(); r = receipts[name]
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
raise ValueError('Snapshot hash/receipt mismatch')
at = r['at']
else:
raw = fetch(url, cache, limit=8*1024*1024)
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
at = sidecar['retrieved_at']
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
raise ValueError('Invalid source size/timezone')
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
return raw.decode('utf-8')
try:
pages = {name: page(name, url) for name, url in PAGES.items()}
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
records = []
for pid in sorted(index):
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
'year': 2026, 'track': 'main', 'status': 'accepted',
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
'observed_at': evidence[name]['observed_at'],
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
'original_bibtex': fields['bibtex'], 'program': program[pid],
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
'policy_evidence': evidence['cfp.html'],
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
records.append(row)
path = out/'rss-proceedings-2026-main.jsonl'
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
path.write_bytes(raw)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
temporary.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
p = argparse.ArgumentParser(prog=prog, description=__doc__)
p.add_argument('--out', required=True, type=Path)
g = p.add_mutually_exclusive_group(required=True)
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
args = p.parse_args(argv)
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
+1 -1
View File
@@ -4,7 +4,7 @@ import re
from paper_atlas.io.publication_ids import mlsys_id
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
def timestamp(value=None):
Binary file not shown.
Binary file not shown.
+15
View File
@@ -0,0 +1,15 @@
{
"receipts": [
{
"url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
"final_url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
"status": 200,
"content_type": "application/json",
"file": "program-json.response",
"bytes": 20447021,
"sha256": "879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997",
"started_at": "2026-11-09T17:57:23.825509+00:00",
"finished_at": "2026-11-09T17:57:40.125580+00:00"
}
]
}
Binary file not shown.
+75
View File
@@ -0,0 +1,75 @@
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import corl2026 as c
class CoRL2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
def test_official_census_and_presentation_dedup(self):
program = c.parse_program(self.bundle['official_html'])
self.assertEqual(len(program), 687)
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
def test_conflicting_official_title_is_not_silently_folded(self):
html = self.bundle['official_html']
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
self.assertIn(title, html)
with self.assertRaises(ValueError):
c.parse_program(html.replace(title, 'Different paper', 1))
def test_foreign_forum_source_is_rejected(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
with self.assertRaisesRegex(ValueError, 'another submission'):
c.join_metadata(program, archive)
def test_unknown_profile_fields_are_rejected(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'][0]['authorids'] = ['profile']
with self.assertRaisesRegex(ValueError, 'profile'):
c.join_metadata(program, archive)
def test_source_sets_must_agree(self):
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
archive['papers'].pop()
with self.assertRaisesRegex(ValueError, 'forum sets'):
c.join_metadata(program, archive)
def test_complete_snapshot_and_explicit_version_provenance(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); official = root/'official'; archive = root/'archive'
official.mkdir(); archive.mkdir()
(official/'accepted.html').write_text(self.bundle['official_html'])
(official/'research.json').write_text(json.dumps(self.bundle['research']))
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
self.assertEqual(report['public_rejected_availability'], 'unknown')
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
self.assertEqual(len({r['external_id'] for r in rows}), 687)
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(out, official_snapshots=official, archive_snapshots=archive)
(official/'accepted.html').write_text('tampered')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
self.assertTrue((root/'failed/failed.json').exists())
self.assertFalse((root/'failed/collection-report.json').exists())
def test_forum_identity_validation(self):
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
with self.assertRaises(ValueError):
c.forum_id(bad)
+174
View File
@@ -0,0 +1,174 @@
import copy
from collections import Counter
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from urllib.parse import parse_qs, urlsplit
from paper_atlas.collectors import neurips_program as program
FIXTURES = Path(__file__).parent/'fixtures'
RAW = gzip.decompress((FIXTURES/'neurips2026-program.json.gz').read_bytes())
RECEIPTS = json.loads((FIXTURES/'neurips2026-receipts.json').read_text())
EVIDENCE = {'url': program.URL, 'observed_at': RECEIPTS['receipts'][0]['finished_at']}
class NeuripsProgramTests(unittest.TestCase):
def sample(self):
rows = json.loads(RAW)['results']
oral = next(r for r in rows if r['id'] == 154866)
poster = next(r for r in rows if r['paper_url'] == oral['paper_url'] and r['event_type'] == 'Poster')
return copy.deepcopy(poster), copy.deepcopy(oral)
def payload(self, rows):
return json.dumps({'count': len(rows), 'next': None, 'previous': None, 'results': rows})
def snapshots(self, folder):
folder.mkdir()
(folder/'program-json.response').write_bytes(RAW)
(folder/'receipts.json').write_text(json.dumps(RECEIPTS))
return folder
def test_full_snapshot_tracks_and_all_titles_authors_affiliations(self):
rows, held, duplicates, folded = program.parse_program(RAW, EVIDENCE)
self.assertEqual((len(rows), len(held), len(duplicates), folded), (9004, 139, 1, 132))
self.assertEqual(Counter(r['track'] for r in rows),
{'main': 7896, 'evaluations_datasets': 971, 'position': 137})
expected = {}
for e in json.loads(RAW)['results']:
if e['sourceurl'] in program.GROUPS and e['event_type'] == 'Poster':
forum = parse_qs(urlsplit(e['paper_url']).query)['id'][0]
expected[forum] = e
self.assertEqual(len(expected), 9004)
for r in rows:
e = expected[r['identifiers']['openreview']]
self.assertEqual(r['title'], e['name'])
self.assertEqual(r['authors'], [a['fullname'] for a in e['authors']])
self.assertEqual(r['metadata']['author_affiliations'], [a.get('institution') for a in e['authors']])
self.assertEqual(r['raw_decision'], e['decision'])
self.assertEqual(r['status'], 'accepted')
self.assertNotIn('decision_date', r)
self.assertIsNone(r['abstract'])
self.assertEqual(r['year'], 2026)
self.assertEqual(duplicates[0]['event_id'], 154866)
def test_same_forum_folds_presentations_and_preserves_signed_source_ids(self):
poster, oral = self.sample()
rows, held, duplicates, folded = program.parse_program(self.payload([oral, poster, oral]), EVIDENCE)
row, = rows
self.assertEqual((len(held), len(duplicates), folded), (0, 1, 1))
self.assertEqual({p['sourceid'] for p in row['metadata']['presentations']},
{poster['sourceid'], oral['sourceid']})
self.assertLess(oral['sourceid'], 0)
self.assertEqual({t['value'] for t in row['tags']}, {'Poster', 'Oral'})
self.assertTrue(all(t['namespace'] == 'neurips_program:presentation_format' for t in row['tags']))
def test_identical_title_and_authors_in_distinct_forums_remain_distinct(self):
poster, _ = self.sample(); other = copy.deepcopy(poster)
other.update(id=999999, uid='0'*32, sourceid=999999, paper_url='https://openreview.net/forum?id=otherForum7',
virtualsite_url='/virtual/2026/poster/999999')
rows, _, _, _ = program.parse_program(self.payload([poster, other]), EVIDENCE)
self.assertEqual(len(rows), 2)
self.assertEqual(len({r['external_id'] for r in rows}), 2)
def test_conflicting_event_ids_or_same_forum_bibliography_are_rejected(self):
poster, oral = self.sample()
for field in ('name', 'authors', 'sourceurl', 'decision'):
changed = copy.deepcopy(oral)
if field == 'authors':
changed['authors'][0]['fullname'] = 'Different Author'
elif field == 'sourceurl':
changed[field] = 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track'
elif field == 'decision':
changed[field] = 'Accept (poster)'
else:
changed[field] += ' changed'
with self.subTest(field=field), self.assertRaises(ValueError):
program.parse_program(self.payload([poster, changed]), EVIDENCE)
changed = copy.deepcopy(poster); changed['name'] += ' changed'
with self.assertRaises(ValueError):
program.parse_program(self.payload([poster, changed]), EVIDENCE)
def test_unreviewed_origin_decision_visibility_and_link_are_held(self):
poster, _ = self.sample()
mutations = [{'sourceurl': 'journal-track/NeurIPS_2026-Posters_TMLR.xlsx'},
{'sourceurl': 'neurips2026-bridging-the-future'},
{'sourceurl': None}, {'sourceurl': 'https://openreview.net/group?id=NeurIPS.cc/2027/Conference'},
{'decision': 'Reject'}, {'visible': False},
{'paper_url': 'https://other.example/forum?id=anotherID7'}]
for mutation in mutations:
changed = copy.deepcopy(poster); changed.update(id=999999, virtualsite_url='/virtual/2026/poster/999999')
changed.update(mutation)
rows, held, _, _ = program.parse_program(self.payload([poster, changed]), EVIDENCE)
self.assertEqual((len(rows), len(held)), (1, 1))
self.assertEqual(held[0]['raw_event'], changed)
def test_rejects_truncation_pagination_invalid_url_and_missing_authors(self):
poster, _ = self.sample()
raw = json.loads(self.payload([poster]))
bad = [dict(raw, count=2), dict(raw, next='https://example.com/next'),
{k: v for k, v in raw.items() if k != 'previous'}]
for payload in bad:
with self.assertRaises(ValueError):
program.parse_program(json.dumps(payload), EVIDENCE)
for mutation in [{'authors': []}, {'virtualsite_url': '/virtual/2025/poster/154865'}, {'id': True}]:
changed = copy.deepcopy(poster); changed.update(mutation)
with self.assertRaises(ValueError):
program.parse_program(self.payload([changed]), EVIDENCE)
def test_frozen_collection_is_partial_and_no_http_or_database_write(self):
with tempfile.TemporaryDirectory() as tmp, patch.object(program, 'fetch', side_effect=AssertionError('No HTTP')):
root = Path(tmp); source = self.snapshots(root/'source')
report = program.collect(root/'out', snapshots=source)
self.assertEqual(report['records'], 9004)
self.assertEqual(len(report['files']), 3)
self.assertFalse(report['database_written'])
self.assertFalse(report['pdfs_downloaded'])
for m in report['files']:
self.assertEqual(m['status'], 'partial')
self.assertEqual(m['missing_abstract_count'], m['count'])
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
with self.assertRaises(ValueError):
program.collect(root/'out', snapshots=source)
def test_frozen_receipt_tamper_path_symlink_status_and_duplicate(self):
for mode in ('tamper', 'path', 'symlink', 'status', 'duplicate'):
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); source = self.snapshots(root/'source')
receipts = copy.deepcopy(RECEIPTS)
if mode == 'tamper':
(source/'program-json.response').write_text('bad')
elif mode == 'path':
receipts['receipts'][0]['file'] = '../outside.json'
elif mode == 'symlink':
(root/'outside.json').write_bytes(RAW)
(source/'program-json.response').unlink()
(source/'program-json.response').symlink_to(root/'outside.json')
elif mode == 'status':
receipts['receipts'][0]['status'] = 403
else:
receipts['receipts'].append(copy.deepcopy(receipts['receipts'][0]))
(source/'receipts.json').write_text(json.dumps(receipts))
with self.assertRaises(ValueError):
program.collect(root/'out', snapshots=source)
self.assertFalse((root/'out/neurips-program-2026.json').exists())
def test_online_mode_only_requests_fixed_program_metadata(self):
poster, _ = self.sample(); raw = self.payload([poster]).encode()
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); cache = root/'cache'; cache.mkdir()
receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': program.URL}
(cache/hashlib.sha256(program.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
with patch.object(program, 'fetch', return_value=raw) as fetch:
report = program.collect(root/'out', cache=cache)
fetch.assert_called_once_with(program.URL, cache, program.LIMIT, delay=1)
self.assertEqual(report['records'], 1)
def test_frozen_fixture_preserves_actual_response_bytes(self):
receipt, = RECEIPTS['receipts']
self.assertEqual(hashlib.sha256(RAW).hexdigest(), receipt['sha256'])
self.assertEqual(len(RAW), receipt['bytes'])
+100
View File
@@ -0,0 +1,100 @@
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import rss2026 as rss
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
class Rss2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
def lists(self,**changes):
pages={**self.pages,**changes}
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
report=rss.collect(self.root/'out',snapshots=self.snap)
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
for row,gold in zip(rows,self.fixture['independent_gold']):
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
self.assertEqual(manifest['missing_abstract_count'],0)
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
self.assertEqual(len(shortened),6)
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
self.assertEqual(len(p['authors']),55)
self.assertFalse(p['program_variants'])
def test_six_reviewed_name_variants_preserve_original_strings(self):
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
self.assertTrue(p['title'].startswith('RLinf-VLA'))
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
def test_unreviewed_author_or_title_variant_stops(self):
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
with self.assertRaisesRegex(ValueError,'Unreviewed'):
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
def test_demo_policy_cannot_be_reused_from2025(self):
with self.assertRaisesRegex(ValueError,'policy'):
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
with self.assertRaises(ValueError):
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
html=self.pages['p001.html']
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
self.assertIn(old,html)
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
def test_index_census_and_preview_tampering_stop(self):
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
self.assertFalse((self.root/'out/collection-report.json').exists())
self.assertTrue((self.root/'out/failed.json').exists())
def test_existing_output_and_unsafe_receipt_are_rejected(self):
out=self.root/'existing';out.mkdir()
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
(self.snap/'receipts.json').write_text(json.dumps(receipts))
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
self.assertTrue((self.root/'unsafe/failed.json').exists())
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
+59
View File
@@ -0,0 +1,59 @@
"""RSS publisher/DBLP observations share one event without losing old decisions."""
from pathlib import Path
import json
import sqlite3
import tempfile
import unittest
from paper_atlas.db.submissions import identities_for_row
from tests.test_aamas_publication_identity import row, ingest
DOI='10.15607/rss.2026.xxii.041'
def rss(source,external,**kwargs):
return row(source,external,venue='RSS',year=2026,track='main',
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
class RssPublicationIdentityTests(unittest.TestCase):
def test_only_verified_accepted_source_adds_publication_identity(self):
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
self.assertEqual(identities_for_row({**r,'status':status}),[])
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
for pair in ((a,b),(b,a)):
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
for r in pair:ingest(db,root,r)
ingest(db,root,a)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
ingest(db,root,old)
with sqlite3.connect(db) as c:
before=c.execute('SELECT * FROM decisions').fetchall()
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
ingest(db,root,new)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
+148
View File
@@ -0,0 +1,148 @@
import gzip
import hashlib
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.audits.source_identity import audit, classify, load_inputs
from paper_atlas.postgres.client import ReadConnection
def record(**updates):
row = dict(source='reviewed_program', external_id='2026/forum-a',
title='A sufficiently specific scientific paper title', authors=['Jane Doe', 'John Smith'],
venue='NeurIPS', year=2026, track='main', status='accepted', identifiers={},
source_record_hits=[], identifier_hits=[], title_candidate_ids=[])
row.update(updates)
return row
def write_input(root, rows):
path = root / 'source.jsonl'
raw = b''.join((json.dumps(row) + '\n').encode() for row in rows)
path.write_bytes(raw)
manifest = {k: rows[0][k] for k in ('source', 'venue', 'year', 'track')}
manifest.update(count=len(rows), file_sha256=hashlib.sha256(raw).hexdigest(),
file='/ignored/untrusted/location.jsonl', status='partial')
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
return path
class CandidateTests(unittest.TestCase):
def test_duplicate_names_do_not_equal_distinct_author_multiset(self):
row = record(authors=['Jane Doe', 'Jane Doe'], title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': ['Jane Doe']}})
self.assertEqual(result['kind'], 'partial_author_overlap_requires_review')
self.assertIn('duplicate_normalized_author_names', result['flags'])
self.assertFalse(result['comparisons'][0]['nonempty_author_multiset_equal'])
def test_arxiv_author_representation_and_order_match_but_never_approve(self):
row = record(title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': [['Smith', 'John', ''], ['Doe', 'Jane', '']]}})
self.assertEqual(result['kind'], 'one_exact_title_author_multiset_requires_review')
self.assertEqual(result['decision'], 'not_approved_for_import')
def test_binding_conflict_overrides_existing_source(self):
row = record(source_record_hits=[{'paper_id': 7}], identifier_hits=[{'paper_id': 8}])
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in (7, 8)}
self.assertEqual(classify(row, entities)['kind'], 'conflicting_existing_bindings_hold')
def test_wrong_doi_binding_retains_author_title_disagreement(self):
row = record(identifier_hits=[{'paper_id': 7}])
result = classify(row, {7: {'title': 'Completely different work', 'authors': ['Somebody Else']}})
self.assertEqual(result['kind'], 'existing_identifier_binding_requires_review')
self.assertFalse(result['comparisons'][0]['normalized_title_equal'])
self.assertEqual(result['comparisons'][0]['author_name_overlap'], [])
def test_no_hit_is_not_novelty_and_missing_authors_never_agree(self):
self.assertEqual(classify(record(), {})['kind'], 'no_exact_match_not_proof_of_novelty')
row = record(authors=[], title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': []}})
self.assertEqual(result['kind'], 'no_author_agreement_hold')
def test_title_candidate_cap_is_explicit(self):
row = record(title_candidate_ids=list(range(21)))
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in range(21)}
self.assertTrue(classify(row, entities)['title_candidates_truncated'])
self.assertEqual(classify(row, entities)['kind'], 'title_candidate_limit_reached_hold')
class InputTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
def test_raw_line_hash_and_manifest_path_not_followed(self):
path = write_input(self.root, [record()])
rows, inputs = load_inputs([path])
self.assertEqual(rows[0]['raw_line_sha256'], hashlib.sha256(path.read_bytes()).hexdigest())
self.assertEqual(inputs[0]['count'], 1)
def test_hash_tamper_rejected_before_database_connection(self):
path = write_input(self.root, [record()])
path.write_bytes(path.read_bytes().replace(b'Jane Doe', b'Jane Roe'))
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
with self.assertRaisesRegex(ValueError, 'hash mismatch'):
audit([path], self.root / 'out', pg_service='test', pg_schema='research')
connection.assert_not_called()
def test_manifest_scope_and_count_tamper_and_duplicate_sources(self):
path = write_input(self.root, [record()])
manifest_path = path.with_suffix('.manifest.json')
manifest = json.loads(manifest_path.read_text())
for key, value in [('year', 2025), ('count', 2)]:
altered = dict(manifest, **{key: value})
manifest_path.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
load_inputs([path])
manifest_path.write_text(json.dumps(manifest))
with self.assertRaisesRegex(ValueError, 'Duplicate source'):
load_inputs([path, path])
def test_existing_output_never_replayed_and_failure_not_completion(self):
path = write_input(self.root, [record()])
out = self.root / 'out'
with patch('paper_atlas.audits.source_identity.ReadConnection', side_effect=RuntimeError('unavailable')):
with self.assertRaises(RuntimeError):
audit([path], out, pg_service='test', pg_schema='research')
self.assertTrue((out / 'failed.json').exists())
self.assertFalse((out / 'postgresql-preflight.json').exists())
with self.assertRaises(FileExistsError):
audit([path], out, pg_service='test', pg_schema='research')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the authorized remote PostgreSQL service')
class ReadOnlyIntegrationTests(unittest.TestCase):
def test_existing_binding_full_evidence_and_no_import_approval(self):
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
connection = ReadConnection(service, 'research')
try:
self.assertEqual(connection.execute('SHOW transaction_read_only').fetchone()[0], 'on')
hit = connection.execute("SELECT value,paper_id FROM identifiers WHERE scheme='arxiv' ORDER BY value LIMIT 1").fetchone()
self.assertIsNotNone(hit)
paper = connection.execute('SELECT title,authors_json FROM papers WHERE id=?', (hit['paper_id'],)).fetchone()
finally:
connection.close()
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
path = write_input(root, [record(title=paper['title'], authors=json.loads(paper['authors_json']),
identifiers={'arxiv': hit['value']})])
report = audit([path], root / 'out', pg_service=service, pg_schema='research')
self.assertTrue(report['transaction_read_only'])
self.assertFalse(report['database_written'])
self.assertEqual(report['approved_for_import'], 0)
row, = report['entries']
self.assertIn(hit['paper_id'], [r['paper_id'] for r in row['identifier_hits']])
shard = report['entities'][str(hit['paper_id'])]
raw = (root / 'out' / shard['file']).read_bytes()
self.assertEqual(hashlib.sha256(raw).hexdigest(), shard['sha256'])
packet = json.loads(gzip.decompress(raw))
self.assertEqual(packet['paper']['id'], hit['paper_id'])
self.assertEqual(packet['paper']['title'], paper['title'])
self.assertEqual(packet['paper']['authors_json'], paper['authors_json'])
self.assertIn('attempts', packet)
self.assertTrue((root / 'out/postgresql-preflight.json').is_file())