Compare commits

...

19 Commits

Author SHA1 Message Date
codex-bot 9093a8ce25 Preserve separate ICAPS registry publication versions and abstracts
Metadata integrity checks / metadata (push) Successful in 2m51s
2026-11-17 00:30:14 +08:00
codex-bot e19cb83cd6 Align ICAPS collector entry point with shared CLI
Metadata integrity checks / metadata (push) Successful in 2m47s
2026-11-17 00:13:30 +08:00
codex-bot 4d2df2a826 Preserve ICAPS 2026 official acceptance and presentation boundaries
Metadata integrity checks / metadata (push) Failing after 2m30s
2026-11-17 00:08:49 +08:00
Zayden-Jung 1cfd44d8b6 Merge pull request '为RSS发表记录保留跨来源的同次投稿身份' (#53) from codex/rss-publication-identity into main
Metadata integrity checks / metadata (push) Successful in 2m48s
Reviewed-on: #53
2026-11-16 08:13:30 +00:00
Zayden-Jung 5de7f6ff22 Merge pull request '封装RSS2026官方论文集采集与来源差异' (#52) from codex-bot/rss2026 into main
Metadata integrity checks / metadata (push) Successful in 3m1s
Reviewed-on: #52
2026-11-16 08:13:16 +00:00
codex-bot b4e249e675 Recognize accepted RSS publisher DOI as an edition-scoped publication identity
Metadata integrity checks / metadata (push) Successful in 2m55s
2026-11-15 23:07:11 +08:00
codex-bot 93f00d1a64 Preserve failed-run checkpoint when snapshot receipts are invalid
Metadata integrity checks / metadata (push) Successful in 2m44s
2026-11-15 22:47:27 +08:00
codex-bot 8b7bc6d871 Include frozen RSS 2026 source fixture and independent 210-record inventory
Metadata integrity checks / metadata (push) Successful in 2m54s
2026-11-15 22:43:21 +08:00
codex-bot 7588a4e904 Collect reviewed RSS 2026 publisher metadata with separate program variants 2026-11-15 22:43:05 +08:00
Zayden-Jung 891a63d982 Merge pull request '封装来源身份只读预检与投稿历史证据' (#51) from codex/source-identity-audit into main
Metadata integrity checks / metadata (push) Successful in 2m49s
Reviewed-on: #51
2026-11-10 06:55:25 +00:00
codex-bot fe3e2502a5 Add read-only source identity candidate audit with evidence shards
Metadata integrity checks / metadata (push) Successful in 2m42s
2026-11-10 02:55:27 +08:00
Zayden-Jung b675a38b97 Merge pull request '适配AAMAS2026官方目录并保留撤稿与撤回证据' (#49) from codex/aamas2026 into main
Metadata integrity checks / metadata (push) Successful in 2m45s
Reviewed-on: #49
2026-11-09 18:49:37 +00:00
Zayden-Jung cb875b8eef Merge pull request '封装NeurIPS2026节目采集,分离投稿与重复展示' (#50) from codex/neurips-program into main
Metadata integrity checks / metadata (push) Successful in 2m34s
Reviewed-on: #50
2026-11-09 18:49:29 +00:00
codex-bot 57a557c66b Merge current main and preserve scope and AAMAS documentation entries
Metadata integrity checks / metadata (push) Successful in 2m27s
2026-11-10 01:12:45 +08:00
codex-bot 5b5e606f0c Correct independent affiliation inventory and target section heading mutation
Metadata integrity checks / metadata (push) Successful in 2m22s
2026-11-10 01:09:25 +08:00
codex-bot 9f84fd7caf Separate italic title content from publisher status notices
Metadata integrity checks / metadata (push) Failing after 2m6s
2026-11-10 01:04:46 +08:00
codex-bot bf05d3cdc8 Keep day-only withdrawal evidence through native ingestion
Metadata integrity checks / metadata (push) Failing after 2m6s
2026-11-10 01:02:54 +08:00
codex-bot 930df2a4d5 Preserve explicit withdrawal and byte-exact AAMAS fixture evidence
Metadata integrity checks / metadata (push) Failing after 2m4s
2026-11-10 01:00:58 +08:00
codex-bot 37380e1d3b Collect reviewed AAMAS 2026 tracks and hold explicit retraction evidence
Metadata integrity checks / metadata (push) Failing after 2m10s
2026-11-10 00:57:09 +08:00
26 changed files with 1818 additions and 6 deletions
+38
View File
@@ -48,6 +48,33 @@ jobs:
outside.mkdir()
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
icaps_snapshots = root/'icaps2026-snapshots'
icaps_snapshots.mkdir()
for name, html in icaps_fixture['pages'].items():
(icaps_snapshots/name).write_text(html)
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
registry_snapshots = root/'icaps2026-registry-snapshots'
registry_snapshots.mkdir()
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
rss_snapshots = root/'rss2026-snapshots'
rss_snapshots.mkdir()
for name, html in rss_fixture['pages'].items():
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
@@ -62,6 +89,16 @@ jobs:
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
aamas2026_snapshots = root/'aamas2026-snapshots'
aamas2026_snapshots.mkdir()
(aamas2026_snapshots/'contents.html').write_text(aamas2026['contents'])
(aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True)
aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text())
assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1
assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1}
assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded']
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
@@ -86,6 +123,7 @@ jobs:
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
+10
View File
@@ -8,6 +8,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
AAMAS 2026 的官方 HTML 目录由 `collect aamas-proceedings --year 2026` 适配,区分 track 和发表形式,keynote 排除、撤稿证据单独保留;[采集与恢复说明](docs/aamas-proceedings.md)。采集不写数据库,仍需身份核验和增量导入。
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
`collect springer-selected` 核验限定的会后选集,分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集,原始会议决定仍须单独核查;配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
@@ -114,3 +116,11 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
+26 -1
View File
@@ -1,6 +1,31 @@
# AAMAS 官方论文集元数据
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025、2026** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025、2026 解析分别封装在 `paper_atlas.collectors.aamas2025`、`paper_atlas.collectors.aamas2026`。
## AAMAS 2026
2026 只读取[官方 HTML 目录](https://www.ifaamas.org/Proceedings/aamas2026/forms/contents.htm),不下载 PDF。冻结目录 SHA256 `ce2bc7fc5da340930343901506a9ce37c0334d4068f07546431c9deee3f13de9`,643 个条目:keynote 3、research 338、extended abstract 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3。
排除 3 个 keynote;research 的 p238 明确标注 `Retracted June 8, 2026`,完整题名、作者、栏目、日期、原文和来源回执保存在采集报告 `held_publications` 中,**不会输出 accepted 或 withdrawn 投稿记录**。当前决定模型没有独立 retracted 状态,后续须单独建模和核验;原始证据不能删除。任何新增或改变的撤稿标记都会停止采集,等待人工核查。
另一个 extended abstract p3178 明确标注 `Withdrawn June 8, 2026`,输出 `withdrawn`,在 metadata 保留日期 `2026-06-08` 和日精度;原网页没有时区/时刻,不能伪造决定 timestamp,不补造其此前的接受决定或 PDF 链接。因此 639 个输出条目是 **638 accepted、1 withdrawn**,按六个 track 分组:`proceedings_unspecified` 522、`aaai_transfer` 19、`blue_sky` 17、`doctoral` 34、`demo` 44、`jaamas` 3。全文/扩展摘要是发表形式,不是研究类型;research 栏目不能证明每篇原始 main/special 投稿轨道。JAAMAS 保留期刊展示语义,不制造另一次独立审稿录用。
[AAAI 转投详细政策](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 要求基于 AAAI-26 rejected 稿件;[总览](https://cyprusconferences.org/aamas2026/calls/) 却写 AAAI-25,须保留冲突。采集器保留 AAAI 栏目归属,不据此生成原 AAAI 投稿 ID、年份或逐篇拒稿决定。目录不是原始决定 note,未来关联仍需逐篇证据。
```sh
paper-atlas collect aamas-proceedings --year 2026 \
--snapshots /data/paper-index/collected/aamas2026-primary-evidence-v1 \
--out /data/paper-index/collected/aamas2026-normalized-v1
```
在线模式将 `--snapshots` 换成 `--cache /data/paper-index/cache/aamas2026`。冻结模式 `receipts.json` 的 `receipts` 至少包含官方目录 URL、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区 `finished_at`。拒绝越界路径、越界符号链接、失败状态、哈希/大小不符的页面。这里只读取目录对应回执,其他已冻结政策网页继续保留。
输出六组 JSONL/manifest、原始 HTML 与最后发布的 `aamas-proceedings-2026.json`;统一入口沿用输出目录锁和暂存目录。manifest 全部为 `partial`,639 条 HTML 摘要缺失如实记录。发布编号使用年份和起始页,随机 PDF 文档 ID 单独保留,不能沿用 2025 的 pN.pdf 规则。
首次部署先核对来源哈希和生产状态,再在新输出目录运行采集、独立比对 643 条标题/作者/栏目分区,随后做 PostgreSQL 身份预检与增量导入。采集、远程 CI 通过或目录含某个年份均不代表生产已导入、摘要齐全或最终去重完成。较早年份不自动启动补采,当前主动补漏范围按用户要求从 2020 年起。
## AAMAS 2025
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
+6
View File
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
### RSS 发表身份
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
+40
View File
@@ -0,0 +1,40 @@
# ICAPS 2026 官方名单
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
```sh
paper-atlas collect icaps2026-program \
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
--out /data/paper-index/collected/icaps2026-normalized-v1
```
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
## 独立注册元数据版本
```sh
paper-atlas collect icaps2026-registry \
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
```
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
+21
View File
@@ -0,0 +1,21 @@
# RSS 2026 论文集来源
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
```sh
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
--out /data/paper-index/collected/rss2026-native-v1
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
--out /data/paper-index/collected/rss2026-native-replay-v1
```
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
+21
View File
@@ -0,0 +1,21 @@
# 来源身份只读预检
`audit source-identities` 在 PostgreSQL 的同一 `REPEATABLE READ READ ONLY` 快照中,检查规范化 JSONL 的来源绑定、稳定标识及精确标题候选,并导出候选论文的原始来源和投稿决定历史。用于入库前评审,不执行导入、合并或标识修改。
```sh
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit source-identities \
--pg-service foundry-research --pg-schema research \
--files /data/paper-index/collected/example/main.jsonl \
/data/paper-index/collected/example/position.jsonl \
--out /data/paper-index/collected/example-identity-preflight-v1
```
每个 JSONL 必须有同名 `.manifest.json`,包含 `source`、`venue`、`year`、`track`、`count` 和 `file_sha256`。逐行校验届次、显式决定状态、标识类型和跨文件来源主键唯一性;文件哈希和数量必须相符。只读取调用者指定的文件,不跟随 manifest 中的绝对 `file` 路径;原始行哈希包含换行符。原始 JSONL 和 manifest 应与报告一起保留。
输出目录必须不存在,防止复用旧报告或覆盖证据。失败会保留 `failed.json` 及已完成分片,不能把部分输出当成功;排查后使用新的输出目录重新核查。成功报告 `postgresql-preflight.json` 最后原子写入,记录输入哈希、数据库快照和每条候选。实体详情存为 `entities/<分片>/<paper_id>.json.gz`,报告中列出压缩文件的 SHA-256;包含完整 paper、标识和限量原始来源、投稿、决定及出处。每篇最多保留 100 条来源、200 次投稿,每次投稿最多 200 条决定,每组决定或投稿出处最多 100 条,所有截断均显式标记。精确标题最多取 21 个候选;达到上限即标记,不能据此推算总数。这里没有计算全表精确数量。
候选分类用于安排审核:既有来源记录需要核对并避免重放,多个绑定冲突必须暂缓;单个 DOI、arXiv 或 OpenReview 绑定也可能错误,必须核对原文、标题、作者和历史。作者比较保留完整规范化姓名的重复次数,但姓名相同不能证明是同一个人。未找到精确标题或标识不能证明是新作品,仍需近似标题、版本和跨会历史核查。
所有条目固定为 `not_approved_for_import`,批准数量为 0。不要直接把此报告当成 curation 计划或导入队列。后续获批导入前应重新检查实时来源绑定;来源摘要缺失不得覆盖已有摘要,旧拒稿与新录用应作为不同投稿经历保留。本命令不能代替所有导入结束后的全库去重、覆盖和约束验收。
构建与测试仅在已确认的 `ces-control:host` 远程 runner 执行。集成测试使用已有 PostgreSQL 服务做真实只读候选查询,不往生产库添加测试记录。
+252
View File
@@ -0,0 +1,252 @@
"""Read-only, evidence-preserving identity candidates for normalized source files.
A candidate is never permission to import or merge. Absence of an exact match is
never proof of a new work. Run again against a fresh snapshot before curation.
"""
import argparse
from collections import Counter
import gzip
import hashlib
import json
from pathlib import Path
from paper_atlas.db.store import author_keys, normalized
from paper_atlas.db.submissions import timestamp
from paper_atlas.postgres.client import ReadConnection
STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'}
def digest(data):
return hashlib.sha256(data).hexdigest()
def authors_multiset(authors):
# author_keys intentionally removes duplicates; identity review must retain them.
return Counter(key for author in authors for key in author_keys([author]))
def load_inputs(files):
records, inputs, seen = [], [], set()
for filename in files:
path = Path(filename).resolve()
manifest_path = path.with_suffix('.manifest.json')
raw_manifest = manifest_path.read_bytes()
manifest = json.loads(raw_manifest)
hasher, count = hashlib.sha256(), 0
with path.open('rb') as stream:
for line_number, line in enumerate(stream, 1):
hasher.update(line)
row = json.loads(line)
for key in ('source', 'external_id', 'title', 'venue', 'track'):
if not isinstance(row.get(key), str) or not row[key].strip():
raise ValueError(f'Invalid {key}: {path.name}:{line_number}')
if type(row.get('year')) is not int or row.get('status') not in STATUSES:
raise ValueError('Invalid year or explicit source status')
for key in ('source', 'venue', 'year', 'track'):
if row[key] != manifest.get(key):
raise ValueError(f'Manifest scope mismatch: {key}')
authors = row.get('authors')
if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors):
raise ValueError('Invalid authors')
identifiers = row.get('identifiers', {})
if not isinstance(identifiers, dict) or any(
not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip()
for k, v in identifiers.items()):
raise ValueError('Identifiers must contain singular nonempty strings')
key = (row['source'], row['external_id'])
if key in seen:
raise ValueError('Duplicate source/external_id across inputs')
seen.add(key)
record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors',
'venue', 'year', 'track', 'status')}
record.update(identifiers=identifiers, file=str(path), line=line_number,
raw_line_sha256=digest(line), source_record_hits=[],
identifier_hits=[], title_candidate_ids=[])
records.append(record)
count += 1
if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'):
raise ValueError('Input count/hash mismatch or empty input')
inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count,
'manifest_sha256': digest(raw_manifest), 'manifest': manifest})
if not records:
raise ValueError('At least one nonempty input is required')
return records, inputs
def _lookup(connection, values, columns, lateral):
"""Bound scalar VALUES allow the existing text codec to encode each key."""
width = len(columns)
for start in range(0, len(values), 100):
batch = values[start:start + 100]
sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join(
'(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true'
for row in connection.execute(sql, tuple(value for item in batch for value in item)):
yield dict(row)
def find_candidates(connection, records):
source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)]
for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), '''
SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id
WHERE s.name=keys.source AND r.external_id=keys.external_id'''):
records[hit.pop('n')]['source_record_hits'].append(hit)
identifiers = []
for i, row in enumerate(records):
pairs = {(row['source'], row['external_id']), *row['identifiers'].items()}
identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs))
for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), '''
SELECT scheme,value,paper_id FROM identifiers
WHERE scheme=keys.scheme AND value=keys.value'''):
records[hit.pop('n')]['identifier_hits'].append(hit)
title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)]
for hit in _lookup(connection, title_keys, ('n', 'title_key'), '''
SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''):
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
def entity_packet(connection, paper_id):
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
if paper is None:
raise RuntimeError('Candidate disappeared from read-only snapshot')
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
FROM source_records r JOIN sources s ON s.id=r.source_id
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
for attempt in packet['attempts']:
observations = [dict(r) for r in connection.execute(
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
for decision in attempt['decisions']:
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
return packet
def classify(record, entities):
anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']}
titles = record['title_candidate_ids']
flags = []
own_authors = authors_multiset(record['authors'])
if not own_authors:
flags.append('missing_source_authors')
if any(n > 1 for n in own_authors.values()):
flags.append('duplicate_normalized_author_names')
if len(normalized(record['title'])) < 25:
flags.append('short_title')
comparisons = []
for pid in sorted(anchors | set(titles)):
entity = entities[pid]
other = authors_multiset(entity['authors'])
comparisons.append({'paper_id': pid,
'normalized_title_equal': normalized(record['title']) == normalized(entity['title']),
'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other,
'author_name_overlap': sorted(own_authors.keys() & other.keys())})
if len(anchors) > 1:
kind = 'conflicting_existing_bindings_hold'
elif record['source_record_hits']:
kind = 'existing_source_record_reconcile_no_replay'
elif anchors:
kind = 'existing_identifier_binding_requires_review'
elif len(titles) >= 21:
kind = 'title_candidate_limit_reached_hold'
elif not titles:
kind = 'no_exact_match_not_proof_of_novelty'
elif len(titles) > 1:
kind = 'multiple_exact_title_entities_hold'
elif comparisons[0]['nonempty_author_multiset_equal']:
kind = 'one_exact_title_author_multiset_requires_review'
elif comparisons[0]['author_name_overlap']:
kind = 'partial_author_overlap_requires_review'
else:
kind = 'no_author_agreement_hold'
return {'kind': kind, 'flags': flags, 'comparisons': comparisons,
'title_candidates_truncated': len(titles) >= 21,
'decision': 'not_approved_for_import'}
def _atomic_json(path, value):
temporary = path.with_suffix(path.suffix + '.part')
with temporary.open('w', encoding='utf-8') as stream:
json.dump(value, stream, ensure_ascii=False, indent=2)
stream.write('\n')
temporary.replace(path)
def audit(files, out, *, pg_service, pg_schema):
# Reject input corruption before opening a database connection or reserving output.
records, inputs = load_inputs(files)
out = Path(out)
out.mkdir(parents=True, exist_ok=False)
(out / 'audit.lock').write_text(timestamp() + '\n')
connection = None
try:
connection = ReadConnection(pg_service, pg_schema)
connection.execute("SET LOCAL statement_timeout='60s'")
if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on':
raise RuntimeError('A read-only transaction is required')
snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema,
transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone())
find_candidates(connection, records)
ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} |
{pid for row in records for pid in row['title_candidate_ids']})
entities, index = {}, {}
for number, pid in enumerate(ids, 1):
packet = entity_packet(connection, pid)
relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz'
target = out / relative
target.parent.mkdir(parents=True, exist_ok=True)
raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0)
target.write_bytes(raw)
index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)}
entities[pid] = {'title': packet['paper']['title'],
'authors': json.loads(packet['paper']['authors_json'])}
if number % 250 == 0 or number == len(ids):
print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True)
kinds = Counter()
for row in records:
row.update(classify(row, entities))
kinds[row['kind']] += 1
report = {'status': 'completed_read_only_candidates_not_import_approval',
'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True,
'database_written': False, 'approved_for_import': 0, 'records': len(records),
'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records,
'limitations': ['Exact keys only; no match does not establish a new work.',
'Name agreement does not prove person or scientific-work identity.',
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
'Source/history limits are explicit in compressed entity packets.',
'Not a final whole-database deduplication or coverage audit.']}
_atomic_json(out / 'postgresql-preflight.json', report)
return report
except Exception as error:
_atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval',
'error_type': type(error).__name__, 'database_written': False})
raise
finally:
if connection is not None:
connection.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--files', nargs='+', required=True, type=Path)
parser.add_argument('--pg-service', required=True)
parser.add_argument('--pg-schema', required=True)
parser.add_argument('--out', required=True, type=Path)
args = parser.parse_args(argv)
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
+4
View File
@@ -23,6 +23,9 @@ COMMANDS = {
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
@@ -46,6 +49,7 @@ COMMANDS = {
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
}
+6 -3
View File
@@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None):
def _collect(year, out, *, cache=None, snapshots=None):
if year == 2026:
from paper_atlas.collectors.aamas2026 import collect as collect_2026
return collect_2026(out, cache=cache, snapshots=snapshots)
if year == 2025:
from paper_atlas.collectors.aamas2025 import collect as collect_2025
return collect_2025(out, cache=cache, snapshots=snapshots)
@@ -193,8 +196,8 @@ def _collect(year, out, *, cache=None, snapshots=None):
def collect(year, out, *, cache=None, snapshots=None):
if year not in (2012, 2025):
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
if year not in (2012, 2025, 2026):
raise ValueError('Only independently reviewed AAMAS 2012/2025/2026 layouts are supported')
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'aamas.collect.lock').open('a') as lock:
@@ -204,7 +207,7 @@ def collect(year, out, *, cache=None, snapshots=None):
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
parser.add_argument('--year', type=int, choices=[2012, 2025, 2026], required=True)
parser.add_argument('--out', required=True)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument('--cache')
+219
View File
@@ -0,0 +1,219 @@
"""Reviewed AAMAS 2026 HTML metadata; retractions stay outside accepted queues."""
from collections import Counter, defaultdict
import hashlib
import json
from pathlib import Path
import re
import tempfile
from paper_atlas.collectors.aamas2025 import one
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2026/'
URL = ROOT+'forms/contents.htm'
TITLE = 'AAMAS 2026: Proceedings of the 2026 International Conference on Autonomous Agents and Multiagent Systems'
SECTIONS = {
'K': ('Keynote Talks', 'keynote', 'keynote', 3),
'R': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 338),
'1': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 185),
'A': ('AAAI Track', 'full_paper', 'aaai_transfer', 19),
'B': ('Blue Sky Ideas Track', 'blue_sky_paper', 'blue_sky', 17),
'C': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 34),
'D': ('Demonstration Track', 'demo', 'demo', 44),
'J': ('JAAMAS Track', 'journal_presentation', 'jaamas', 3),
}
PDF = re.compile(r'\.\./pdfs/([A-Z]{4}[0-9]{4})\.pdf')
RETRACTION_TITLE = 'Unifying Model-Free Efficiency and Model-Based Representations via Latent Dynamics'
WITHDRAWAL_TITLE = 'Cognitive Heterogeneity and Behavioral Biases in Multi-Stage Supply Chains: Evidence from LLM-Based Agent Simulations'
def parse_authors(paragraph):
font = one([n for n in paragraph.walk() if n.tag == 'font' and
n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
authors, affiliations = [], []
name, affiliation = '', ''
def flush():
nonlocal name, affiliation
if clean(name) or clean(affiliation):
if not clean(name):
raise ValueError('Affiliation without author')
authors.append(clean(name)); affiliations.append(clean(affiliation))
name, affiliation = '', ''
for child in font.children:
if isinstance(child, str):
name += child
elif child.tag == 'br':
flush()
elif child.tag == 'i':
affiliation += child.text()
else:
raise ValueError('Unreviewed author markup: '+child.tag)
flush()
if not authors:
raise ValueError('Empty author list')
return authors, affiliations
def parse_contents(html, evidence):
doc = Document(html)
if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
raise ValueError('Unexpected AAMAS proceedings title/year')
section = None
headers, records, excluded, held = [], [], [], []
counts, seen_pages, seen_pdf_ids = Counter(), set(), set()
for node in doc.root.walk():
if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
continue
anchors = [n.attrs['name'] for n in node.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
if anchors:
section = one(anchors, 'section anchor')
if clean(node.text()) != SECTIONS[section][0]:
raise ValueError('Publisher section heading differs from reviewed layout')
headers.append(section)
page_fonts = [n for n in node.walk() if n.tag == 'font' and
n.attrs.get('color', '').lower() == '#444444']
if not page_fonts:
continue
page_text = clean(one(page_fonts, 'page marker').text())
page_match = re.fullmatch(r'\(Page ([1-9][0-9]*)\)', page_text)
if not page_match or section not in SECTIONS:
raise ValueError('Invalid page or paper outside reviewed sections')
page = int(page_match[1])
if page in seen_pages:
raise ValueError('Repeated publisher page ID')
seen_pages.add(page)
title_node = one([n for n in node.walk() if n.tag == 'strong'], 'paper title')
title = clean(title_node.text())
if not title:
raise ValueError('Empty paper title')
authors, affiliations = parse_authors(node)
link = one([n for n in node.walk() if n.tag == 'a' and 'href' in n.attrs], 'paper link')
href = link.attrs['href']
title_nodes = {id(n) for n in title_node.walk()}
retractions = [clean(n.text()) for n in link.walk() if n.tag == 'i' and id(n) not in title_nodes]
heading, publication_format, track, _ = SECTIONS[section]
counts[section] += 1
metadata = {'publisher_page': page, 'section_url': URL, 'section_anchor': section,
'section_heading': heading, 'publication_format': publication_format,
'author_affiliations': affiliations, 'source_evidence': evidence,
'decision_date_precision': 'not_supplied',
'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
'publication_evidence': 'Published proceedings entry, not an original decision note.'}
withdrawn = retractions == ['Withdrawn June 8, 2026']
if withdrawn:
if (section, page, title, href) != ('1', 3178, WITHDRAWAL_TITLE, ''):
raise ValueError('Unreviewed withdrawal notice')
metadata.update({'publication_state': 'withdrawn',
'withdrawal_date': '2026-06-08',
'raw_publication_state': retractions[0],
'decision_date_precision': 'day',
'publication_evidence': 'Explicit dated publisher withdrawal notice, not an original review decision note.'})
elif retractions:
if (section, page, title, href, retractions) != ('R', 238, RETRACTION_TITLE, '', ['Retracted June 8, 2026']):
raise ValueError('Unreviewed retraction or publication status markup')
held.append({'source': 'aamas_proceedings', 'external_id': '2026/p238',
'title': title, 'authors': authors, 'year': 2026, 'track': track,
'url': URL+'#R', 'abstract': None, 'identifiers': {},
'observed_at': evidence['observed_at'], 'metadata': metadata,
'publication_state': 'retracted', 'retraction_date': '2026-06-08',
'raw_publication_state': retractions[0],
'hold_reason': 'Explicit retraction; not an accepted/withdrawn submission assertion. Dedicated decision modeling and identity review required.'})
continue
if not withdrawn:
match = PDF.fullmatch(href)
if not match or match[1] in seen_pdf_ids:
raise ValueError('Invalid or repeated publisher PDF ID')
seen_pdf_ids.add(match[1])
metadata.update({'publisher_document_id': match[1], 'paper_url': ROOT+'pdfs/'+match[1]+'.pdf'})
if section == 'K':
excluded.append({'publisher_page': page, 'title': title, 'authors': authors,
'metadata': metadata, 'reason': 'Publisher keynote, not a research submission.'})
continue
metadata['track_scope_note'] = ('Published format known; individual main/special submission track not established.'
if track == 'proceedings_unspecified' else 'Explicit publisher section.')
if section == 'A':
metadata['prior_submission_note'] = 'AAAI transfer section does not supply original submission IDs or individual prior decision notes; none inferred.'
if section == 'J':
metadata['track_scope_note'] = 'Journal presentation; not another independent journal review or research-track acceptance.'
records.append({'source': 'aamas_proceedings', 'external_id': '2026/p'+str(page),
'url': URL+'#'+section, 'title': title, 'abstract': None,
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
'year': 2026, 'track': track, 'status': 'withdrawn' if withdrawn else 'accepted',
'observed_at': evidence['observed_at'],
'raw_decision': retractions[0] if withdrawn else 'listed in official AAMAS 2026 published proceedings '+heading,
'metadata': metadata,
'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]})
if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
raise ValueError('Publisher section inventory differs from reviewed 2026 layout')
if (len(records), len(excluded), len(held)) != (639, 3, 1):
raise ValueError('Unexpected accepted/keynote/retraction partition')
if Counter(r['status'] for r in records) != {'accepted': 638, 'withdrawn': 1}:
raise ValueError('Unexpected publisher publication-state partition')
return records, excluded, held
def read_page(cache, snapshots):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one cache or frozen snapshot directory')
if snapshots:
base = Path(snapshots).resolve()
frozen = json.loads((base/'receipts.json').read_text())['receipts']
r = one([r for r in frozen if r['url'] == URL], 'frozen response receipt')
filename = Path(r['file'])
if (filename.is_absolute() or '..' in filename.parts or
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
raise ValueError('Unsafe or unsuccessful publisher response')
data = (base/filename).read_bytes()
if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
raise ValueError('Frozen publisher response checksum/size mismatch')
observed = r['finished_at']
else:
data = fetch(URL, cache, 8*1024**2, delay=1)
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
observed = r['retrieved_at']
timestamp(observed)
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
def collect(out, *, cache=None, snapshots=None):
data, evidence = read_page(cache, snapshots)
records, excluded, held = parse_contents(data.decode('utf-8'), evidence)
grouped = defaultdict(list)
for row in records:
grouped[row['track']].append(row)
out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
with tempfile.TemporaryDirectory(prefix='.aamas2026-', dir=out) as temporary:
stage = Path(temporary); (stage/'evidence').mkdir()
(stage/evidence['file']).write_bytes(data)
for track, group in sorted(grouped.items()):
p = stage/f'aamas-proceedings-2026-{track}.jsonl'
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': 'aamas_proceedings', 'parser_version': 3, 'venue': 'AAMAS', 'year': 2026,
'track': track, 'count': len(group), 'file': str(out/p.name),
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
'status': 'partial', 'missing_abstract_count': len(group),
'status_counts': dict(Counter(r['status'] for r in group)),
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
'note': 'Reviewed TOC partition excluding keynote/retracted entries; identity and full submission history remain unverified.'}
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': 'aamas_proceedings', 'year': 2026, 'finished_at': timestamp(),
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
'excluded_publications': excluded, 'held_publications': held,
'status_counts': dict(Counter(r['status'] for r in records)),
'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False,
'note': 'Retraction evidence held separately; publication is not an original decision note, and absence is not rejection.'}
report = 'aamas-proceedings-2026.json'
(stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
(out/'evidence').mkdir(exist_ok=True)
for p in (stage/'evidence').iterdir():
p.replace(out/'evidence'/p.name)
for p in stage.iterdir():
if p.is_file() and p.name != report:
p.replace(out/p.name)
(stage/report).replace(out/report)
return result
+155
View File
@@ -0,0 +1,155 @@
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
from pathlib import Path
from paper_atlas.io.html import Document, Node, clean
SOURCE = 'icaps_official_program'
ROOT = 'https://icaps26.icaps-conference.org/'
PAGES = {
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
}
MAIN = 'Papers accepted to the Main Tracks'
PREVIOUS = 'Previously Published Papers'
TRACK = 'main_tracks_unclassified'
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def authors_from_literal(literal):
# This frozen listing uses commas between authors and a final "and".
# Preserve the complete literal too; never infer person-name aliases.
if literal.count(' and ') != 1 or ', and ' in literal:
raise ValueError('Unreviewed author-list punctuation')
first, last = literal.rsplit(' and ', 1)
authors = first.split(', ') + [last]
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
raise ValueError('Incomplete or ambiguous author list')
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
raise ValueError('Author literal cannot be reconstructed')
return authors
def parse_listing(raw):
nodes = list(Document(raw).root.walk())
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
if headings != ['List of Papers Accepted to ICAPS 2026']:
raise ValueError('Unexpected official conference edition')
section = None; sections = []; rows = []; seen = set()
for n in nodes:
if n.tag == 'h2':
section = clean(n.text()); sections.append(section)
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
continue
if section not in (MAIN, PREVIOUS):
raise ValueError('Paper belongs to an unreviewed section')
# Require the reviewed title and complete-author blocks; fail on layout drift.
blocks = [b for b in n.children if isinstance(b, Node)]
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
raise ValueError('Changed paper metadata layout')
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
raise ValueError('Changed paper title/link layout')
title = clean(titles[0].text()); literal = clean(blocks[1].text())
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
raise ValueError('Empty or repeated paper title')
seen.add(title.casefold())
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
'authors_literal': literal})
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
raise ValueError('Incomplete reviewed official listing')
return rows
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
root = Path(snapshots)
manifest_path = root/'manifest.json'
if manifest_path.is_symlink():
raise ValueError('Snapshot symlinks rejected')
receipt = json.loads(manifest_path.read_bytes())
observed = receipt['at']
if (receipt.get('status') != 'collected_not_parsed_not_imported'
or receipt.get('database_written') is not False
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
raise ValueError('Invalid reviewed snapshot receipt')
receipts = receipt['sources']
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
raise ValueError('Unexpected snapshot source set')
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
for name, (url, expected) in PAGES.items():
matches = [r for r in receipts if r['file'] == name]
if len(matches) != 1:
raise ValueError('Repeated snapshot receipt')
r = matches[0]; p = root/name
if p.is_symlink() or p.stat().st_size > 2*1024**2:
raise ValueError('Unsafe or oversized snapshot')
raw = p.read_bytes()
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
raise ValueError('Reviewed snapshot hash changed')
if r['url'] != url or r['final_url'] != url:
raise ValueError('Unexpected snapshot URL or redirect')
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
bodies[name] = raw.decode('utf-8')
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
for entry in entries:
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
if entry['section'] == PREVIOUS:
previous.append({**entry, 'entry_key': entry_key,
'presentation_kind': 'previously_published_research',
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
continue
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
'metadata': {'official_listing': entry, 'source_evidence': evidence,
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
'abstract_availability': 'not_on_official_listing',
'original_decision_signature_verified': False,
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
'status': 'partial', 'missing_abstract_count': 90}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
sidecar = out/'previously-published-presentations.json'
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
@@ -0,0 +1,131 @@
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
import argparse
from datetime import datetime, date
import hashlib
from html import unescape
import json
from pathlib import Path
import re
from paper_atlas.io.html import clean
SOURCE = 'crossref_icaps_registry'
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
TRACK = 'main_tracks_unclassified'
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def abstract_text(raw):
if not isinstance(raw, str):
raise ValueError('Missing registry abstract')
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
if not match or re.search(r'<[^>]*>', match[1]):
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
text = clean(unescape(match[1]))
if not text:
raise ValueError('Empty registry abstract')
return text
def registry_papers(snapshot):
observed = snapshot['at']
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
or snapshot.get('database_written') is not False
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
raise ValueError('Unreviewed registry snapshot receipt')
papers = []; excluded = []; seen = set()
for row in snapshot['records']:
doi = row.get('DOI')
if not isinstance(doi, str) or doi in seen:
raise ValueError('Missing or repeated registry DOI')
seen.add(doi)
if row.get('volume') == '35':
excluded.append(doi)
continue
if (row.get('volume') != '36' or row.get('issue') != '1'
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
or row.get('URL') != 'https://doi.org/'+doi
or not isinstance(row.get('title'), list) or len(row['title']) != 1
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
raise ValueError('Unreviewed publication volume, DOI, type or title')
day, = row['published']['date-parts']
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
raise ValueError('Unreviewed publication date')
publication_date = date(*day).isoformat()
authors = row.get('authors')
if (not isinstance(authors, list) or not authors
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
or not a.get('given') or not a.get('family') for a in authors)):
raise ValueError('Incomplete registry authors')
names = [clean(a['given']+' '+a['family']) for a in authors]
if (len(set(n.casefold() for n in names)) != len(names)
or any('@' in n or 'anonymous' in n.casefold() for n in names)
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
raise ValueError('Unreviewed registry author order or identity')
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
if len(papers) != 90 or len(excluded) != 1:
raise ValueError('Incomplete reviewed volume census')
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
path = Path(snapshots)/'crossref-filtered.json'
if path.is_symlink() or path.stat().st_size > 4*1024**2:
raise ValueError('Unsafe or oversized registry snapshot')
raw = path.read_bytes()
if digest(raw) != SNAPSHOT_SHA:
raise ValueError('Reviewed registry snapshot hash changed')
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
for paper in papers:
original = paper['original']; doi = original['DOI']
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
'metadata': {'registry_fields': original, 'source_evidence': evidence,
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
+222
View File
@@ -0,0 +1,222 @@
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
import argparse
from datetime import datetime, date
import hashlib
import json
from pathlib import Path
import re
import unicodedata
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'rss_proceedings'
ROOT = 'https://www.roboticsproceedings.org/rss22/'
HEADING = 'Robotics: Science and Systems XXII'
PAGES = {'index.html': ROOT,
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def one(values):
if len(values) != 1:
raise ValueError('Expected one reviewed HTML field')
return values[0]
def comparable(value):
return unicodedata.normalize('NFC', clean(value))
def parse_lists(index_html, program_html, policy_html, sponsors_html):
policy = clean(Document(policy_html).root.text())
for text in ('RSS 2026 will not have a separate category for demo papers.',
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
if text not in policy:
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
raise ValueError('Sponsor activity page changed; never classify these as papers')
nodes = list(Document(index_html).root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
raise ValueError('Unexpected publisher edition')
index = {}
for row in (n for n in nodes if n.tag == 'tr'):
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
if not links:
continue
link = one(links); pid = int(link.attrs['href'][1:4])
if pid in index:
raise ValueError('Repeated publisher ID')
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
program = {}
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
raise ValueError('Unexpected accepted program')
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
if not cells:
continue
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
raise ValueError('Unexpected program row')
pid = int(clean(cells[0].text()))
if pid in program:
raise ValueError('Repeated program ID')
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
if not preview or not full.startswith(preview):
raise ValueError('Author preview is not a prefix of complete author evidence')
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
if links != [f'/2026/program/papers/{pid}/']:
raise ValueError('Program display link changed')
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
'authors_preview': preview, 'session': clean(cells[1].text()),
'program_url': 'https://roboticsconference.org'+links[0]}
if set(index) != set(program) or set(index) != set(range(1, 211)):
raise ValueError('Reviewed 210-entry source census changed')
return index, program
def parse_paper(html, pid, index, program):
nodes = list(Document(html).root.walk()); meta = {}
for n in nodes:
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
day = one(meta.get('citation_publication_date', []))
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
raise ValueError('Unexpected publication date')
date.fromisoformat(day.replace('/', '-'))
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
or not authors or any(not clean(a) for a in authors)
or comparable(title) != comparable(index['title'])
or comparable(', '.join(authors)) != comparable(index['authors'])
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
raise ValueError('Publisher citation/index/visible metadata disagree')
pdf = one(meta.get('citation_pdf_url', []))
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
bib = one([n.text() for n in nodes if n.tag == 'pre'])
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
raise ValueError('BibTeX DOI/year mismatch')
variants = []
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
raise ValueError('Unreviewed complete author variant')
variants.append('reviewed_program_author_name_variant_not_person_alias')
if comparable(title) != comparable(program['title']):
if (pid, title, program['title']) != (89,
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
raise ValueError('Unreviewed program title variant')
variants.append('reviewed_program_title_variant_not_erratum')
inside = False; parts = []; markers = []
for n in (n for n in nodes if n.tag == 'p'):
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
if 'Abstract:' in labels:
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
if value: parts.append(value)
elif 'Download:' in labels:
inside = False; markers.append('end')
elif inside:
parts.append(clean(n.text()))
if markers != ['start', 'end'] or not any(parts):
raise ValueError('Missing or changed publisher abstract layout')
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
def collect(out, *, snapshots=None, cache=None):
if (snapshots is None) == (cache is None):
raise ValueError('Specify snapshots or cache')
out = Path(out)
if out.exists():
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
out.parent.mkdir(parents=True, exist_ok=True)
out.mkdir()
evidence = {}; receipts = {}
if snapshots is not None:
root = Path(snapshots)
try:
for r in json.loads((root/'receipts.json').read_bytes()):
name = r['file']
if not isinstance(name, str) or Path(name).name != name or name in receipts:
raise ValueError('Unsafe or repeated snapshot receipt')
receipts[name] = r
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def page(name, url):
if snapshots is not None:
path = root/name
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
raw = path.read_bytes(); r = receipts[name]
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
raise ValueError('Snapshot hash/receipt mismatch')
at = r['at']
else:
raw = fetch(url, cache, limit=8*1024*1024)
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
at = sidecar['retrieved_at']
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
raise ValueError('Invalid source size/timezone')
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
return raw.decode('utf-8')
try:
pages = {name: page(name, url) for name, url in PAGES.items()}
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
records = []
for pid in sorted(index):
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
'year': 2026, 'track': 'main', 'status': 'accepted',
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
'observed_at': evidence[name]['observed_at'],
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
'original_bibtex': fields['bibtex'], 'program': program[pid],
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
'policy_evidence': evidence['cfp.html'],
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
records.append(row)
path = out/'rss-proceedings-2026-main.jsonl'
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
path.write_bytes(raw)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
'status': 'partial', 'missing_abstract_count': 0}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
temporary.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
raise
def main(argv=None, *, prog=None):
p = argparse.ArgumentParser(prog=prog, description=__doc__)
p.add_argument('--out', required=True, type=Path)
g = p.add_mutually_exclusive_group(required=True)
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
args = p.parse_args(argv)
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
+1 -1
View File
@@ -4,7 +4,7 @@ import re
from paper_atlas.io.publication_ids import mlsys_id
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
def timestamp(value=None):
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+167
View File
@@ -0,0 +1,167 @@
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import sqlite3
import unittest
from unittest.mock import patch
from paper_atlas.collectors import aamas, aamas2026
from paper_atlas.ingestion.records import ingest_file
FIXTURE = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/aamas2026-fixture.json.gz').read_bytes()))
EVIDENCE = {'url': aamas2026.URL, 'observed_at': FIXTURE['receipts']['receipts'][0]['finished_at']}
class Aamas2026Tests(unittest.TestCase):
def parse(self, html=None):
return aamas2026.parse_contents(FIXTURE['contents'] if html is None else html, EVIDENCE)
def snapshots(self, folder):
folder.mkdir()
(folder/'contents.html').write_text(FIXTURE['contents'])
(folder/'receipts.json').write_text(json.dumps(FIXTURE['receipts']))
return folder
def test_every_publisher_title_author_affiliation_and_page_preserved(self):
rows, excluded, held = self.parse()
self.assertEqual((len(rows), len(excluded), len(held)), (639, 3, 1))
expected = {r['publisher_page']: r for r in FIXTURE['expected_entries']}
self.assertEqual(len(expected), 643)
all_rows = rows+excluded+held
self.assertEqual({r['metadata']['publisher_page'] for r in all_rows}, set(expected))
for r in all_rows:
e = expected[r['metadata']['publisher_page']]
self.assertEqual(r['title'], e['title'])
self.assertEqual(r['authors'], e['authors'])
self.assertEqual(r['metadata']['author_affiliations'], e['affiliations'])
self.assertEqual(r['metadata']['section_heading'], e['section'])
for r in rows:
self.assertIsNone(r['abstract'])
self.assertEqual(r['identifiers'], {})
if r['external_id'] != '2026/p3178':
self.assertEqual(r['status'], 'accepted')
self.assertNotIn('decision_date', r)
def test_explicit_withdrawal_preserves_status_date_and_no_pdf(self):
rows, _, _ = self.parse()
row, = [r for r in rows if r['external_id'] == '2026/p3178']
self.assertEqual(row['status'], 'withdrawn')
self.assertEqual(row['metadata']['withdrawal_date'], '2026-06-08')
self.assertNotIn('decision_date', row)
self.assertEqual(row['raw_decision'], 'Withdrawn June 8, 2026')
self.assertNotIn('paper_url', row['metadata'])
self.assertEqual(row['metadata']['publication_state'], 'withdrawn')
self.assertEqual(sum(r['status'] == 'accepted' for r in rows), 638)
def test_withdrawal_ingestion_retains_date_precision_without_invented_acceptance(self):
rows, _, _ = self.parse()
row, = [r for r in rows if r['external_id'] == '2026/p3178']
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); path = root/'withdrawn.jsonl'; db = root/'sample.sqlite'
path.write_text(json.dumps(row, ensure_ascii=False)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({
'source': 'aamas_proceedings', 'list_url': aamas2026.URL,
'venue': 'AAMAS', 'year': 2026, 'track': row['track'],
'count': 1, 'status': 'partial', 'missing_abstract_count': 1,
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}))
ingest_file(db, path)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT status,raw_value,decision_date FROM decisions').fetchall(),
[('withdrawn', 'Withdrawn June 8, 2026', None)])
metadata = json.loads(c.execute('SELECT metadata_json FROM source_records').fetchone()[0])
self.assertEqual(metadata['withdrawal_date'], '2026-06-08')
self.assertEqual(metadata['decision_date_precision'], 'day')
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
def test_retraction_is_not_accepted_or_withdrawn_and_retains_date(self):
rows, _, held = self.parse()
self.assertNotIn('2026/p238', {r['external_id'] for r in rows})
h, = held
self.assertEqual(h['publication_state'], 'retracted')
self.assertEqual(h['retraction_date'], '2026-06-08')
self.assertEqual(h['raw_publication_state'], 'Retracted June 8, 2026')
self.assertNotIn('status', h)
self.assertNotIn('paper_url', h['metadata'])
self.assertEqual(h['metadata']['source_evidence'], EVIDENCE)
def test_tracks_formats_and_prior_decisions_are_not_conflated(self):
rows, _, _ = self.parse()
counts = {t: sum(r['track'] == t for r in rows) for t in {r['track'] for r in rows}}
self.assertEqual(counts, {'proceedings_unspecified': 522, 'aaai_transfer': 19,
'blue_sky': 17, 'doctoral': 34, 'demo': 44, 'jaamas': 3})
for row in rows:
self.assertEqual(row['tags'][0]['namespace'], 'aamas_proceedings:publication_format')
if row['track'] == 'aaai_transfer':
self.assertIn('none inferred', row['metadata']['prior_submission_note'])
self.assertNotIn('decision_events', row)
if row['track'] == 'jaamas':
self.assertEqual(row['metadata']['publication_format'], 'journal_presentation')
self.assertIn('not another independent', row['metadata']['track_scope_note'])
def test_refuses_changed_year_section_page_link_author_and_retraction(self):
html = FIXTURE['contents']
replacements = [('AAMAS 2026:', 'AAMAS 2027:'),
('name="R" id="R"></a><font color="#010101">Research Paper Track',
'name="R" id="R"></a><font color="#010101">Workshop Track'),
('(Page 4)', '(Page 1)'),
('../pdfs/HQQZ1937.pdf', 'https://other.example/paper.pdf'),
('Michael Wooldridge <i>', '<i>'),
('Retracted June 8, 2026', 'Withdrawn June 8, 2026')]
for old, new in replacements:
self.assertIn(old, html)
with self.subTest(change=old), self.assertRaises(ValueError):
self.parse(html.replace(old, new, 1))
def test_frozen_collection_is_partial_with_full_held_evidence(self):
with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2026, 'fetch', side_effect=AssertionError('No HTTP')):
root = Path(tmp)
report = aamas.collect(2026, root/'out', snapshots=self.snapshots(root/'source'))
self.assertEqual(report['records'], 639)
self.assertEqual(report['status_counts'], {'accepted': 638, 'withdrawn': 1})
self.assertEqual(len(report['files']), 6)
self.assertEqual(len(report['held_publications']), 1)
self.assertFalse(report['database_written'])
self.assertFalse(report['pdfs_downloaded'])
for m in report['files']:
self.assertEqual(m['status'], 'partial')
self.assertEqual(m['missing_abstract_count'], m['count'])
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
self.assertTrue((root/'out/aamas-proceedings-2026.json').is_file())
def test_frozen_response_rejects_tamper_unsafe_path_status_and_symlink(self):
for mode in ('tamper', 'path', 'status', 'symlink'):
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); source = self.snapshots(root/'source')
r = json.loads((source/'receipts.json').read_text())
if mode == 'tamper':
(source/'contents.html').write_text('bad')
elif mode == 'path':
r['receipts'][0]['file'] = '../outside.html'
elif mode == 'status':
r['receipts'][0]['status'] = 403
else:
(root/'outside.html').write_text(FIXTURE['contents'])
(source/'contents.html').unlink()
(source/'contents.html').symlink_to(root/'outside.html')
(source/'receipts.json').write_text(json.dumps(r))
with self.assertRaises(ValueError):
aamas.collect(2026, root/'out', snapshots=source)
self.assertFalse((root/'out/aamas-proceedings-2026.json').exists())
def test_live_mode_fetches_only_html_and_preserves_receipt(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); cache = root/'cache'; cache.mkdir()
receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': aamas2026.URL}
(cache/hashlib.sha256(aamas2026.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
with patch.object(aamas2026, 'fetch', return_value=FIXTURE['contents'].encode()) as fetch:
report = aamas.collect(2026, root/'out', cache=cache)
fetch.assert_called_once_with(aamas2026.URL, cache, 8*1024**2, delay=1)
self.assertEqual(report['source_receipts'][0]['response_receipt'], receipt)
def test_frozen_fixture_is_original_hashed_response(self):
r, = FIXTURE['receipts']['receipts']
self.assertEqual(hashlib.sha256(FIXTURE['contents'].encode()).hexdigest(), r['sha256'])
self.assertEqual(r['bytes'], len(FIXTURE['contents'].encode()))
+1 -1
View File
@@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase):
aamas.collect(2012, root/'out', snapshots=source)
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
with self.assertRaises(ValueError):
aamas.collect(2026, root/'future', snapshots=source)
aamas.collect(2027, root/'future', snapshots=source)
self.assertFalse((root/'future').exists())
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
+95
View File
@@ -0,0 +1,95 @@
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import icaps2026 as c
class ICAPS2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
def snapshots(self, root):
root.mkdir()
for name, html in self.bundle['pages'].items():
(root/name).write_text(html)
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
return root
def test_full_listing_matches_independently_recorded_literals(self):
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
def test_incomplete_and_duplicate_listing_rejected(self):
html = self.bundle['pages']['accepted.html']
with self.assertRaises(ValueError):
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
def test_changed_edition_or_section_rejected(self):
html = self.bundle['pages']['accepted.html']
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
html.replace(c.MAIN, 'Workshop Papers')]:
with self.assertRaises(ValueError):
c.parse_listing(changed)
def test_ambiguous_author_lists_not_guessed(self):
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
with self.assertRaises(ValueError):
c.authors_from_literal(literal)
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
def test_collection_separates_presentations_and_unresolved_tracks(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
report = c.collect(out, snapshots=src)
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
self.assertEqual(report['public_rejected_availability'], 'unknown')
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
previous = json.loads((out/report['previously_published_file']).read_text())
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
self.assertEqual(len({r['external_id'] for r in rows}), 90)
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
self.assertEqual(report['files'][0]['status'], 'partial')
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(out, snapshots=src)
def test_tampering_retains_failure_without_success_receipt(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src')
(src/'accepted.html').write_text('tampered')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'out', snapshots=src)
self.assertTrue((root/'out/failed.json').exists())
self.assertFalse((root/'out/collection-report.json').exists())
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
for kind in ('redirect', 'time', 'duplicate'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
p.write_text(json.dumps(r))
with self.assertRaises(ValueError):
c.collect(root/'out', snapshots=src)
self.assertFalse((root/'out/collection-report.json').exists())
def test_snapshot_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
with self.assertRaisesRegex(ValueError, 'Unsafe'):
c.collect(root/'out', snapshots=src)
+96
View File
@@ -0,0 +1,96 @@
import copy
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import icaps_registry2026 as c
class ICAPSRegistry2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
cls.snapshot = json.loads(cls.raw)
def test_exact_edition_not_publication_year_alone(self):
papers, excluded = c.registry_papers(self.snapshot)
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
report = c.collect(root/'out', snapshots=src)
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
self.assertEqual(report['public_rejected_availability'], 'unknown')
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
self.assertEqual({r['external_id'] for r in rows}, set(original))
for r in rows:
doi = r['external_id']; o = original[doi]
self.assertEqual(r['metadata']['registry_fields'], o)
self.assertEqual(r['title'], o['title'][0])
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
self.assertEqual(r['identifiers'], {'doi': doi})
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
self.assertEqual(r['track'], 'main_tracks_unclassified')
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
self.assertIn('Stephen F. Smith', changed['authors'])
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(root/'out', snapshots=src)
def test_math_markup_is_not_flattened_as_plain_numbers(self):
self.assertEqual(c.abstract_text('<jats:p>A &lt; B and C &amp; D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
with self.assertRaises(ValueError):
c.abstract_text(raw)
def test_invalid_or_repeated_doi_and_edition_rejected(self):
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
with self.subTest(kind=kind):
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
if kind == 'doi': r['DOI'] = '10.9999/foreign'
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
if kind == 'volume': r['volume'] = '37'
if kind == 'container': r['container-title'] = ['Other Conference']
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_missing_authors_and_changed_order_evidence_rejected(self):
for kind in ('missing', 'sequence', 'profile'):
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
if kind == 'missing': r['authors'][0].pop('family')
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_receipt_scope_and_timezone_rejected(self):
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
s = copy.deepcopy(self.snapshot); s[key] = value
with self.assertRaises(ValueError):
c.registry_papers(s)
def test_tamper_failure_retained_and_success_not_published(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
with self.assertRaisesRegex(ValueError, 'hash changed'):
c.collect(root/'out', snapshots=src)
self.assertTrue((root/'out/failed.json').exists())
self.assertFalse((root/'out/collection-report.json').exists())
def test_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
with self.assertRaisesRegex(ValueError, 'Unsafe'):
c.collect(root/'out', snapshots=src)
+100
View File
@@ -0,0 +1,100 @@
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import rss2026 as rss
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
class Rss2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
def lists(self,**changes):
pages={**self.pages,**changes}
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
report=rss.collect(self.root/'out',snapshots=self.snap)
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
for row,gold in zip(rows,self.fixture['independent_gold']):
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
self.assertEqual(manifest['missing_abstract_count'],0)
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
self.assertEqual(len(shortened),6)
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
self.assertEqual(len(p['authors']),55)
self.assertFalse(p['program_variants'])
def test_six_reviewed_name_variants_preserve_original_strings(self):
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
self.assertTrue(p['title'].startswith('RLinf-VLA'))
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
def test_unreviewed_author_or_title_variant_stops(self):
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
with self.assertRaisesRegex(ValueError,'Unreviewed'):
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
def test_demo_policy_cannot_be_reused_from2025(self):
with self.assertRaisesRegex(ValueError,'policy'):
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
with self.assertRaises(ValueError):
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
html=self.pages['p001.html']
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
self.assertIn(old,html)
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
def test_index_census_and_preview_tampering_stop(self):
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
self.assertFalse((self.root/'out/collection-report.json').exists())
self.assertTrue((self.root/'out/failed.json').exists())
def test_existing_output_and_unsafe_receipt_are_rejected(self):
out=self.root/'existing';out.mkdir()
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
(self.snap/'receipts.json').write_text(json.dumps(receipts))
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
self.assertTrue((self.root/'unsafe/failed.json').exists())
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
+59
View File
@@ -0,0 +1,59 @@
"""RSS publisher/DBLP observations share one event without losing old decisions."""
from pathlib import Path
import json
import sqlite3
import tempfile
import unittest
from paper_atlas.db.submissions import identities_for_row
from tests.test_aamas_publication_identity import row, ingest
DOI='10.15607/rss.2026.xxii.041'
def rss(source,external,**kwargs):
return row(source,external,venue='RSS',year=2026,track='main',
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
class RssPublicationIdentityTests(unittest.TestCase):
def test_only_verified_accepted_source_adds_publication_identity(self):
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
self.assertEqual(identities_for_row({**r,'status':status}),[])
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
for pair in ((a,b),(b,a)):
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
for r in pair:ingest(db,root,r)
ingest(db,root,a)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'papers.sqlite'
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
ingest(db,root,old)
with sqlite3.connect(db) as c:
before=c.execute('SELECT * FROM decisions').fetchall()
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
ingest(db,root,new)
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
+148
View File
@@ -0,0 +1,148 @@
import gzip
import hashlib
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.audits.source_identity import audit, classify, load_inputs
from paper_atlas.postgres.client import ReadConnection
def record(**updates):
row = dict(source='reviewed_program', external_id='2026/forum-a',
title='A sufficiently specific scientific paper title', authors=['Jane Doe', 'John Smith'],
venue='NeurIPS', year=2026, track='main', status='accepted', identifiers={},
source_record_hits=[], identifier_hits=[], title_candidate_ids=[])
row.update(updates)
return row
def write_input(root, rows):
path = root / 'source.jsonl'
raw = b''.join((json.dumps(row) + '\n').encode() for row in rows)
path.write_bytes(raw)
manifest = {k: rows[0][k] for k in ('source', 'venue', 'year', 'track')}
manifest.update(count=len(rows), file_sha256=hashlib.sha256(raw).hexdigest(),
file='/ignored/untrusted/location.jsonl', status='partial')
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
return path
class CandidateTests(unittest.TestCase):
def test_duplicate_names_do_not_equal_distinct_author_multiset(self):
row = record(authors=['Jane Doe', 'Jane Doe'], title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': ['Jane Doe']}})
self.assertEqual(result['kind'], 'partial_author_overlap_requires_review')
self.assertIn('duplicate_normalized_author_names', result['flags'])
self.assertFalse(result['comparisons'][0]['nonempty_author_multiset_equal'])
def test_arxiv_author_representation_and_order_match_but_never_approve(self):
row = record(title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': [['Smith', 'John', ''], ['Doe', 'Jane', '']]}})
self.assertEqual(result['kind'], 'one_exact_title_author_multiset_requires_review')
self.assertEqual(result['decision'], 'not_approved_for_import')
def test_binding_conflict_overrides_existing_source(self):
row = record(source_record_hits=[{'paper_id': 7}], identifier_hits=[{'paper_id': 8}])
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in (7, 8)}
self.assertEqual(classify(row, entities)['kind'], 'conflicting_existing_bindings_hold')
def test_wrong_doi_binding_retains_author_title_disagreement(self):
row = record(identifier_hits=[{'paper_id': 7}])
result = classify(row, {7: {'title': 'Completely different work', 'authors': ['Somebody Else']}})
self.assertEqual(result['kind'], 'existing_identifier_binding_requires_review')
self.assertFalse(result['comparisons'][0]['normalized_title_equal'])
self.assertEqual(result['comparisons'][0]['author_name_overlap'], [])
def test_no_hit_is_not_novelty_and_missing_authors_never_agree(self):
self.assertEqual(classify(record(), {})['kind'], 'no_exact_match_not_proof_of_novelty')
row = record(authors=[], title_candidate_ids=[7])
result = classify(row, {7: {'title': row['title'], 'authors': []}})
self.assertEqual(result['kind'], 'no_author_agreement_hold')
def test_title_candidate_cap_is_explicit(self):
row = record(title_candidate_ids=list(range(21)))
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in range(21)}
self.assertTrue(classify(row, entities)['title_candidates_truncated'])
self.assertEqual(classify(row, entities)['kind'], 'title_candidate_limit_reached_hold')
class InputTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.root = Path(self.temp.name)
def test_raw_line_hash_and_manifest_path_not_followed(self):
path = write_input(self.root, [record()])
rows, inputs = load_inputs([path])
self.assertEqual(rows[0]['raw_line_sha256'], hashlib.sha256(path.read_bytes()).hexdigest())
self.assertEqual(inputs[0]['count'], 1)
def test_hash_tamper_rejected_before_database_connection(self):
path = write_input(self.root, [record()])
path.write_bytes(path.read_bytes().replace(b'Jane Doe', b'Jane Roe'))
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
with self.assertRaisesRegex(ValueError, 'hash mismatch'):
audit([path], self.root / 'out', pg_service='test', pg_schema='research')
connection.assert_not_called()
def test_manifest_scope_and_count_tamper_and_duplicate_sources(self):
path = write_input(self.root, [record()])
manifest_path = path.with_suffix('.manifest.json')
manifest = json.loads(manifest_path.read_text())
for key, value in [('year', 2025), ('count', 2)]:
altered = dict(manifest, **{key: value})
manifest_path.write_text(json.dumps(altered))
with self.assertRaises(ValueError):
load_inputs([path])
manifest_path.write_text(json.dumps(manifest))
with self.assertRaisesRegex(ValueError, 'Duplicate source'):
load_inputs([path, path])
def test_existing_output_never_replayed_and_failure_not_completion(self):
path = write_input(self.root, [record()])
out = self.root / 'out'
with patch('paper_atlas.audits.source_identity.ReadConnection', side_effect=RuntimeError('unavailable')):
with self.assertRaises(RuntimeError):
audit([path], out, pg_service='test', pg_schema='research')
self.assertTrue((out / 'failed.json').exists())
self.assertFalse((out / 'postgresql-preflight.json').exists())
with self.assertRaises(FileExistsError):
audit([path], out, pg_service='test', pg_schema='research')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the authorized remote PostgreSQL service')
class ReadOnlyIntegrationTests(unittest.TestCase):
def test_existing_binding_full_evidence_and_no_import_approval(self):
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
connection = ReadConnection(service, 'research')
try:
self.assertEqual(connection.execute('SHOW transaction_read_only').fetchone()[0], 'on')
hit = connection.execute("SELECT value,paper_id FROM identifiers WHERE scheme='arxiv' ORDER BY value LIMIT 1").fetchone()
self.assertIsNotNone(hit)
paper = connection.execute('SELECT title,authors_json FROM papers WHERE id=?', (hit['paper_id'],)).fetchone()
finally:
connection.close()
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
path = write_input(root, [record(title=paper['title'], authors=json.loads(paper['authors_json']),
identifiers={'arxiv': hit['value']})])
report = audit([path], root / 'out', pg_service=service, pg_schema='research')
self.assertTrue(report['transaction_read_only'])
self.assertFalse(report['database_written'])
self.assertEqual(report['approved_for_import'], 0)
row, = report['entries']
self.assertIn(hit['paper_id'], [r['paper_id'] for r in row['identifier_hits']])
shard = report['entities'][str(hit['paper_id'])]
raw = (root / 'out' / shard['file']).read_bytes()
self.assertEqual(hashlib.sha256(raw).hexdigest(), shard['sha256'])
packet = json.loads(gzip.decompress(raw))
self.assertEqual(packet['paper']['id'], hit['paper_id'])
self.assertEqual(packet['paper']['title'], paper['title'])
self.assertEqual(packet['paper']['authors_json'], paper['authors_json'])
self.assertIn('attempts', packet)
self.assertTrue((root / 'out/postgresql-preflight.json').is_file())