Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c97ea94b49 | |||
| 59053fd4fb | |||
| b50136d5bc | |||
| 87a0689fcc | |||
| 9093a8ce25 | |||
| e19cb83cd6 | |||
| 4d2df2a826 | |||
| 19d03ced54 | |||
| c33f5422de | |||
| 1cfd44d8b6 | |||
| 5de7f6ff22 | |||
| b4e249e675 | |||
| 93f00d1a64 | |||
| 8b7bc6d871 | |||
| 7588a4e904 | |||
| 891a63d982 | |||
| fe3e2502a5 | |||
| b675a38b97 | |||
| cb875b8eef | |||
| 57a557c66b | |||
| 5b5e606f0c | |||
| 9f84fd7caf | |||
| bf05d3cdc8 | |||
| 930df2a4d5 | |||
| 37380e1d3b |
@@ -48,6 +48,61 @@ jobs:
|
|||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
|
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
eccv_official = root/'eccv2026-official'
|
||||||
|
eccv_papers = root/'eccv2026-paper-samples'
|
||||||
|
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||||
|
for name, html in eccv_fixture['official_pages'].items():
|
||||||
|
(eccv_official/name).write_text(html)
|
||||||
|
(eccv_official/'manifest.json').write_text(eccv_fixture['official_manifest'])
|
||||||
|
for pid, html in eccv_fixture['paper_pages'].items():
|
||||||
|
(eccv_papers/(pid+'.html')).write_text(html)
|
||||||
|
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||||
|
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||||
|
assert eccv_report['records'] == 3 and eccv_report['expected_program_ids'] == 2834
|
||||||
|
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2831
|
||||||
|
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||||
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
|
icaps_snapshots.mkdir()
|
||||||
|
for name, html in icaps_fixture['pages'].items():
|
||||||
|
(icaps_snapshots/name).write_text(html)
|
||||||
|
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
||||||
|
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
||||||
|
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
||||||
|
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
||||||
|
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
||||||
|
registry_snapshots = root/'icaps2026-registry-snapshots'
|
||||||
|
registry_snapshots.mkdir()
|
||||||
|
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
|
||||||
|
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
|
||||||
|
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
|
||||||
|
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
|
||||||
|
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||||
|
rss_snapshots = root/'rss2026-snapshots'
|
||||||
|
rss_snapshots.mkdir()
|
||||||
|
for name, html in rss_fixture['pages'].items():
|
||||||
|
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
corl_official = root/'corl2026-official'
|
||||||
|
corl_archive = root/'corl2026-archive'
|
||||||
|
corl_official.mkdir()
|
||||||
|
corl_archive.mkdir()
|
||||||
|
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||||
|
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||||
|
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||||
|
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||||
|
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||||
|
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||||
|
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||||
@@ -62,6 +117,16 @@ jobs:
|
|||||||
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
|
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
||||||
|
aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
|
||||||
|
aamas2026_snapshots = root/'aamas2026-snapshots'
|
||||||
|
aamas2026_snapshots.mkdir()
|
||||||
|
(aamas2026_snapshots/'contents.html').write_text(aamas2026['contents'])
|
||||||
|
(aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True)
|
||||||
|
aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text())
|
||||||
|
assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1
|
||||||
|
assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1}
|
||||||
|
assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
||||||
@@ -86,6 +151,7 @@ jobs:
|
|||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
|
||||||
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
|
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
|
||||||
|
|||||||
@@ -8,6 +8,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
|
|||||||
|
|
||||||
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
|
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
|
||||||
|
|
||||||
|
AAMAS 2026 的官方 HTML 目录由 `collect aamas-proceedings --year 2026` 适配,区分 track 和发表形式,keynote 排除、撤稿证据单独保留;[采集与恢复说明](docs/aamas-proceedings.md)。采集不写数据库,仍需身份核验和增量导入。
|
||||||
|
|
||||||
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
|
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
|
||||||
|
|
||||||
`collect springer-selected` 核验限定的会后选集,分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集,原始会议决定仍须单独核查;配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
|
`collect springer-selected` 核验限定的会后选集,分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集,原始会议决定仍须单独核查;配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
|
||||||
@@ -114,3 +116,17 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
|||||||
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
||||||
|
|
||||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||||
|
|
||||||
|
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||||
|
|
||||||
|
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||||
|
|
||||||
|
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||||
|
|
||||||
|
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
||||||
|
|
||||||
|
### CoRL 2026 官方程序快照
|
||||||
|
|
||||||
|
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||||
|
|
||||||
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|||||||
@@ -1,6 +1,31 @@
|
|||||||
# AAMAS 官方论文集元数据
|
# AAMAS 官方论文集元数据
|
||||||
|
|
||||||
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
|
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025、2026** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025、2026 解析分别封装在 `paper_atlas.collectors.aamas2025`、`paper_atlas.collectors.aamas2026`。
|
||||||
|
|
||||||
|
|
||||||
|
## AAMAS 2026
|
||||||
|
|
||||||
|
2026 只读取[官方 HTML 目录](https://www.ifaamas.org/Proceedings/aamas2026/forms/contents.htm),不下载 PDF。冻结目录 SHA256 `ce2bc7fc5da340930343901506a9ce37c0334d4068f07546431c9deee3f13de9`,643 个条目:keynote 3、research 338、extended abstract 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3。
|
||||||
|
|
||||||
|
排除 3 个 keynote;research 的 p238 明确标注 `Retracted June 8, 2026`,完整题名、作者、栏目、日期、原文和来源回执保存在采集报告 `held_publications` 中,**不会输出 accepted 或 withdrawn 投稿记录**。当前决定模型没有独立 retracted 状态,后续须单独建模和核验;原始证据不能删除。任何新增或改变的撤稿标记都会停止采集,等待人工核查。
|
||||||
|
|
||||||
|
另一个 extended abstract p3178 明确标注 `Withdrawn June 8, 2026`,输出 `withdrawn`,在 metadata 保留日期 `2026-06-08` 和日精度;原网页没有时区/时刻,不能伪造决定 timestamp,不补造其此前的接受决定或 PDF 链接。因此 639 个输出条目是 **638 accepted、1 withdrawn**,按六个 track 分组:`proceedings_unspecified` 522、`aaai_transfer` 19、`blue_sky` 17、`doctoral` 34、`demo` 44、`jaamas` 3。全文/扩展摘要是发表形式,不是研究类型;research 栏目不能证明每篇原始 main/special 投稿轨道。JAAMAS 保留期刊展示语义,不制造另一次独立审稿录用。
|
||||||
|
|
||||||
|
[AAAI 转投详细政策](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 要求基于 AAAI-26 rejected 稿件;[总览](https://cyprusconferences.org/aamas2026/calls/) 却写 AAAI-25,须保留冲突。采集器保留 AAAI 栏目归属,不据此生成原 AAAI 投稿 ID、年份或逐篇拒稿决定。目录不是原始决定 note,未来关联仍需逐篇证据。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect aamas-proceedings --year 2026 \
|
||||||
|
--snapshots /data/paper-index/collected/aamas2026-primary-evidence-v1 \
|
||||||
|
--out /data/paper-index/collected/aamas2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
在线模式将 `--snapshots` 换成 `--cache /data/paper-index/cache/aamas2026`。冻结模式 `receipts.json` 的 `receipts` 至少包含官方目录 URL、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区 `finished_at`。拒绝越界路径、越界符号链接、失败状态、哈希/大小不符的页面。这里只读取目录对应回执,其他已冻结政策网页继续保留。
|
||||||
|
|
||||||
|
输出六组 JSONL/manifest、原始 HTML 与最后发布的 `aamas-proceedings-2026.json`;统一入口沿用输出目录锁和暂存目录。manifest 全部为 `partial`,639 条 HTML 摘要缺失如实记录。发布编号使用年份和起始页,随机 PDF 文档 ID 单独保留,不能沿用 2025 的 pN.pdf 规则。
|
||||||
|
|
||||||
|
首次部署先核对来源哈希和生产状态,再在新输出目录运行采集、独立比对 643 条标题/作者/栏目分区,随后做 PostgreSQL 身份预检与增量导入。采集、远程 CI 通过或目录含某个年份均不代表生产已导入、摘要齐全或最终去重完成。较早年份不自动启动补采,当前主动补漏范围按用户要求从 2020 年起。
|
||||||
|
|
||||||
|
## AAMAS 2025
|
||||||
|
|
||||||
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
|
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# CoRL 2026:官方程序与公开档案
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect corl2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||||
|
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||||
|
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||||
|
|
||||||
|
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||||
|
|
||||||
|
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||||
|
|
||||||
|
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||||
|
|
||||||
|
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||||
|
|
||||||
|
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||||
|
|
||||||
|
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||||
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
|
|||||||
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
|
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
|
||||||
|
|
||||||
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
|
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
|
||||||
|
|
||||||
|
### RSS 发表身份
|
||||||
|
|
||||||
|
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
|
||||||
|
|
||||||
|
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
# ECCV 2026:暂定节目与公开元数据
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect eccv2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||||
|
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||||
|
--max-new-pages 200 \
|
||||||
|
--out /data/paper-index/collected/eccv2026-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||||
|
|
||||||
|
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||||
|
|
||||||
|
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||||
|
|
||||||
|
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||||
|
|
||||||
|
## 来源边界
|
||||||
|
|
||||||
|
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||||
|
|
||||||
|
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||||
|
|
||||||
|
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||||
|
|
||||||
|
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||||
|
|
||||||
|
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICAPS 2026 官方名单
|
||||||
|
|
||||||
|
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-program \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
|
||||||
|
|
||||||
|
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
|
||||||
|
|
||||||
|
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
|
||||||
|
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
|
||||||
|
|
||||||
|
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
|
||||||
|
|
||||||
|
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
|
||||||
|
|
||||||
|
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
|
||||||
|
|
||||||
|
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
|
||||||
|
|
||||||
|
## 独立注册元数据版本
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
|
||||||
|
|
||||||
|
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
|
||||||
|
|
||||||
|
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
|
||||||
|
|
||||||
|
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# RSS 2026 论文集来源
|
||||||
|
|
||||||
|
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-v1
|
||||||
|
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
|
||||||
|
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-replay-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
|
||||||
|
|
||||||
|
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
|
||||||
|
|
||||||
|
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
|
||||||
|
|
||||||
|
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
|
||||||
|
|
||||||
|
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# 来源身份只读预检
|
||||||
|
|
||||||
|
`audit source-identities` 在 PostgreSQL 的同一 `REPEATABLE READ READ ONLY` 快照中,检查规范化 JSONL 的来源绑定、稳定标识及精确标题候选,并导出候选论文的原始来源和投稿决定历史。用于入库前评审,不执行导入、合并或标识修改。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit source-identities \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--files /data/paper-index/collected/example/main.jsonl \
|
||||||
|
/data/paper-index/collected/example/position.jsonl \
|
||||||
|
--out /data/paper-index/collected/example-identity-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
每个 JSONL 必须有同名 `.manifest.json`,包含 `source`、`venue`、`year`、`track`、`count` 和 `file_sha256`。逐行校验届次、显式决定状态、标识类型和跨文件来源主键唯一性;文件哈希和数量必须相符。只读取调用者指定的文件,不跟随 manifest 中的绝对 `file` 路径;原始行哈希包含换行符。原始 JSONL 和 manifest 应与报告一起保留。
|
||||||
|
|
||||||
|
输出目录必须不存在,防止复用旧报告或覆盖证据。失败会保留 `failed.json` 及已完成分片,不能把部分输出当成功;排查后使用新的输出目录重新核查。成功报告 `postgresql-preflight.json` 最后原子写入,记录输入哈希、数据库快照和每条候选。实体详情存为 `entities/<分片>/<paper_id>.json.gz`,报告中列出压缩文件的 SHA-256;包含完整 paper、标识和限量原始来源、投稿、决定及出处。每篇最多保留 100 条来源、200 次投稿,每次投稿最多 200 条决定,每组决定或投稿出处最多 100 条,所有截断均显式标记。精确标题最多取 21 个候选;达到上限即标记,不能据此推算总数。这里没有计算全表精确数量。
|
||||||
|
|
||||||
|
候选分类用于安排审核:既有来源记录需要核对并避免重放,多个绑定冲突必须暂缓;单个 DOI、arXiv 或 OpenReview 绑定也可能错误,必须核对原文、标题、作者和历史。作者比较保留完整规范化姓名的重复次数,但姓名相同不能证明是同一个人。未找到精确标题或标识不能证明是新作品,仍需近似标题、版本和跨会历史核查。
|
||||||
|
|
||||||
|
所有条目固定为 `not_approved_for_import`,批准数量为 0。不要直接把此报告当成 curation 计划或导入队列。后续获批导入前应重新检查实时来源绑定;来源摘要缺失不得覆盖已有摘要,旧拒稿与新录用应作为不同投稿经历保留。本命令不能代替所有导入结束后的全库去重、覆盖和约束验收。
|
||||||
|
|
||||||
|
构建与测试仅在已确认的 `ces-control:host` 远程 runner 执行。集成测试使用已有 PostgreSQL 服务做真实只读候选查询,不往生产库添加测试记录。
|
||||||
@@ -0,0 +1,252 @@
|
|||||||
|
"""Read-only, evidence-preserving identity candidates for normalized source files.
|
||||||
|
|
||||||
|
A candidate is never permission to import or merge. Absence of an exact match is
|
||||||
|
never proof of a new work. Run again against a fresh snapshot before curation.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from paper_atlas.db.store import author_keys, normalized
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.postgres.client import ReadConnection
|
||||||
|
|
||||||
|
STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(data):
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def authors_multiset(authors):
|
||||||
|
# author_keys intentionally removes duplicates; identity review must retain them.
|
||||||
|
return Counter(key for author in authors for key in author_keys([author]))
|
||||||
|
|
||||||
|
|
||||||
|
def load_inputs(files):
|
||||||
|
records, inputs, seen = [], [], set()
|
||||||
|
for filename in files:
|
||||||
|
path = Path(filename).resolve()
|
||||||
|
manifest_path = path.with_suffix('.manifest.json')
|
||||||
|
raw_manifest = manifest_path.read_bytes()
|
||||||
|
manifest = json.loads(raw_manifest)
|
||||||
|
hasher, count = hashlib.sha256(), 0
|
||||||
|
with path.open('rb') as stream:
|
||||||
|
for line_number, line in enumerate(stream, 1):
|
||||||
|
hasher.update(line)
|
||||||
|
row = json.loads(line)
|
||||||
|
for key in ('source', 'external_id', 'title', 'venue', 'track'):
|
||||||
|
if not isinstance(row.get(key), str) or not row[key].strip():
|
||||||
|
raise ValueError(f'Invalid {key}: {path.name}:{line_number}')
|
||||||
|
if type(row.get('year')) is not int or row.get('status') not in STATUSES:
|
||||||
|
raise ValueError('Invalid year or explicit source status')
|
||||||
|
for key in ('source', 'venue', 'year', 'track'):
|
||||||
|
if row[key] != manifest.get(key):
|
||||||
|
raise ValueError(f'Manifest scope mismatch: {key}')
|
||||||
|
authors = row.get('authors')
|
||||||
|
if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors):
|
||||||
|
raise ValueError('Invalid authors')
|
||||||
|
identifiers = row.get('identifiers', {})
|
||||||
|
if not isinstance(identifiers, dict) or any(
|
||||||
|
not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip()
|
||||||
|
for k, v in identifiers.items()):
|
||||||
|
raise ValueError('Identifiers must contain singular nonempty strings')
|
||||||
|
key = (row['source'], row['external_id'])
|
||||||
|
if key in seen:
|
||||||
|
raise ValueError('Duplicate source/external_id across inputs')
|
||||||
|
seen.add(key)
|
||||||
|
record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors',
|
||||||
|
'venue', 'year', 'track', 'status')}
|
||||||
|
record.update(identifiers=identifiers, file=str(path), line=line_number,
|
||||||
|
raw_line_sha256=digest(line), source_record_hits=[],
|
||||||
|
identifier_hits=[], title_candidate_ids=[])
|
||||||
|
records.append(record)
|
||||||
|
count += 1
|
||||||
|
if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'):
|
||||||
|
raise ValueError('Input count/hash mismatch or empty input')
|
||||||
|
inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count,
|
||||||
|
'manifest_sha256': digest(raw_manifest), 'manifest': manifest})
|
||||||
|
if not records:
|
||||||
|
raise ValueError('At least one nonempty input is required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def _lookup(connection, values, columns, lateral):
|
||||||
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||||
|
width = len(columns)
|
||||||
|
for start in range(0, len(values), 100):
|
||||||
|
batch = values[start:start + 100]
|
||||||
|
sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join(
|
||||||
|
'(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true'
|
||||||
|
for row in connection.execute(sql, tuple(value for item in batch for value in item)):
|
||||||
|
yield dict(row)
|
||||||
|
|
||||||
|
|
||||||
|
def find_candidates(connection, records):
|
||||||
|
source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)]
|
||||||
|
for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), '''
|
||||||
|
SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id
|
||||||
|
WHERE s.name=keys.source AND r.external_id=keys.external_id'''):
|
||||||
|
records[hit.pop('n')]['source_record_hits'].append(hit)
|
||||||
|
identifiers = []
|
||||||
|
for i, row in enumerate(records):
|
||||||
|
pairs = {(row['source'], row['external_id']), *row['identifiers'].items()}
|
||||||
|
identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs))
|
||||||
|
for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), '''
|
||||||
|
SELECT scheme,value,paper_id FROM identifiers
|
||||||
|
WHERE scheme=keys.scheme AND value=keys.value'''):
|
||||||
|
records[hit.pop('n')]['identifier_hits'].append(hit)
|
||||||
|
title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)]
|
||||||
|
for hit in _lookup(connection, title_keys, ('n', 'title_key'), '''
|
||||||
|
SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''):
|
||||||
|
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
||||||
|
|
||||||
|
|
||||||
|
def entity_packet(connection, paper_id):
|
||||||
|
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
|
||||||
|
if paper is None:
|
||||||
|
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
||||||
|
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
|
||||||
|
FROM source_records r JOIN sources s ON s.id=r.source_id
|
||||||
|
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
|
||||||
|
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
||||||
|
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||||
|
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
|
||||||
|
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
|
||||||
|
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
|
||||||
|
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
||||||
|
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
||||||
|
for attempt in packet['attempts']:
|
||||||
|
observations = [dict(r) for r in connection.execute(
|
||||||
|
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
|
||||||
|
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
||||||
|
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
|
||||||
|
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
||||||
|
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
||||||
|
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
|
||||||
|
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
|
for decision in attempt['decisions']:
|
||||||
|
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
|
||||||
|
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
||||||
|
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
||||||
|
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
|
||||||
|
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
|
return packet
|
||||||
|
|
||||||
|
|
||||||
|
def classify(record, entities):
|
||||||
|
anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']}
|
||||||
|
titles = record['title_candidate_ids']
|
||||||
|
flags = []
|
||||||
|
own_authors = authors_multiset(record['authors'])
|
||||||
|
if not own_authors:
|
||||||
|
flags.append('missing_source_authors')
|
||||||
|
if any(n > 1 for n in own_authors.values()):
|
||||||
|
flags.append('duplicate_normalized_author_names')
|
||||||
|
if len(normalized(record['title'])) < 25:
|
||||||
|
flags.append('short_title')
|
||||||
|
comparisons = []
|
||||||
|
for pid in sorted(anchors | set(titles)):
|
||||||
|
entity = entities[pid]
|
||||||
|
other = authors_multiset(entity['authors'])
|
||||||
|
comparisons.append({'paper_id': pid,
|
||||||
|
'normalized_title_equal': normalized(record['title']) == normalized(entity['title']),
|
||||||
|
'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other,
|
||||||
|
'author_name_overlap': sorted(own_authors.keys() & other.keys())})
|
||||||
|
if len(anchors) > 1:
|
||||||
|
kind = 'conflicting_existing_bindings_hold'
|
||||||
|
elif record['source_record_hits']:
|
||||||
|
kind = 'existing_source_record_reconcile_no_replay'
|
||||||
|
elif anchors:
|
||||||
|
kind = 'existing_identifier_binding_requires_review'
|
||||||
|
elif len(titles) >= 21:
|
||||||
|
kind = 'title_candidate_limit_reached_hold'
|
||||||
|
elif not titles:
|
||||||
|
kind = 'no_exact_match_not_proof_of_novelty'
|
||||||
|
elif len(titles) > 1:
|
||||||
|
kind = 'multiple_exact_title_entities_hold'
|
||||||
|
elif comparisons[0]['nonempty_author_multiset_equal']:
|
||||||
|
kind = 'one_exact_title_author_multiset_requires_review'
|
||||||
|
elif comparisons[0]['author_name_overlap']:
|
||||||
|
kind = 'partial_author_overlap_requires_review'
|
||||||
|
else:
|
||||||
|
kind = 'no_author_agreement_hold'
|
||||||
|
return {'kind': kind, 'flags': flags, 'comparisons': comparisons,
|
||||||
|
'title_candidates_truncated': len(titles) >= 21,
|
||||||
|
'decision': 'not_approved_for_import'}
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path, value):
|
||||||
|
temporary = path.with_suffix(path.suffix + '.part')
|
||||||
|
with temporary.open('w', encoding='utf-8') as stream:
|
||||||
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||||
|
stream.write('\n')
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def audit(files, out, *, pg_service, pg_schema):
|
||||||
|
# Reject input corruption before opening a database connection or reserving output.
|
||||||
|
records, inputs = load_inputs(files)
|
||||||
|
out = Path(out)
|
||||||
|
out.mkdir(parents=True, exist_ok=False)
|
||||||
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||||
|
connection = None
|
||||||
|
try:
|
||||||
|
connection = ReadConnection(pg_service, pg_schema)
|
||||||
|
connection.execute("SET LOCAL statement_timeout='60s'")
|
||||||
|
if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on':
|
||||||
|
raise RuntimeError('A read-only transaction is required')
|
||||||
|
snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema,
|
||||||
|
transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone())
|
||||||
|
find_candidates(connection, records)
|
||||||
|
ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} |
|
||||||
|
{pid for row in records for pid in row['title_candidate_ids']})
|
||||||
|
entities, index = {}, {}
|
||||||
|
for number, pid in enumerate(ids, 1):
|
||||||
|
packet = entity_packet(connection, pid)
|
||||||
|
relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz'
|
||||||
|
target = out / relative
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0)
|
||||||
|
target.write_bytes(raw)
|
||||||
|
index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)}
|
||||||
|
entities[pid] = {'title': packet['paper']['title'],
|
||||||
|
'authors': json.loads(packet['paper']['authors_json'])}
|
||||||
|
if number % 250 == 0 or number == len(ids):
|
||||||
|
print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True)
|
||||||
|
kinds = Counter()
|
||||||
|
for row in records:
|
||||||
|
row.update(classify(row, entities))
|
||||||
|
kinds[row['kind']] += 1
|
||||||
|
report = {'status': 'completed_read_only_candidates_not_import_approval',
|
||||||
|
'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True,
|
||||||
|
'database_written': False, 'approved_for_import': 0, 'records': len(records),
|
||||||
|
'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records,
|
||||||
|
'limitations': ['Exact keys only; no match does not establish a new work.',
|
||||||
|
'Name agreement does not prove person or scientific-work identity.',
|
||||||
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||||
|
'Source/history limits are explicit in compressed entity packets.',
|
||||||
|
'Not a final whole-database deduplication or coverage audit.']}
|
||||||
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
_atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval',
|
||||||
|
'error_type': type(error).__name__, 'database_written': False})
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if connection is not None:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
||||||
|
parser.add_argument('--pg-service', required=True)
|
||||||
|
parser.add_argument('--pg-schema', required=True)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
||||||
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||||
@@ -23,9 +23,14 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||||
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||||
@@ -46,6 +51,7 @@ COMMANDS = {
|
|||||||
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
|
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
|
||||||
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
|
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
|
||||||
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
|
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
|
||||||
|
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
|
||||||
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
|
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None):
|
|||||||
|
|
||||||
|
|
||||||
def _collect(year, out, *, cache=None, snapshots=None):
|
def _collect(year, out, *, cache=None, snapshots=None):
|
||||||
|
if year == 2026:
|
||||||
|
from paper_atlas.collectors.aamas2026 import collect as collect_2026
|
||||||
|
return collect_2026(out, cache=cache, snapshots=snapshots)
|
||||||
if year == 2025:
|
if year == 2025:
|
||||||
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
||||||
return collect_2025(out, cache=cache, snapshots=snapshots)
|
return collect_2025(out, cache=cache, snapshots=snapshots)
|
||||||
@@ -193,8 +196,8 @@ def _collect(year, out, *, cache=None, snapshots=None):
|
|||||||
|
|
||||||
|
|
||||||
def collect(year, out, *, cache=None, snapshots=None):
|
def collect(year, out, *, cache=None, snapshots=None):
|
||||||
if year not in (2012, 2025):
|
if year not in (2012, 2025, 2026):
|
||||||
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
|
raise ValueError('Only independently reviewed AAMAS 2012/2025/2026 layouts are supported')
|
||||||
out = Path(out)
|
out = Path(out)
|
||||||
out.mkdir(parents=True, exist_ok=True)
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
with (out/'aamas.collect.lock').open('a') as lock:
|
with (out/'aamas.collect.lock').open('a') as lock:
|
||||||
@@ -204,7 +207,7 @@ def collect(year, out, *, cache=None, snapshots=None):
|
|||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
def main(argv=None, *, prog=None):
|
||||||
parser = argparse.ArgumentParser(prog=prog)
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
|
parser.add_argument('--year', type=int, choices=[2012, 2025, 2026], required=True)
|
||||||
parser.add_argument('--out', required=True)
|
parser.add_argument('--out', required=True)
|
||||||
source = parser.add_mutually_exclusive_group(required=True)
|
source = parser.add_mutually_exclusive_group(required=True)
|
||||||
source.add_argument('--cache')
|
source.add_argument('--cache')
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""Reviewed AAMAS 2026 HTML metadata; retractions stay outside accepted queues."""
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from paper_atlas.collectors.aamas2025 import one
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2026/'
|
||||||
|
URL = ROOT+'forms/contents.htm'
|
||||||
|
TITLE = 'AAMAS 2026: Proceedings of the 2026 International Conference on Autonomous Agents and Multiagent Systems'
|
||||||
|
SECTIONS = {
|
||||||
|
'K': ('Keynote Talks', 'keynote', 'keynote', 3),
|
||||||
|
'R': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 338),
|
||||||
|
'1': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 185),
|
||||||
|
'A': ('AAAI Track', 'full_paper', 'aaai_transfer', 19),
|
||||||
|
'B': ('Blue Sky Ideas Track', 'blue_sky_paper', 'blue_sky', 17),
|
||||||
|
'C': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 34),
|
||||||
|
'D': ('Demonstration Track', 'demo', 'demo', 44),
|
||||||
|
'J': ('JAAMAS Track', 'journal_presentation', 'jaamas', 3),
|
||||||
|
}
|
||||||
|
PDF = re.compile(r'\.\./pdfs/([A-Z]{4}[0-9]{4})\.pdf')
|
||||||
|
RETRACTION_TITLE = 'Unifying Model-Free Efficiency and Model-Based Representations via Latent Dynamics'
|
||||||
|
WITHDRAWAL_TITLE = 'Cognitive Heterogeneity and Behavioral Biases in Multi-Stage Supply Chains: Evidence from LLM-Based Agent Simulations'
|
||||||
|
|
||||||
|
|
||||||
|
def parse_authors(paragraph):
|
||||||
|
font = one([n for n in paragraph.walk() if n.tag == 'font' and
|
||||||
|
n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
|
||||||
|
authors, affiliations = [], []
|
||||||
|
name, affiliation = '', ''
|
||||||
|
def flush():
|
||||||
|
nonlocal name, affiliation
|
||||||
|
if clean(name) or clean(affiliation):
|
||||||
|
if not clean(name):
|
||||||
|
raise ValueError('Affiliation without author')
|
||||||
|
authors.append(clean(name)); affiliations.append(clean(affiliation))
|
||||||
|
name, affiliation = '', ''
|
||||||
|
for child in font.children:
|
||||||
|
if isinstance(child, str):
|
||||||
|
name += child
|
||||||
|
elif child.tag == 'br':
|
||||||
|
flush()
|
||||||
|
elif child.tag == 'i':
|
||||||
|
affiliation += child.text()
|
||||||
|
else:
|
||||||
|
raise ValueError('Unreviewed author markup: '+child.tag)
|
||||||
|
flush()
|
||||||
|
if not authors:
|
||||||
|
raise ValueError('Empty author list')
|
||||||
|
return authors, affiliations
|
||||||
|
|
||||||
|
|
||||||
|
def parse_contents(html, evidence):
|
||||||
|
doc = Document(html)
|
||||||
|
if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
|
||||||
|
raise ValueError('Unexpected AAMAS proceedings title/year')
|
||||||
|
section = None
|
||||||
|
headers, records, excluded, held = [], [], [], []
|
||||||
|
counts, seen_pages, seen_pdf_ids = Counter(), set(), set()
|
||||||
|
for node in doc.root.walk():
|
||||||
|
if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
|
||||||
|
continue
|
||||||
|
anchors = [n.attrs['name'] for n in node.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
|
||||||
|
if anchors:
|
||||||
|
section = one(anchors, 'section anchor')
|
||||||
|
if clean(node.text()) != SECTIONS[section][0]:
|
||||||
|
raise ValueError('Publisher section heading differs from reviewed layout')
|
||||||
|
headers.append(section)
|
||||||
|
page_fonts = [n for n in node.walk() if n.tag == 'font' and
|
||||||
|
n.attrs.get('color', '').lower() == '#444444']
|
||||||
|
if not page_fonts:
|
||||||
|
continue
|
||||||
|
page_text = clean(one(page_fonts, 'page marker').text())
|
||||||
|
page_match = re.fullmatch(r'\(Page ([1-9][0-9]*)\)', page_text)
|
||||||
|
if not page_match or section not in SECTIONS:
|
||||||
|
raise ValueError('Invalid page or paper outside reviewed sections')
|
||||||
|
page = int(page_match[1])
|
||||||
|
if page in seen_pages:
|
||||||
|
raise ValueError('Repeated publisher page ID')
|
||||||
|
seen_pages.add(page)
|
||||||
|
title_node = one([n for n in node.walk() if n.tag == 'strong'], 'paper title')
|
||||||
|
title = clean(title_node.text())
|
||||||
|
if not title:
|
||||||
|
raise ValueError('Empty paper title')
|
||||||
|
authors, affiliations = parse_authors(node)
|
||||||
|
link = one([n for n in node.walk() if n.tag == 'a' and 'href' in n.attrs], 'paper link')
|
||||||
|
href = link.attrs['href']
|
||||||
|
title_nodes = {id(n) for n in title_node.walk()}
|
||||||
|
retractions = [clean(n.text()) for n in link.walk() if n.tag == 'i' and id(n) not in title_nodes]
|
||||||
|
heading, publication_format, track, _ = SECTIONS[section]
|
||||||
|
counts[section] += 1
|
||||||
|
metadata = {'publisher_page': page, 'section_url': URL, 'section_anchor': section,
|
||||||
|
'section_heading': heading, 'publication_format': publication_format,
|
||||||
|
'author_affiliations': affiliations, 'source_evidence': evidence,
|
||||||
|
'decision_date_precision': 'not_supplied',
|
||||||
|
'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
|
||||||
|
'publication_evidence': 'Published proceedings entry, not an original decision note.'}
|
||||||
|
withdrawn = retractions == ['Withdrawn June 8, 2026']
|
||||||
|
if withdrawn:
|
||||||
|
if (section, page, title, href) != ('1', 3178, WITHDRAWAL_TITLE, ''):
|
||||||
|
raise ValueError('Unreviewed withdrawal notice')
|
||||||
|
metadata.update({'publication_state': 'withdrawn',
|
||||||
|
'withdrawal_date': '2026-06-08',
|
||||||
|
'raw_publication_state': retractions[0],
|
||||||
|
'decision_date_precision': 'day',
|
||||||
|
'publication_evidence': 'Explicit dated publisher withdrawal notice, not an original review decision note.'})
|
||||||
|
elif retractions:
|
||||||
|
if (section, page, title, href, retractions) != ('R', 238, RETRACTION_TITLE, '', ['Retracted June 8, 2026']):
|
||||||
|
raise ValueError('Unreviewed retraction or publication status markup')
|
||||||
|
held.append({'source': 'aamas_proceedings', 'external_id': '2026/p238',
|
||||||
|
'title': title, 'authors': authors, 'year': 2026, 'track': track,
|
||||||
|
'url': URL+'#R', 'abstract': None, 'identifiers': {},
|
||||||
|
'observed_at': evidence['observed_at'], 'metadata': metadata,
|
||||||
|
'publication_state': 'retracted', 'retraction_date': '2026-06-08',
|
||||||
|
'raw_publication_state': retractions[0],
|
||||||
|
'hold_reason': 'Explicit retraction; not an accepted/withdrawn submission assertion. Dedicated decision modeling and identity review required.'})
|
||||||
|
continue
|
||||||
|
if not withdrawn:
|
||||||
|
match = PDF.fullmatch(href)
|
||||||
|
if not match or match[1] in seen_pdf_ids:
|
||||||
|
raise ValueError('Invalid or repeated publisher PDF ID')
|
||||||
|
seen_pdf_ids.add(match[1])
|
||||||
|
metadata.update({'publisher_document_id': match[1], 'paper_url': ROOT+'pdfs/'+match[1]+'.pdf'})
|
||||||
|
if section == 'K':
|
||||||
|
excluded.append({'publisher_page': page, 'title': title, 'authors': authors,
|
||||||
|
'metadata': metadata, 'reason': 'Publisher keynote, not a research submission.'})
|
||||||
|
continue
|
||||||
|
metadata['track_scope_note'] = ('Published format known; individual main/special submission track not established.'
|
||||||
|
if track == 'proceedings_unspecified' else 'Explicit publisher section.')
|
||||||
|
if section == 'A':
|
||||||
|
metadata['prior_submission_note'] = 'AAAI transfer section does not supply original submission IDs or individual prior decision notes; none inferred.'
|
||||||
|
if section == 'J':
|
||||||
|
metadata['track_scope_note'] = 'Journal presentation; not another independent journal review or research-track acceptance.'
|
||||||
|
records.append({'source': 'aamas_proceedings', 'external_id': '2026/p'+str(page),
|
||||||
|
'url': URL+'#'+section, 'title': title, 'abstract': None,
|
||||||
|
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': track, 'status': 'withdrawn' if withdrawn else 'accepted',
|
||||||
|
'observed_at': evidence['observed_at'],
|
||||||
|
'raw_decision': retractions[0] if withdrawn else 'listed in official AAMAS 2026 published proceedings '+heading,
|
||||||
|
'metadata': metadata,
|
||||||
|
'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]})
|
||||||
|
if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
|
||||||
|
raise ValueError('Publisher section inventory differs from reviewed 2026 layout')
|
||||||
|
if (len(records), len(excluded), len(held)) != (639, 3, 1):
|
||||||
|
raise ValueError('Unexpected accepted/keynote/retraction partition')
|
||||||
|
if Counter(r['status'] for r in records) != {'accepted': 638, 'withdrawn': 1}:
|
||||||
|
raise ValueError('Unexpected publisher publication-state partition')
|
||||||
|
return records, excluded, held
|
||||||
|
|
||||||
|
|
||||||
|
def read_page(cache, snapshots):
|
||||||
|
if bool(cache) == bool(snapshots):
|
||||||
|
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
||||||
|
if snapshots:
|
||||||
|
base = Path(snapshots).resolve()
|
||||||
|
frozen = json.loads((base/'receipts.json').read_text())['receipts']
|
||||||
|
r = one([r for r in frozen if r['url'] == URL], 'frozen response receipt')
|
||||||
|
filename = Path(r['file'])
|
||||||
|
if (filename.is_absolute() or '..' in filename.parts or
|
||||||
|
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
|
||||||
|
raise ValueError('Unsafe or unsuccessful publisher response')
|
||||||
|
data = (base/filename).read_bytes()
|
||||||
|
if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
|
||||||
|
raise ValueError('Frozen publisher response checksum/size mismatch')
|
||||||
|
observed = r['finished_at']
|
||||||
|
else:
|
||||||
|
data = fetch(URL, cache, 8*1024**2, delay=1)
|
||||||
|
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
|
||||||
|
observed = r['retrieved_at']
|
||||||
|
timestamp(observed)
|
||||||
|
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
||||||
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
|
||||||
|
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, cache=None, snapshots=None):
|
||||||
|
data, evidence = read_page(cache, snapshots)
|
||||||
|
records, excluded, held = parse_contents(data.decode('utf-8'), evidence)
|
||||||
|
grouped = defaultdict(list)
|
||||||
|
for row in records:
|
||||||
|
grouped[row['track']].append(row)
|
||||||
|
out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.aamas2026-', dir=out) as temporary:
|
||||||
|
stage = Path(temporary); (stage/'evidence').mkdir()
|
||||||
|
(stage/evidence['file']).write_bytes(data)
|
||||||
|
for track, group in sorted(grouped.items()):
|
||||||
|
p = stage/f'aamas-proceedings-2026-{track}.jsonl'
|
||||||
|
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
||||||
|
manifest = {'source': 'aamas_proceedings', 'parser_version': 3, 'venue': 'AAMAS', 'year': 2026,
|
||||||
|
'track': track, 'count': len(group), 'file': str(out/p.name),
|
||||||
|
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(group),
|
||||||
|
'status_counts': dict(Counter(r['status'] for r in group)),
|
||||||
|
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
|
||||||
|
'note': 'Reviewed TOC partition excluding keynote/retracted entries; identity and full submission history remain unverified.'}
|
||||||
|
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
files.append(manifest)
|
||||||
|
result = {'source': 'aamas_proceedings', 'year': 2026, 'finished_at': timestamp(),
|
||||||
|
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
|
||||||
|
'excluded_publications': excluded, 'held_publications': held,
|
||||||
|
'status_counts': dict(Counter(r['status'] for r in records)),
|
||||||
|
'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'note': 'Retraction evidence held separately; publication is not an original decision note, and absence is not rejection.'}
|
||||||
|
report = 'aamas-proceedings-2026.json'
|
||||||
|
(stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
(out/'evidence').mkdir(exist_ok=True)
|
||||||
|
for p in (stage/'evidence').iterdir():
|
||||||
|
p.replace(out/'evidence'/p.name)
|
||||||
|
for p in stage.iterdir():
|
||||||
|
if p.is_file() and p.name != report:
|
||||||
|
p.replace(out/p.name)
|
||||||
|
(stage/report).replace(out/report)
|
||||||
|
return result
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'corl_program_archive'
|
||||||
|
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||||
|
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||||
|
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||||
|
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||||
|
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||||
|
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||||
|
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||||
|
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||||
|
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||||
|
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||||
|
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def forum_id(url):
|
||||||
|
if not isinstance(url, str):
|
||||||
|
raise ValueError('Forum URL must be text')
|
||||||
|
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||||
|
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||||
|
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||||
|
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||||
|
raise ValueError('Invalid original forum identity')
|
||||||
|
return q['id'][0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||||
|
if len(embeds) != 1:
|
||||||
|
raise ValueError('Expected one official embedded program')
|
||||||
|
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected program edition')
|
||||||
|
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||||
|
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||||
|
raise ValueError('Reviewed program census changed')
|
||||||
|
papers = {}; sessions = {}; occurrences = []
|
||||||
|
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||||
|
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||||
|
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||||
|
raise ValueError('Unexpected or repeated presentation session')
|
||||||
|
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||||
|
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||||
|
if len(headings) != 1:
|
||||||
|
raise ValueError('Ambiguous session heading')
|
||||||
|
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||||
|
seen = set()
|
||||||
|
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||||
|
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||||
|
if fid in seen or not title:
|
||||||
|
raise ValueError('Repeated or empty paper in session')
|
||||||
|
seen.add(fid)
|
||||||
|
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||||
|
if paper['title'] != title:
|
||||||
|
raise ValueError('Conflicting official titles share a forum')
|
||||||
|
event = {'session_id': sid, **sessions[sid]}
|
||||||
|
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||||
|
counts = Counter(kind for _, _, kind in occurrences)
|
||||||
|
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||||
|
raise ValueError('Incomplete official program')
|
||||||
|
for paper in papers.values():
|
||||||
|
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||||
|
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||||
|
raise ValueError('Invalid poster/talk multiplicity')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def join_metadata(program, archive):
|
||||||
|
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||||
|
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||||
|
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||||
|
or not isinstance(archive.get('papers'), list)):
|
||||||
|
raise ValueError('Unpinned public metadata provenance')
|
||||||
|
indexed = {}
|
||||||
|
for row in archive['papers']:
|
||||||
|
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||||
|
raise ValueError('Unexpected metadata/profile fields')
|
||||||
|
fid = forum_id(row.get('openreview'))
|
||||||
|
if fid != row.get('id') or fid in indexed:
|
||||||
|
raise ValueError('Repeated or mismatched archive forum')
|
||||||
|
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||||
|
raise ValueError('Field source belongs to another submission')
|
||||||
|
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||||
|
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||||
|
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||||
|
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||||
|
raise ValueError('Incomplete scholarly metadata')
|
||||||
|
indexed[fid] = row
|
||||||
|
if set(indexed) != set(program):
|
||||||
|
raise ValueError('Official and archive forum sets differ')
|
||||||
|
return indexed
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, archive_snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||||
|
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||||
|
if official.is_symlink() or filtered.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||||
|
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||||
|
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Missing official source receipt')
|
||||||
|
receipt = matches[0]; observed = receipt['at']
|
||||||
|
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||||
|
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid official receipt')
|
||||||
|
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
sources = {}
|
||||||
|
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||||
|
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||||
|
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||||
|
rows = []; variants = []
|
||||||
|
for fid in sorted(program):
|
||||||
|
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||||
|
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||||
|
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||||
|
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||||
|
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||||
|
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||||
|
'observed_at': observed, 'tags': [], 'metadata': {
|
||||||
|
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||||
|
'source_evidence': sources, 'archive_revision': REVISION,
|
||||||
|
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||||
|
if len(variants) != 34:
|
||||||
|
raise ValueError('Reviewed title-variant census changed')
|
||||||
|
path = out/'corl-program-archive-2026-main.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||||
|
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--out', type=Path, required=True)
|
||||||
|
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||||
|
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -0,0 +1,228 @@
|
|||||||
|
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import fcntl
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||||
|
|
||||||
|
SOURCE = 'eccv_program'
|
||||||
|
ORIGIN = 'https://eccv.ecva.net'
|
||||||
|
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||||
|
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||||
|
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected one reviewed metadata field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, value):
|
||||||
|
return value in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||||
|
raise ValueError('Unexpected ECCV program edition')
|
||||||
|
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||||
|
raise ValueError('Reviewed provisional-publication notice changed')
|
||||||
|
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||||
|
papers = {}; occurrences = 0
|
||||||
|
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 3:
|
||||||
|
raise ValueError('Unexpected program columns')
|
||||||
|
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||||
|
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||||
|
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||||
|
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||||
|
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||||
|
if not row['title'] or not authors:
|
||||||
|
raise ValueError('Incomplete public program fields')
|
||||||
|
occurrences += 1
|
||||||
|
if pid in papers:
|
||||||
|
if papers[pid]['listing'] != row:
|
||||||
|
raise ValueError('Conflicting entries share a program ID')
|
||||||
|
papers[pid]['row_ordinals'].append(occurrences)
|
||||||
|
else:
|
||||||
|
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||||
|
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||||
|
raise ValueError('Incomplete reviewed program census')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(raw, url, program):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||||
|
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||||
|
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||||
|
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||||
|
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||||
|
authors = ld.get('author')
|
||||||
|
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||||
|
or not title or ld.get('name') != title or not abstract
|
||||||
|
or not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||||
|
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||||
|
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||||
|
names = [a['name'] for a in authors]
|
||||||
|
if ' ⋅ '.join(names) != authors_literal:
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||||
|
if title != program['listing']['title']:
|
||||||
|
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||||
|
if authors_literal != program['listing']['authors_literal']:
|
||||||
|
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||||
|
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||||
|
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup}
|
||||||
|
|
||||||
|
|
||||||
|
def official_snapshot(root):
|
||||||
|
root = Path(root); manifest = root/'manifest.json'
|
||||||
|
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed official snapshot manifest changed')
|
||||||
|
m = json.loads(manifest.read_bytes())
|
||||||
|
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Official snapshot requires a timezone')
|
||||||
|
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||||
|
raise ValueError('Official source set changed')
|
||||||
|
pages = {}; evidence = {}
|
||||||
|
for r in m['sources']:
|
||||||
|
name = r['file']; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe official snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||||
|
raise ValueError('Invalid official source receipt')
|
||||||
|
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||||
|
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||||
|
return m, pages, evidence, program
|
||||||
|
|
||||||
|
|
||||||
|
def paper_snapshots(root, program):
|
||||||
|
root = Path(root); path = root/'manifest.json'
|
||||||
|
if root.is_symlink() or path.is_symlink():
|
||||||
|
raise ValueError('Paper snapshot symlinks rejected')
|
||||||
|
if path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Oversized paper manifest')
|
||||||
|
m = json.loads(path.read_bytes()); result = {}
|
||||||
|
for r in m['records']:
|
||||||
|
pid = r['paper_id']
|
||||||
|
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||||
|
raise ValueError('Foreign or repeated paper snapshot')
|
||||||
|
p = root/r['file']
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe paper snapshot')
|
||||||
|
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||||
|
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||||
|
if (paper_snapshots_dir is None) == (cache is None):
|
||||||
|
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||||
|
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||||
|
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
lock = None
|
||||||
|
try:
|
||||||
|
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||||
|
directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in pages.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||||
|
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||||
|
client = None
|
||||||
|
if cache is not None:
|
||||||
|
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||||
|
raise ValueError('Cache lock symlinks rejected')
|
||||||
|
lock = (cache/'.collector.lock').open('a+')
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
client = PublicHTMLCache(cache, ORIGIN)
|
||||||
|
rows = []; missing = []; fetched = 0; reused = 0
|
||||||
|
for pid in sorted(program, key=int):
|
||||||
|
p = program[pid]
|
||||||
|
if snapshots is not None:
|
||||||
|
response = snapshots.get(pid)
|
||||||
|
else:
|
||||||
|
response = client.cached(p['url'])
|
||||||
|
if response is not None:
|
||||||
|
reused += 1
|
||||||
|
elif max_new_pages is None or fetched < max_new_pages:
|
||||||
|
response = client.get(p['url']); fetched += 1
|
||||||
|
if response is None:
|
||||||
|
missing.append(pid); continue
|
||||||
|
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||||
|
keyword = p['listing']['keywords_literal']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||||
|
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||||
|
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||||
|
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||||
|
'evidence_url': ACCEPTED,
|
||||||
|
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||||
|
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||||
|
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||||
|
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||||
|
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||||
|
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||||
|
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||||
|
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||||
|
if len(rows) % 10 == 0:
|
||||||
|
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||||
|
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||||
|
files = []
|
||||||
|
if rows:
|
||||||
|
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||||
|
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||||
|
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||||
|
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||||
|
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||||
|
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||||
|
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if lock is not None:
|
||||||
|
lock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--official-snapshots', required=True)
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument('--paper-snapshots')
|
||||||
|
group.add_argument('--cache')
|
||||||
|
parser.add_argument('--max-new-pages', type=int)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||||
|
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icaps_official_program'
|
||||||
|
ROOT = 'https://icaps26.icaps-conference.org/'
|
||||||
|
PAGES = {
|
||||||
|
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
|
||||||
|
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
|
||||||
|
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
|
||||||
|
}
|
||||||
|
MAIN = 'Papers accepted to the Main Tracks'
|
||||||
|
PREVIOUS = 'Previously Published Papers'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def authors_from_literal(literal):
|
||||||
|
# This frozen listing uses commas between authors and a final "and".
|
||||||
|
# Preserve the complete literal too; never infer person-name aliases.
|
||||||
|
if literal.count(' and ') != 1 or ', and ' in literal:
|
||||||
|
raise ValueError('Unreviewed author-list punctuation')
|
||||||
|
first, last = literal.rsplit(' and ', 1)
|
||||||
|
authors = first.split(', ') + [last]
|
||||||
|
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
|
||||||
|
raise ValueError('Incomplete or ambiguous author list')
|
||||||
|
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
|
||||||
|
raise ValueError('Author literal cannot be reconstructed')
|
||||||
|
return authors
|
||||||
|
|
||||||
|
|
||||||
|
def parse_listing(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
|
||||||
|
if headings != ['List of Papers Accepted to ICAPS 2026']:
|
||||||
|
raise ValueError('Unexpected official conference edition')
|
||||||
|
section = None; sections = []; rows = []; seen = set()
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'h2':
|
||||||
|
section = clean(n.text()); sections.append(section)
|
||||||
|
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
|
||||||
|
continue
|
||||||
|
if section not in (MAIN, PREVIOUS):
|
||||||
|
raise ValueError('Paper belongs to an unreviewed section')
|
||||||
|
# Require the reviewed title and complete-author blocks; fail on layout drift.
|
||||||
|
blocks = [b for b in n.children if isinstance(b, Node)]
|
||||||
|
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
|
||||||
|
raise ValueError('Changed paper metadata layout')
|
||||||
|
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
|
||||||
|
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
|
||||||
|
raise ValueError('Changed paper title/link layout')
|
||||||
|
title = clean(titles[0].text()); literal = clean(blocks[1].text())
|
||||||
|
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
|
||||||
|
raise ValueError('Empty or repeated paper title')
|
||||||
|
seen.add(title.casefold())
|
||||||
|
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
|
||||||
|
'authors_literal': literal})
|
||||||
|
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
|
||||||
|
raise ValueError('Incomplete reviewed official listing')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
root = Path(snapshots)
|
||||||
|
manifest_path = root/'manifest.json'
|
||||||
|
if manifest_path.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
receipt = json.loads(manifest_path.read_bytes())
|
||||||
|
observed = receipt['at']
|
||||||
|
if (receipt.get('status') != 'collected_not_parsed_not_imported'
|
||||||
|
or receipt.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid reviewed snapshot receipt')
|
||||||
|
receipts = receipt['sources']
|
||||||
|
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
|
||||||
|
raise ValueError('Unexpected snapshot source set')
|
||||||
|
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (url, expected) in PAGES.items():
|
||||||
|
matches = [r for r in receipts if r['file'] == name]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Repeated snapshot receipt')
|
||||||
|
r = matches[0]; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
if r['url'] != url or r['final_url'] != url:
|
||||||
|
raise ValueError('Unexpected snapshot URL or redirect')
|
||||||
|
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
|
||||||
|
bodies[name] = raw.decode('utf-8')
|
||||||
|
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
|
||||||
|
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
|
||||||
|
for entry in entries:
|
||||||
|
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if entry['section'] == PREVIOUS:
|
||||||
|
previous.append({**entry, 'entry_key': entry_key,
|
||||||
|
'presentation_kind': 'previously_published_research',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
|
||||||
|
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
|
||||||
|
continue
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
|
||||||
|
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
|
||||||
|
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
|
||||||
|
'metadata': {'official_listing': entry, 'source_evidence': evidence,
|
||||||
|
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
|
||||||
|
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
|
||||||
|
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
|
||||||
|
'abstract_availability': 'not_on_official_listing',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
|
||||||
|
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
|
||||||
|
'status': 'partial', 'missing_abstract_count': 90}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'previously-published-presentations.json'
|
||||||
|
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
|
||||||
|
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
|
||||||
|
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
|
||||||
|
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
|
||||||
|
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icaps_registry'
|
||||||
|
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
|
||||||
|
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
|
||||||
|
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
|
||||||
|
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def abstract_text(raw):
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise ValueError('Missing registry abstract')
|
||||||
|
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
|
||||||
|
if not match or re.search(r'<[^>]*>', match[1]):
|
||||||
|
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
|
||||||
|
text = clean(unescape(match[1]))
|
||||||
|
if not text:
|
||||||
|
raise ValueError('Empty registry abstract')
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(snapshot):
|
||||||
|
observed = snapshot['at']
|
||||||
|
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
|
||||||
|
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
|
||||||
|
or snapshot.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
|
||||||
|
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
|
||||||
|
raise ValueError('Unreviewed registry snapshot receipt')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for row in snapshot['records']:
|
||||||
|
doi = row.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if row.get('volume') == '35':
|
||||||
|
excluded.append(doi)
|
||||||
|
continue
|
||||||
|
if (row.get('volume') != '36' or row.get('issue') != '1'
|
||||||
|
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
|
||||||
|
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
|
||||||
|
or row.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or not isinstance(row.get('title'), list) or len(row['title']) != 1
|
||||||
|
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
|
||||||
|
raise ValueError('Unreviewed publication volume, DOI, type or title')
|
||||||
|
day, = row['published']['date-parts']
|
||||||
|
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
|
||||||
|
raise ValueError('Unreviewed publication date')
|
||||||
|
publication_date = date(*day).isoformat()
|
||||||
|
authors = row.get('authors')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
|
||||||
|
or not a.get('given') or not a.get('family') for a in authors)):
|
||||||
|
raise ValueError('Incomplete registry authors')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if (len(set(n.casefold() for n in names)) != len(names)
|
||||||
|
or any('@' in n or 'anonymous' in n.casefold() for n in names)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Unreviewed registry author order or identity')
|
||||||
|
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
|
||||||
|
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
|
||||||
|
if len(papers) != 90 or len(excluded) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed volume census')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
path = Path(snapshots)/'crossref-filtered.json'
|
||||||
|
if path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized registry snapshot')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != SNAPSHOT_SHA:
|
||||||
|
raise ValueError('Reviewed registry snapshot hash changed')
|
||||||
|
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
|
||||||
|
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
|
||||||
|
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
|
||||||
|
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'source_evidence': evidence,
|
||||||
|
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
|
||||||
|
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
|
||||||
|
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
|
||||||
|
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
|
||||||
|
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
|
||||||
|
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
|
||||||
|
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
||||||
|
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
SOURCE = 'rss_proceedings'
|
||||||
|
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
||||||
|
HEADING = 'Robotics: Science and Systems XXII'
|
||||||
|
PAGES = {'index.html': ROOT,
|
||||||
|
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
||||||
|
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
||||||
|
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(values):
|
||||||
|
if len(values) != 1:
|
||||||
|
raise ValueError('Expected one reviewed HTML field')
|
||||||
|
return values[0]
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(value):
|
||||||
|
return unicodedata.normalize('NFC', clean(value))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
||||||
|
policy = clean(Document(policy_html).root.text())
|
||||||
|
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
||||||
|
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
||||||
|
if text not in policy:
|
||||||
|
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
||||||
|
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
||||||
|
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
||||||
|
nodes = list(Document(index_html).root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
||||||
|
raise ValueError('Unexpected publisher edition')
|
||||||
|
index = {}
|
||||||
|
for row in (n for n in nodes if n.tag == 'tr'):
|
||||||
|
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
||||||
|
if not links:
|
||||||
|
continue
|
||||||
|
link = one(links); pid = int(link.attrs['href'][1:4])
|
||||||
|
if pid in index:
|
||||||
|
raise ValueError('Repeated publisher ID')
|
||||||
|
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
||||||
|
program = {}
|
||||||
|
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected accepted program')
|
||||||
|
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
||||||
|
raise ValueError('Unexpected program row')
|
||||||
|
pid = int(clean(cells[0].text()))
|
||||||
|
if pid in program:
|
||||||
|
raise ValueError('Repeated program ID')
|
||||||
|
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
||||||
|
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
||||||
|
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
||||||
|
if not preview or not full.startswith(preview):
|
||||||
|
raise ValueError('Author preview is not a prefix of complete author evidence')
|
||||||
|
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
||||||
|
if links != [f'/2026/program/papers/{pid}/']:
|
||||||
|
raise ValueError('Program display link changed')
|
||||||
|
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
||||||
|
'authors_preview': preview, 'session': clean(cells[1].text()),
|
||||||
|
'program_url': 'https://roboticsconference.org'+links[0]}
|
||||||
|
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
||||||
|
raise ValueError('Reviewed 210-entry source census changed')
|
||||||
|
return index, program
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(html, pid, index, program):
|
||||||
|
nodes = list(Document(html).root.walk()); meta = {}
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
||||||
|
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
||||||
|
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
||||||
|
day = one(meta.get('citation_publication_date', []))
|
||||||
|
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
||||||
|
raise ValueError('Unexpected publication date')
|
||||||
|
date.fromisoformat(day.replace('/', '-'))
|
||||||
|
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
||||||
|
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
||||||
|
or not authors or any(not clean(a) for a in authors)
|
||||||
|
or comparable(title) != comparable(index['title'])
|
||||||
|
or comparable(', '.join(authors)) != comparable(index['authors'])
|
||||||
|
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
||||||
|
raise ValueError('Publisher citation/index/visible metadata disagree')
|
||||||
|
pdf = one(meta.get('citation_pdf_url', []))
|
||||||
|
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
||||||
|
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
||||||
|
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
||||||
|
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
||||||
|
raise ValueError('BibTeX DOI/year mismatch')
|
||||||
|
variants = []
|
||||||
|
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
||||||
|
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
||||||
|
raise ValueError('Unreviewed complete author variant')
|
||||||
|
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
||||||
|
if comparable(title) != comparable(program['title']):
|
||||||
|
if (pid, title, program['title']) != (89,
|
||||||
|
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
||||||
|
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
||||||
|
raise ValueError('Unreviewed program title variant')
|
||||||
|
variants.append('reviewed_program_title_variant_not_erratum')
|
||||||
|
inside = False; parts = []; markers = []
|
||||||
|
for n in (n for n in nodes if n.tag == 'p'):
|
||||||
|
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
||||||
|
if 'Abstract:' in labels:
|
||||||
|
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
||||||
|
if value: parts.append(value)
|
||||||
|
elif 'Download:' in labels:
|
||||||
|
inside = False; markers.append('end')
|
||||||
|
elif inside:
|
||||||
|
parts.append(clean(n.text()))
|
||||||
|
if markers != ['start', 'end'] or not any(parts):
|
||||||
|
raise ValueError('Missing or changed publisher abstract layout')
|
||||||
|
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
||||||
|
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots=None, cache=None):
|
||||||
|
if (snapshots is None) == (cache is None):
|
||||||
|
raise ValueError('Specify snapshots or cache')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
||||||
|
out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
out.mkdir()
|
||||||
|
evidence = {}; receipts = {}
|
||||||
|
if snapshots is not None:
|
||||||
|
root = Path(snapshots)
|
||||||
|
try:
|
||||||
|
for r in json.loads((root/'receipts.json').read_bytes()):
|
||||||
|
name = r['file']
|
||||||
|
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
||||||
|
raise ValueError('Unsafe or repeated snapshot receipt')
|
||||||
|
receipts[name] = r
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
def page(name, url):
|
||||||
|
if snapshots is not None:
|
||||||
|
path = root/name
|
||||||
|
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
||||||
|
raw = path.read_bytes(); r = receipts[name]
|
||||||
|
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
||||||
|
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
||||||
|
raise ValueError('Snapshot hash/receipt mismatch')
|
||||||
|
at = r['at']
|
||||||
|
else:
|
||||||
|
raw = fetch(url, cache, limit=8*1024*1024)
|
||||||
|
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
||||||
|
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
||||||
|
at = sidecar['retrieved_at']
|
||||||
|
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Invalid source size/timezone')
|
||||||
|
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
||||||
|
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
||||||
|
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
||||||
|
return raw.decode('utf-8')
|
||||||
|
try:
|
||||||
|
pages = {name: page(name, url) for name, url in PAGES.items()}
|
||||||
|
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
||||||
|
records = []
|
||||||
|
for pid in sorted(index):
|
||||||
|
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
||||||
|
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
||||||
|
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
||||||
|
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
||||||
|
'observed_at': evidence[name]['observed_at'],
|
||||||
|
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
||||||
|
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
||||||
|
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
||||||
|
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
||||||
|
'policy_evidence': evidence['cfp.html'],
|
||||||
|
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
||||||
|
records.append(row)
|
||||||
|
path = out/'rss-proceedings-2026-main.jsonl'
|
||||||
|
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
||||||
|
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
||||||
|
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
p.add_argument('--out', required=True, type=Path)
|
||||||
|
g = p.add_mutually_exclusive_group(required=True)
|
||||||
|
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -4,7 +4,7 @@ import re
|
|||||||
from paper_atlas.io.publication_ids import mlsys_id
|
from paper_atlas.io.publication_ids import mlsys_id
|
||||||
|
|
||||||
|
|
||||||
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
|
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
|
||||||
|
|
||||||
|
|
||||||
def timestamp(value=None):
|
def timestamp(value=None):
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def checked_html(raw, receipt, url, limit):
|
||||||
|
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||||
|
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||||
|
or receipt.get('sha256') != digest(raw)
|
||||||
|
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||||
|
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||||
|
raise ValueError('Access challenge; stop without bypass')
|
||||||
|
return raw, receipt
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *args, **kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCache:
|
||||||
|
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||||
|
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||||
|
self.root = Path(directory)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self.root.is_symlink():
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
self.origin = origin
|
||||||
|
self.limit = limit
|
||||||
|
self.delay = delay
|
||||||
|
self.opener = urllib.request.build_opener(NoRedirect())
|
||||||
|
|
||||||
|
def paths(self, url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||||
|
raise ValueError('Unapproved public metadata origin')
|
||||||
|
key = digest(url.encode())
|
||||||
|
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||||
|
|
||||||
|
def cached(self, url):
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||||
|
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
if any(p.exists() for p in parts):
|
||||||
|
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||||
|
if failure.exists():
|
||||||
|
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||||
|
if body.exists() != receipt.exists():
|
||||||
|
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||||
|
if not body.exists():
|
||||||
|
return None
|
||||||
|
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||||
|
raise ValueError('Oversized cache entry')
|
||||||
|
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
cached = self.cached(url)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
try:
|
||||||
|
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||||
|
with self.opener.open(request, timeout=30) as response:
|
||||||
|
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||||
|
'content_type': response.headers.get('Content-Type', '')}
|
||||||
|
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||||
|
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||||
|
raw = response.read(self.limit+1)
|
||||||
|
if len(raw) > self.limit:
|
||||||
|
raise ValueError('Metadata response exceeds size limit')
|
||||||
|
expected = response.headers.get('Content-Length')
|
||||||
|
if expected is not None and int(expected) != len(raw):
|
||||||
|
raise ConnectionError('Incomplete metadata response')
|
||||||
|
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||||
|
checked_html(raw, rec, url, self.limit)
|
||||||
|
except (urllib.error.HTTPError, ValueError) as error:
|
||||||
|
if isinstance(error, urllib.error.HTTPError):
|
||||||
|
status = error.code
|
||||||
|
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||||
|
error.close()
|
||||||
|
else:
|
||||||
|
status = None
|
||||||
|
retry_after = None
|
||||||
|
# Access errors remain sticky until the operator verifies availability.
|
||||||
|
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||||
|
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||||
|
raise
|
||||||
|
part = body.with_suffix('.html.part')
|
||||||
|
part.write_bytes(raw); part.replace(body)
|
||||||
|
part = receipt.with_suffix('.json.part')
|
||||||
|
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||||
|
time.sleep(self.delay)
|
||||||
|
return raw, rec
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,167 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import sqlite3
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import aamas, aamas2026
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
|
||||||
|
FIXTURE = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/aamas2026-fixture.json.gz').read_bytes()))
|
||||||
|
EVIDENCE = {'url': aamas2026.URL, 'observed_at': FIXTURE['receipts']['receipts'][0]['finished_at']}
|
||||||
|
|
||||||
|
|
||||||
|
class Aamas2026Tests(unittest.TestCase):
|
||||||
|
def parse(self, html=None):
|
||||||
|
return aamas2026.parse_contents(FIXTURE['contents'] if html is None else html, EVIDENCE)
|
||||||
|
|
||||||
|
def snapshots(self, folder):
|
||||||
|
folder.mkdir()
|
||||||
|
(folder/'contents.html').write_text(FIXTURE['contents'])
|
||||||
|
(folder/'receipts.json').write_text(json.dumps(FIXTURE['receipts']))
|
||||||
|
return folder
|
||||||
|
|
||||||
|
def test_every_publisher_title_author_affiliation_and_page_preserved(self):
|
||||||
|
rows, excluded, held = self.parse()
|
||||||
|
self.assertEqual((len(rows), len(excluded), len(held)), (639, 3, 1))
|
||||||
|
expected = {r['publisher_page']: r for r in FIXTURE['expected_entries']}
|
||||||
|
self.assertEqual(len(expected), 643)
|
||||||
|
all_rows = rows+excluded+held
|
||||||
|
self.assertEqual({r['metadata']['publisher_page'] for r in all_rows}, set(expected))
|
||||||
|
for r in all_rows:
|
||||||
|
e = expected[r['metadata']['publisher_page']]
|
||||||
|
self.assertEqual(r['title'], e['title'])
|
||||||
|
self.assertEqual(r['authors'], e['authors'])
|
||||||
|
self.assertEqual(r['metadata']['author_affiliations'], e['affiliations'])
|
||||||
|
self.assertEqual(r['metadata']['section_heading'], e['section'])
|
||||||
|
for r in rows:
|
||||||
|
self.assertIsNone(r['abstract'])
|
||||||
|
self.assertEqual(r['identifiers'], {})
|
||||||
|
if r['external_id'] != '2026/p3178':
|
||||||
|
self.assertEqual(r['status'], 'accepted')
|
||||||
|
self.assertNotIn('decision_date', r)
|
||||||
|
|
||||||
|
def test_explicit_withdrawal_preserves_status_date_and_no_pdf(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
row, = [r for r in rows if r['external_id'] == '2026/p3178']
|
||||||
|
self.assertEqual(row['status'], 'withdrawn')
|
||||||
|
self.assertEqual(row['metadata']['withdrawal_date'], '2026-06-08')
|
||||||
|
self.assertNotIn('decision_date', row)
|
||||||
|
self.assertEqual(row['raw_decision'], 'Withdrawn June 8, 2026')
|
||||||
|
self.assertNotIn('paper_url', row['metadata'])
|
||||||
|
self.assertEqual(row['metadata']['publication_state'], 'withdrawn')
|
||||||
|
self.assertEqual(sum(r['status'] == 'accepted' for r in rows), 638)
|
||||||
|
|
||||||
|
def test_withdrawal_ingestion_retains_date_precision_without_invented_acceptance(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
row, = [r for r in rows if r['external_id'] == '2026/p3178']
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); path = root/'withdrawn.jsonl'; db = root/'sample.sqlite'
|
||||||
|
path.write_text(json.dumps(row, ensure_ascii=False)+'\n')
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps({
|
||||||
|
'source': 'aamas_proceedings', 'list_url': aamas2026.URL,
|
||||||
|
'venue': 'AAMAS', 'year': 2026, 'track': row['track'],
|
||||||
|
'count': 1, 'status': 'partial', 'missing_abstract_count': 1,
|
||||||
|
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}))
|
||||||
|
ingest_file(db, path)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT status,raw_value,decision_date FROM decisions').fetchall(),
|
||||||
|
[('withdrawn', 'Withdrawn June 8, 2026', None)])
|
||||||
|
metadata = json.loads(c.execute('SELECT metadata_json FROM source_records').fetchone()[0])
|
||||||
|
self.assertEqual(metadata['withdrawal_date'], '2026-06-08')
|
||||||
|
self.assertEqual(metadata['decision_date_precision'], 'day')
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||||
|
|
||||||
|
def test_retraction_is_not_accepted_or_withdrawn_and_retains_date(self):
|
||||||
|
rows, _, held = self.parse()
|
||||||
|
self.assertNotIn('2026/p238', {r['external_id'] for r in rows})
|
||||||
|
h, = held
|
||||||
|
self.assertEqual(h['publication_state'], 'retracted')
|
||||||
|
self.assertEqual(h['retraction_date'], '2026-06-08')
|
||||||
|
self.assertEqual(h['raw_publication_state'], 'Retracted June 8, 2026')
|
||||||
|
self.assertNotIn('status', h)
|
||||||
|
self.assertNotIn('paper_url', h['metadata'])
|
||||||
|
self.assertEqual(h['metadata']['source_evidence'], EVIDENCE)
|
||||||
|
|
||||||
|
def test_tracks_formats_and_prior_decisions_are_not_conflated(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
counts = {t: sum(r['track'] == t for r in rows) for t in {r['track'] for r in rows}}
|
||||||
|
self.assertEqual(counts, {'proceedings_unspecified': 522, 'aaai_transfer': 19,
|
||||||
|
'blue_sky': 17, 'doctoral': 34, 'demo': 44, 'jaamas': 3})
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row['tags'][0]['namespace'], 'aamas_proceedings:publication_format')
|
||||||
|
if row['track'] == 'aaai_transfer':
|
||||||
|
self.assertIn('none inferred', row['metadata']['prior_submission_note'])
|
||||||
|
self.assertNotIn('decision_events', row)
|
||||||
|
if row['track'] == 'jaamas':
|
||||||
|
self.assertEqual(row['metadata']['publication_format'], 'journal_presentation')
|
||||||
|
self.assertIn('not another independent', row['metadata']['track_scope_note'])
|
||||||
|
|
||||||
|
def test_refuses_changed_year_section_page_link_author_and_retraction(self):
|
||||||
|
html = FIXTURE['contents']
|
||||||
|
replacements = [('AAMAS 2026:', 'AAMAS 2027:'),
|
||||||
|
('name="R" id="R"></a><font color="#010101">Research Paper Track',
|
||||||
|
'name="R" id="R"></a><font color="#010101">Workshop Track'),
|
||||||
|
('(Page 4)', '(Page 1)'),
|
||||||
|
('../pdfs/HQQZ1937.pdf', 'https://other.example/paper.pdf'),
|
||||||
|
('Michael Wooldridge <i>', '<i>'),
|
||||||
|
('Retracted June 8, 2026', 'Withdrawn June 8, 2026')]
|
||||||
|
for old, new in replacements:
|
||||||
|
self.assertIn(old, html)
|
||||||
|
with self.subTest(change=old), self.assertRaises(ValueError):
|
||||||
|
self.parse(html.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_frozen_collection_is_partial_with_full_held_evidence(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2026, 'fetch', side_effect=AssertionError('No HTTP')):
|
||||||
|
root = Path(tmp)
|
||||||
|
report = aamas.collect(2026, root/'out', snapshots=self.snapshots(root/'source'))
|
||||||
|
self.assertEqual(report['records'], 639)
|
||||||
|
self.assertEqual(report['status_counts'], {'accepted': 638, 'withdrawn': 1})
|
||||||
|
self.assertEqual(len(report['files']), 6)
|
||||||
|
self.assertEqual(len(report['held_publications']), 1)
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
for m in report['files']:
|
||||||
|
self.assertEqual(m['status'], 'partial')
|
||||||
|
self.assertEqual(m['missing_abstract_count'], m['count'])
|
||||||
|
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
|
||||||
|
self.assertTrue((root/'out/aamas-proceedings-2026.json').is_file())
|
||||||
|
|
||||||
|
def test_frozen_response_rejects_tamper_unsafe_path_status_and_symlink(self):
|
||||||
|
for mode in ('tamper', 'path', 'status', 'symlink'):
|
||||||
|
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); source = self.snapshots(root/'source')
|
||||||
|
r = json.loads((source/'receipts.json').read_text())
|
||||||
|
if mode == 'tamper':
|
||||||
|
(source/'contents.html').write_text('bad')
|
||||||
|
elif mode == 'path':
|
||||||
|
r['receipts'][0]['file'] = '../outside.html'
|
||||||
|
elif mode == 'status':
|
||||||
|
r['receipts'][0]['status'] = 403
|
||||||
|
else:
|
||||||
|
(root/'outside.html').write_text(FIXTURE['contents'])
|
||||||
|
(source/'contents.html').unlink()
|
||||||
|
(source/'contents.html').symlink_to(root/'outside.html')
|
||||||
|
(source/'receipts.json').write_text(json.dumps(r))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
aamas.collect(2026, root/'out', snapshots=source)
|
||||||
|
self.assertFalse((root/'out/aamas-proceedings-2026.json').exists())
|
||||||
|
|
||||||
|
def test_live_mode_fetches_only_html_and_preserves_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); cache = root/'cache'; cache.mkdir()
|
||||||
|
receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': aamas2026.URL}
|
||||||
|
(cache/hashlib.sha256(aamas2026.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
|
||||||
|
with patch.object(aamas2026, 'fetch', return_value=FIXTURE['contents'].encode()) as fetch:
|
||||||
|
report = aamas.collect(2026, root/'out', cache=cache)
|
||||||
|
fetch.assert_called_once_with(aamas2026.URL, cache, 8*1024**2, delay=1)
|
||||||
|
self.assertEqual(report['source_receipts'][0]['response_receipt'], receipt)
|
||||||
|
|
||||||
|
def test_frozen_fixture_is_original_hashed_response(self):
|
||||||
|
r, = FIXTURE['receipts']['receipts']
|
||||||
|
self.assertEqual(hashlib.sha256(FIXTURE['contents'].encode()).hexdigest(), r['sha256'])
|
||||||
|
self.assertEqual(r['bytes'], len(FIXTURE['contents'].encode()))
|
||||||
@@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase):
|
|||||||
aamas.collect(2012, root/'out', snapshots=source)
|
aamas.collect(2012, root/'out', snapshots=source)
|
||||||
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
aamas.collect(2026, root/'future', snapshots=source)
|
aamas.collect(2027, root/'future', snapshots=source)
|
||||||
self.assertFalse((root/'future').exists())
|
self.assertFalse((root/'future').exists())
|
||||||
|
|
||||||
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
||||||
|
|||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import corl2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class CoRL2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def test_official_census_and_presentation_dedup(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html'])
|
||||||
|
self.assertEqual(len(program), 687)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||||
|
|
||||||
|
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||||
|
html = self.bundle['official_html']
|
||||||
|
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||||
|
self.assertIn(title, html)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||||
|
|
||||||
|
def test_foreign_forum_source_is_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_unknown_profile_fields_are_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['authorids'] = ['profile']
|
||||||
|
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_source_sets_must_agree(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'].pop()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||||
|
official.mkdir(); archive.mkdir()
|
||||||
|
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||||
|
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||||
|
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||||
|
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||||
|
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||||
|
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||||
|
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||||
|
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
(official/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertTrue((root/'failed/failed.json').exists())
|
||||||
|
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_forum_identity_validation(self):
|
||||||
|
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.forum_id(bad)
|
||||||
@@ -0,0 +1,104 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import eccv2026 as c
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||||
|
|
||||||
|
|
||||||
|
class ECCV2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||||
|
for name, raw in self.bundle['official_pages'].items():
|
||||||
|
(official/name).write_text(raw)
|
||||||
|
(official/'manifest.json').write_text(self.bundle['official_manifest'])
|
||||||
|
for pid, raw in self.bundle['paper_pages'].items():
|
||||||
|
(paper/(pid+'.html')).write_text(raw)
|
||||||
|
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||||
|
return official, paper
|
||||||
|
|
||||||
|
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||||
|
self.assertEqual(len(self.program), 2834)
|
||||||
|
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||||
|
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||||
|
item = self.program[gold['paper_id']]
|
||||||
|
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||||
|
self.assertIn(ordinal, item['row_ordinals'])
|
||||||
|
|
||||||
|
def test_conflicting_duplicate_not_silently_folded(self):
|
||||||
|
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||||
|
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||||
|
|
||||||
|
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||||
|
html = self.bundle['official_pages']['accepted.html']
|
||||||
|
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||||
|
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_public_page_fields_match_independent_sample_review(self):
|
||||||
|
for gold in self.bundle['paper_gold']['records']:
|
||||||
|
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||||
|
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||||
|
|
||||||
|
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||||
|
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_paper(changed, url, self.program['4048'])
|
||||||
|
|
||||||
|
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||||
|
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||||
|
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||||
|
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||||
|
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||||
|
|
||||||
|
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root)
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||||
|
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||||
|
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
|
||||||
|
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||||
|
for r in self.bundle['paper_manifest']['records']:
|
||||||
|
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||||
|
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||||
|
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||||
|
|
||||||
|
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_structured_math_markup_requires_explicit_review(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']
|
||||||
|
self.assertIn('<p>Proactive tamper', raw)
|
||||||
|
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||||
|
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||||
|
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||||
|
|
||||||
|
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCacheTests(unittest.TestCase):
|
||||||
|
def receipt(self, raw):
|
||||||
|
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||||
|
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||||
|
|
||||||
|
def test_old_cache_reused_without_network_or_expiry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||||
|
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||||
|
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||||
|
for kind in ('hash','orphan','part'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||||
|
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||||
|
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||||
|
c.opener = MagicMock()
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||||
|
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||||
|
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||||
|
self.assertEqual(c.opener.open.call_count,1)
|
||||||
|
|
||||||
|
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||||
|
for kind in ('redirect','type','challenge','size'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||||
|
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||||
|
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||||
|
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||||
|
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||||
|
if kind=='size':r.read.return_value=b'x'*101
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
self.assertFalse(c.paths(URL)[0].exists())
|
||||||
|
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||||
|
|
||||||
|
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||||
|
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||||
|
|
||||||
|
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||||
|
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||||
|
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
|
||||||
|
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = URL
|
||||||
|
raw = b'<html>public metadata</html>'
|
||||||
|
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||||
|
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||||
|
self.assertEqual(c.opener.open.call_count, 1)
|
||||||
|
body, receipt, _ = c.paths(URL)
|
||||||
|
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||||
|
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||||
|
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||||
|
|
||||||
|
def test_changed_url_is_rejected_before_reading_body(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||||
|
response.headers = {'Content-Type':'text/html'}
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
response.read.assert_not_called()
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPS2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
root.mkdir()
|
||||||
|
for name, html in self.bundle['pages'].items():
|
||||||
|
(root/name).write_text(html)
|
||||||
|
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
|
||||||
|
return root
|
||||||
|
|
||||||
|
def test_full_listing_matches_independently_recorded_literals(self):
|
||||||
|
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
|
||||||
|
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
|
||||||
|
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
|
||||||
|
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
|
||||||
|
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
|
||||||
|
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
|
||||||
|
|
||||||
|
def test_incomplete_and_duplicate_listing_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
|
||||||
|
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
|
||||||
|
|
||||||
|
def test_changed_edition_or_section_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
|
||||||
|
html.replace(c.MAIN, 'Workshop Papers')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(changed)
|
||||||
|
|
||||||
|
def test_ambiguous_author_lists_not_guessed(self):
|
||||||
|
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.authors_from_literal(literal)
|
||||||
|
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
|
||||||
|
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
|
||||||
|
|
||||||
|
def test_collection_separates_presentations_and_unresolved_tracks(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
|
||||||
|
report = c.collect(out, snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
previous = json.loads((out/report['previously_published_file']).read_text())
|
||||||
|
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
|
||||||
|
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
|
||||||
|
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 90)
|
||||||
|
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['files'][0]['status'], 'partial')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, snapshots=src)
|
||||||
|
|
||||||
|
def test_tampering_retains_failure_without_success_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src')
|
||||||
|
(src/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
|
||||||
|
for kind in ('redirect', 'time', 'duplicate'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
|
||||||
|
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
|
||||||
|
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
|
||||||
|
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
|
||||||
|
p.write_text(json.dumps(r))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_snapshot_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps_registry2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPSRegistry2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
|
||||||
|
cls.snapshot = json.loads(cls.raw)
|
||||||
|
|
||||||
|
def test_exact_edition_not_publication_year_alone(self):
|
||||||
|
papers, excluded = c.registry_papers(self.snapshot)
|
||||||
|
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
|
||||||
|
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
|
||||||
|
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
|
||||||
|
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
|
||||||
|
|
||||||
|
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, set(original))
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; o = original[doi]
|
||||||
|
self.assertEqual(r['metadata']['registry_fields'], o)
|
||||||
|
self.assertEqual(r['title'], o['title'][0])
|
||||||
|
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
|
||||||
|
self.assertEqual(r['identifiers'], {'doi': doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
|
||||||
|
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(r['track'], 'main_tracks_unclassified')
|
||||||
|
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
|
||||||
|
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
|
||||||
|
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
|
||||||
|
self.assertIn('Stephen F. Smith', changed['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
|
||||||
|
def test_math_markup_is_not_flattened_as_plain_numbers(self):
|
||||||
|
self.assertEqual(c.abstract_text('<jats:p>A < B and C & D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
|
||||||
|
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.abstract_text(raw)
|
||||||
|
|
||||||
|
def test_invalid_or_repeated_doi_and_edition_rejected(self):
|
||||||
|
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
|
||||||
|
if kind == 'doi': r['DOI'] = '10.9999/foreign'
|
||||||
|
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
|
||||||
|
if kind == 'volume': r['volume'] = '37'
|
||||||
|
if kind == 'container': r['container-title'] = ['Other Conference']
|
||||||
|
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_missing_authors_and_changed_order_evidence_rejected(self):
|
||||||
|
for kind in ('missing', 'sequence', 'profile'):
|
||||||
|
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
|
||||||
|
if kind == 'missing': r['authors'][0].pop('family')
|
||||||
|
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_receipt_scope_and_timezone_rejected(self):
|
||||||
|
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
|
||||||
|
s = copy.deepcopy(self.snapshot); s[key] = value
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_tamper_failure_retained_and_success_not_published(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from paper_atlas.collectors import rss2026 as rss
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
|
||||||
|
|
||||||
|
class Rss2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
|
||||||
|
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
|
||||||
|
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
|
||||||
|
|
||||||
|
def lists(self,**changes):
|
||||||
|
pages={**self.pages,**changes}
|
||||||
|
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
|
||||||
|
|
||||||
|
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
|
||||||
|
report=rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
|
||||||
|
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
for row,gold in zip(rows,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
|
||||||
|
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
|
||||||
|
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
|
||||||
|
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
|
||||||
|
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
|
||||||
|
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
|
||||||
|
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'],0)
|
||||||
|
|
||||||
|
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
|
||||||
|
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
|
||||||
|
self.assertEqual(len(shortened),6)
|
||||||
|
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
|
||||||
|
self.assertEqual(len(p['authors']),55)
|
||||||
|
self.assertFalse(p['program_variants'])
|
||||||
|
|
||||||
|
def test_six_reviewed_name_variants_preserve_original_strings(self):
|
||||||
|
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
|
||||||
|
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
|
||||||
|
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
|
||||||
|
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
|
||||||
|
|
||||||
|
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
|
||||||
|
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
|
||||||
|
self.assertTrue(p['title'].startswith('RLinf-VLA'))
|
||||||
|
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
|
||||||
|
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
|
||||||
|
|
||||||
|
def test_unreviewed_author_or_title_variant_stops(self):
|
||||||
|
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unreviewed'):
|
||||||
|
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
|
||||||
|
|
||||||
|
def test_demo_policy_cannot_be_reused_from2025(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'policy'):
|
||||||
|
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
|
||||||
|
|
||||||
|
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
|
||||||
|
html=self.pages['p001.html']
|
||||||
|
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
|
||||||
|
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
|
||||||
|
self.assertIn(old,html)
|
||||||
|
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
|
||||||
|
|
||||||
|
def test_index_census_and_preview_tampering_stop(self):
|
||||||
|
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
|
||||||
|
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
|
||||||
|
|
||||||
|
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
|
||||||
|
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
|
||||||
|
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
|
||||||
|
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertFalse((self.root/'out/collection-report.json').exists())
|
||||||
|
self.assertTrue((self.root/'out/failed.json').exists())
|
||||||
|
|
||||||
|
def test_existing_output_and_unsafe_receipt_are_rejected(self):
|
||||||
|
out=self.root/'existing';out.mkdir()
|
||||||
|
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
|
||||||
|
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(receipts))
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
|
||||||
|
self.assertTrue((self.root/'unsafe/failed.json').exists())
|
||||||
|
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""RSS publisher/DBLP observations share one event without losing old decisions."""
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from paper_atlas.db.submissions import identities_for_row
|
||||||
|
from tests.test_aamas_publication_identity import row, ingest
|
||||||
|
|
||||||
|
DOI='10.15607/rss.2026.xxii.041'
|
||||||
|
|
||||||
|
def rss(source,external,**kwargs):
|
||||||
|
return row(source,external,venue='RSS',year=2026,track='main',
|
||||||
|
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
|
||||||
|
|
||||||
|
class RssPublicationIdentityTests(unittest.TestCase):
|
||||||
|
def test_only_verified_accepted_source_adds_publication_identity(self):
|
||||||
|
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
|
||||||
|
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
|
||||||
|
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
|
||||||
|
self.assertEqual(identities_for_row({**r,'status':status}),[])
|
||||||
|
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
|
||||||
|
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
|
||||||
|
|
||||||
|
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
|
||||||
|
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
|
||||||
|
for pair in ((a,b),(b,a)):
|
||||||
|
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'papers.sqlite'
|
||||||
|
for r in pair:ingest(db,root,r)
|
||||||
|
ingest(db,root,a)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
|
||||||
|
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
|
||||||
|
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||||
|
|
||||||
|
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'papers.sqlite'
|
||||||
|
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
|
||||||
|
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
|
||||||
|
ingest(db,root,old)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
before=c.execute('SELECT * FROM decisions').fetchall()
|
||||||
|
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
|
||||||
|
ingest(db,root,new)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.audits.source_identity import audit, classify, load_inputs
|
||||||
|
from paper_atlas.postgres.client import ReadConnection
|
||||||
|
|
||||||
|
|
||||||
|
def record(**updates):
|
||||||
|
row = dict(source='reviewed_program', external_id='2026/forum-a',
|
||||||
|
title='A sufficiently specific scientific paper title', authors=['Jane Doe', 'John Smith'],
|
||||||
|
venue='NeurIPS', year=2026, track='main', status='accepted', identifiers={},
|
||||||
|
source_record_hits=[], identifier_hits=[], title_candidate_ids=[])
|
||||||
|
row.update(updates)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def write_input(root, rows):
|
||||||
|
path = root / 'source.jsonl'
|
||||||
|
raw = b''.join((json.dumps(row) + '\n').encode() for row in rows)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest = {k: rows[0][k] for k in ('source', 'venue', 'year', 'track')}
|
||||||
|
manifest.update(count=len(rows), file_sha256=hashlib.sha256(raw).hexdigest(),
|
||||||
|
file='/ignored/untrusted/location.jsonl', status='partial')
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateTests(unittest.TestCase):
|
||||||
|
def test_duplicate_names_do_not_equal_distinct_author_multiset(self):
|
||||||
|
row = record(authors=['Jane Doe', 'Jane Doe'], title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': ['Jane Doe']}})
|
||||||
|
self.assertEqual(result['kind'], 'partial_author_overlap_requires_review')
|
||||||
|
self.assertIn('duplicate_normalized_author_names', result['flags'])
|
||||||
|
self.assertFalse(result['comparisons'][0]['nonempty_author_multiset_equal'])
|
||||||
|
|
||||||
|
def test_arxiv_author_representation_and_order_match_but_never_approve(self):
|
||||||
|
row = record(title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': [['Smith', 'John', ''], ['Doe', 'Jane', '']]}})
|
||||||
|
self.assertEqual(result['kind'], 'one_exact_title_author_multiset_requires_review')
|
||||||
|
self.assertEqual(result['decision'], 'not_approved_for_import')
|
||||||
|
|
||||||
|
def test_binding_conflict_overrides_existing_source(self):
|
||||||
|
row = record(source_record_hits=[{'paper_id': 7}], identifier_hits=[{'paper_id': 8}])
|
||||||
|
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in (7, 8)}
|
||||||
|
self.assertEqual(classify(row, entities)['kind'], 'conflicting_existing_bindings_hold')
|
||||||
|
|
||||||
|
def test_wrong_doi_binding_retains_author_title_disagreement(self):
|
||||||
|
row = record(identifier_hits=[{'paper_id': 7}])
|
||||||
|
result = classify(row, {7: {'title': 'Completely different work', 'authors': ['Somebody Else']}})
|
||||||
|
self.assertEqual(result['kind'], 'existing_identifier_binding_requires_review')
|
||||||
|
self.assertFalse(result['comparisons'][0]['normalized_title_equal'])
|
||||||
|
self.assertEqual(result['comparisons'][0]['author_name_overlap'], [])
|
||||||
|
|
||||||
|
def test_no_hit_is_not_novelty_and_missing_authors_never_agree(self):
|
||||||
|
self.assertEqual(classify(record(), {})['kind'], 'no_exact_match_not_proof_of_novelty')
|
||||||
|
row = record(authors=[], title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': []}})
|
||||||
|
self.assertEqual(result['kind'], 'no_author_agreement_hold')
|
||||||
|
|
||||||
|
def test_title_candidate_cap_is_explicit(self):
|
||||||
|
row = record(title_candidate_ids=list(range(21)))
|
||||||
|
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in range(21)}
|
||||||
|
self.assertTrue(classify(row, entities)['title_candidates_truncated'])
|
||||||
|
self.assertEqual(classify(row, entities)['kind'], 'title_candidate_limit_reached_hold')
|
||||||
|
|
||||||
|
|
||||||
|
class InputTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temp.cleanup)
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
|
||||||
|
def test_raw_line_hash_and_manifest_path_not_followed(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
rows, inputs = load_inputs([path])
|
||||||
|
self.assertEqual(rows[0]['raw_line_sha256'], hashlib.sha256(path.read_bytes()).hexdigest())
|
||||||
|
self.assertEqual(inputs[0]['count'], 1)
|
||||||
|
|
||||||
|
def test_hash_tamper_rejected_before_database_connection(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
path.write_bytes(path.read_bytes().replace(b'Jane Doe', b'Jane Roe'))
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash mismatch'):
|
||||||
|
audit([path], self.root / 'out', pg_service='test', pg_schema='research')
|
||||||
|
connection.assert_not_called()
|
||||||
|
|
||||||
|
def test_manifest_scope_and_count_tamper_and_duplicate_sources(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
manifest_path = path.with_suffix('.manifest.json')
|
||||||
|
manifest = json.loads(manifest_path.read_text())
|
||||||
|
for key, value in [('year', 2025), ('count', 2)]:
|
||||||
|
altered = dict(manifest, **{key: value})
|
||||||
|
manifest_path.write_text(json.dumps(altered))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
load_inputs([path])
|
||||||
|
manifest_path.write_text(json.dumps(manifest))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Duplicate source'):
|
||||||
|
load_inputs([path, path])
|
||||||
|
|
||||||
|
def test_existing_output_never_replayed_and_failure_not_completion(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
out = self.root / 'out'
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection', side_effect=RuntimeError('unavailable')):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
audit([path], out, pg_service='test', pg_schema='research')
|
||||||
|
self.assertTrue((out / 'failed.json').exists())
|
||||||
|
self.assertFalse((out / 'postgresql-preflight.json').exists())
|
||||||
|
with self.assertRaises(FileExistsError):
|
||||||
|
audit([path], out, pg_service='test', pg_schema='research')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the authorized remote PostgreSQL service')
|
||||||
|
class ReadOnlyIntegrationTests(unittest.TestCase):
|
||||||
|
def test_existing_binding_full_evidence_and_no_import_approval(self):
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
|
||||||
|
connection = ReadConnection(service, 'research')
|
||||||
|
try:
|
||||||
|
self.assertEqual(connection.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
||||||
|
hit = connection.execute("SELECT value,paper_id FROM identifiers WHERE scheme='arxiv' ORDER BY value LIMIT 1").fetchone()
|
||||||
|
self.assertIsNotNone(hit)
|
||||||
|
paper = connection.execute('SELECT title,authors_json FROM papers WHERE id=?', (hit['paper_id'],)).fetchone()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
path = write_input(root, [record(title=paper['title'], authors=json.loads(paper['authors_json']),
|
||||||
|
identifiers={'arxiv': hit['value']})])
|
||||||
|
report = audit([path], root / 'out', pg_service=service, pg_schema='research')
|
||||||
|
self.assertTrue(report['transaction_read_only'])
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertEqual(report['approved_for_import'], 0)
|
||||||
|
row, = report['entries']
|
||||||
|
self.assertIn(hit['paper_id'], [r['paper_id'] for r in row['identifier_hits']])
|
||||||
|
shard = report['entities'][str(hit['paper_id'])]
|
||||||
|
raw = (root / 'out' / shard['file']).read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), shard['sha256'])
|
||||||
|
packet = json.loads(gzip.decompress(raw))
|
||||||
|
self.assertEqual(packet['paper']['id'], hit['paper_id'])
|
||||||
|
self.assertEqual(packet['paper']['title'], paper['title'])
|
||||||
|
self.assertEqual(packet['paper']['authors_json'], paper['authors_json'])
|
||||||
|
self.assertIn('attempts', packet)
|
||||||
|
self.assertTrue((root / 'out/postgresql-preflight.json').is_file())
|
||||||
Reference in New Issue
Block a user