Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 06467835c7 | |||
| 1ea17c810d | |||
| 28255743ef | |||
| 59053fd4fb | |||
| b50136d5bc | |||
| 87a0689fcc | |||
| 9093a8ce25 | |||
| e19cb83cd6 | |||
| 4d2df2a826 | |||
| 19d03ced54 | |||
| c33f5422de | |||
| 1cfd44d8b6 | |||
| 5de7f6ff22 | |||
| b4e249e675 | |||
| 93f00d1a64 | |||
| 8b7bc6d871 | |||
| 7588a4e904 | |||
| 891a63d982 |
@@ -48,6 +48,63 @@ jobs:
|
||||
outside.mkdir()
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
target = sdm_snapshots/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(text.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
||||
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
||||
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
||||
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
||||
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
||||
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
||||
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
for name, html in icaps_fixture['pages'].items():
|
||||
(icaps_snapshots/name).write_text(html)
|
||||
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
||||
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
||||
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
||||
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
||||
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
||||
registry_snapshots = root/'icaps2026-registry-snapshots'
|
||||
registry_snapshots.mkdir()
|
||||
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
|
||||
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
|
||||
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
|
||||
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
|
||||
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||
rss_snapshots = root/'rss2026-snapshots'
|
||||
rss_snapshots.mkdir()
|
||||
for name, html in rss_fixture['pages'].items():
|
||||
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
corl_official = root/'corl2026-official'
|
||||
corl_archive = root/'corl2026-archive'
|
||||
corl_official.mkdir()
|
||||
corl_archive.mkdir()
|
||||
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -118,3 +118,15 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||
|
||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||
|
||||
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||
|
||||
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||
|
||||
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
||||
|
||||
### CoRL 2026 官方程序快照
|
||||
|
||||
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# CoRL 2026:官方程序与公开档案
|
||||
|
||||
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||
|
||||
```sh
|
||||
paper-atlas collect corl2026-program \
|
||||
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||
```
|
||||
|
||||
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||
|
||||
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||
|
||||
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||
|
||||
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||
|
||||
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||
|
||||
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||
|
||||
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
|
||||
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
|
||||
|
||||
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
|
||||
|
||||
### RSS 发表身份
|
||||
|
||||
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
|
||||
|
||||
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# ICAPS 2026 官方名单
|
||||
|
||||
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
|
||||
|
||||
```sh
|
||||
paper-atlas collect icaps2026-program \
|
||||
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
|
||||
--out /data/paper-index/collected/icaps2026-normalized-v1
|
||||
```
|
||||
|
||||
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
|
||||
|
||||
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
|
||||
|
||||
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
|
||||
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
|
||||
|
||||
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
|
||||
|
||||
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
|
||||
|
||||
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
|
||||
|
||||
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
|
||||
|
||||
## 独立注册元数据版本
|
||||
|
||||
```sh
|
||||
paper-atlas collect icaps2026-registry \
|
||||
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
|
||||
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
|
||||
```
|
||||
|
||||
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
|
||||
|
||||
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
|
||||
|
||||
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
|
||||
|
||||
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
|
||||
@@ -0,0 +1,21 @@
|
||||
# RSS 2026 论文集来源
|
||||
|
||||
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
|
||||
|
||||
```sh
|
||||
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
|
||||
--out /data/paper-index/collected/rss2026-native-v1
|
||||
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
|
||||
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
|
||||
--out /data/paper-index/collected/rss2026-native-replay-v1
|
||||
```
|
||||
|
||||
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
|
||||
|
||||
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
|
||||
|
||||
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
|
||||
|
||||
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
|
||||
|
||||
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
|
||||
@@ -0,0 +1,78 @@
|
||||
# SDM 2026:生产规范化与身份核查记录
|
||||
|
||||
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
||||
|
||||
## 已核实的生产输出
|
||||
|
||||
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||
|
||||
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||
|
||||
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||
|
||||
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||
|
||||
| 证据文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||
|
||||
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||
|
||||
## 当前数据库只读预检
|
||||
|
||||
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||
|
||||
| 候选类别 | 记录数 | 含义 |
|
||||
| --- | ---: | --- |
|
||||
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||
|
||||
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||
|
||||
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||
|
||||
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||
| --- | ---: | --- |
|
||||
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||
|
||||
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||
|
||||
## 需要保护的较早拒稿历史
|
||||
|
||||
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||
| --- | ---: | --- |
|
||||
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||
|
||||
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
|
||||
## 未精确匹配的补充检索
|
||||
|
||||
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||
|
||||
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||
|
||||
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||
|
||||
| 服务器恢复文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||
|
||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||
|
||||
## 下次恢复
|
||||
|
||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||
@@ -0,0 +1,46 @@
|
||||
# SDM 2026:论文集与日程版本
|
||||
|
||||
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
|
||||
|
||||
```sh
|
||||
paper-atlas collect sdm2026-proceedings \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/sdm2026-normalized-pass01
|
||||
```
|
||||
|
||||
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
|
||||
|
||||
## 两种输出
|
||||
|
||||
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
|
||||
|
||||
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
|
||||
|
||||
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
|
||||
|
||||
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
|
||||
|
||||
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
|
||||
|
||||
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
|
||||
|
||||
## 公开拒稿与访问限制
|
||||
|
||||
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
|
||||
|
||||
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
|
||||
|
||||
## 来源恢复与后续核查
|
||||
|
||||
| 冻结目录 | manifest SHA-256 |
|
||||
| --- | --- |
|
||||
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
|
||||
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
|
||||
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
|
||||
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
|
||||
|
||||
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||
|
||||
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||
|
||||
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||
@@ -23,9 +23,14 @@ COMMANDS = {
|
||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
SOURCE = 'corl_program_archive'
|
||||
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def comparable(text):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def forum_id(url):
|
||||
if not isinstance(url, str):
|
||||
raise ValueError('Forum URL must be text')
|
||||
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||
raise ValueError('Invalid original forum identity')
|
||||
return q['id'][0]
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||
if len(embeds) != 1:
|
||||
raise ValueError('Expected one official embedded program')
|
||||
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||
raise ValueError('Unexpected program edition')
|
||||
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||
raise ValueError('Reviewed program census changed')
|
||||
papers = {}; sessions = {}; occurrences = []
|
||||
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||
raise ValueError('Unexpected or repeated presentation session')
|
||||
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||
if len(headings) != 1:
|
||||
raise ValueError('Ambiguous session heading')
|
||||
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||
seen = set()
|
||||
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||
if fid in seen or not title:
|
||||
raise ValueError('Repeated or empty paper in session')
|
||||
seen.add(fid)
|
||||
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||
if paper['title'] != title:
|
||||
raise ValueError('Conflicting official titles share a forum')
|
||||
event = {'session_id': sid, **sessions[sid]}
|
||||
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||
counts = Counter(kind for _, _, kind in occurrences)
|
||||
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||
raise ValueError('Incomplete official program')
|
||||
for paper in papers.values():
|
||||
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||
raise ValueError('Invalid poster/talk multiplicity')
|
||||
return papers
|
||||
|
||||
|
||||
def join_metadata(program, archive):
|
||||
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||
or not isinstance(archive.get('papers'), list)):
|
||||
raise ValueError('Unpinned public metadata provenance')
|
||||
indexed = {}
|
||||
for row in archive['papers']:
|
||||
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||
raise ValueError('Unexpected metadata/profile fields')
|
||||
fid = forum_id(row.get('openreview'))
|
||||
if fid != row.get('id') or fid in indexed:
|
||||
raise ValueError('Repeated or mismatched archive forum')
|
||||
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||
raise ValueError('Field source belongs to another submission')
|
||||
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||
raise ValueError('Incomplete scholarly metadata')
|
||||
indexed[fid] = row
|
||||
if set(indexed) != set(program):
|
||||
raise ValueError('Official and archive forum sets differ')
|
||||
return indexed
|
||||
|
||||
|
||||
def collect(out, *, official_snapshots, archive_snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||
if official.is_symlink() or filtered.is_symlink():
|
||||
raise ValueError('Snapshot symlinks rejected')
|
||||
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||
raise ValueError('Reviewed snapshot hash changed')
|
||||
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||
if len(matches) != 1:
|
||||
raise ValueError('Missing official source receipt')
|
||||
receipt = matches[0]; observed = receipt['at']
|
||||
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid official receipt')
|
||||
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
sources = {}
|
||||
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||
rows = []; variants = []
|
||||
for fid in sorted(program):
|
||||
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||
'observed_at': observed, 'tags': [], 'metadata': {
|
||||
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||
'source_evidence': sources, 'archive_revision': REVISION,
|
||||
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||
'original_decision_signature_verified': False,
|
||||
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||
if len(variants) != 34:
|
||||
raise ValueError('Reviewed title-variant census changed')
|
||||
path = out/'corl-program-archive-2026-main.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||
'status': 'partial', 'missing_abstract_count': 0}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||
@@ -0,0 +1,155 @@
|
||||
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'icaps_official_program'
|
||||
ROOT = 'https://icaps26.icaps-conference.org/'
|
||||
PAGES = {
|
||||
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
|
||||
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
|
||||
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
|
||||
}
|
||||
MAIN = 'Papers accepted to the Main Tracks'
|
||||
PREVIOUS = 'Previously Published Papers'
|
||||
TRACK = 'main_tracks_unclassified'
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def authors_from_literal(literal):
|
||||
# This frozen listing uses commas between authors and a final "and".
|
||||
# Preserve the complete literal too; never infer person-name aliases.
|
||||
if literal.count(' and ') != 1 or ', and ' in literal:
|
||||
raise ValueError('Unreviewed author-list punctuation')
|
||||
first, last = literal.rsplit(' and ', 1)
|
||||
authors = first.split(', ') + [last]
|
||||
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
|
||||
raise ValueError('Incomplete or ambiguous author list')
|
||||
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
|
||||
raise ValueError('Author literal cannot be reconstructed')
|
||||
return authors
|
||||
|
||||
|
||||
def parse_listing(raw):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
|
||||
if headings != ['List of Papers Accepted to ICAPS 2026']:
|
||||
raise ValueError('Unexpected official conference edition')
|
||||
section = None; sections = []; rows = []; seen = set()
|
||||
for n in nodes:
|
||||
if n.tag == 'h2':
|
||||
section = clean(n.text()); sections.append(section)
|
||||
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
|
||||
continue
|
||||
if section not in (MAIN, PREVIOUS):
|
||||
raise ValueError('Paper belongs to an unreviewed section')
|
||||
# Require the reviewed title and complete-author blocks; fail on layout drift.
|
||||
blocks = [b for b in n.children if isinstance(b, Node)]
|
||||
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
|
||||
raise ValueError('Changed paper metadata layout')
|
||||
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
|
||||
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
|
||||
raise ValueError('Changed paper title/link layout')
|
||||
title = clean(titles[0].text()); literal = clean(blocks[1].text())
|
||||
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
|
||||
raise ValueError('Empty or repeated paper title')
|
||||
seen.add(title.casefold())
|
||||
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
|
||||
'authors_literal': literal})
|
||||
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
|
||||
raise ValueError('Incomplete reviewed official listing')
|
||||
return rows
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
root = Path(snapshots)
|
||||
manifest_path = root/'manifest.json'
|
||||
if manifest_path.is_symlink():
|
||||
raise ValueError('Snapshot symlinks rejected')
|
||||
receipt = json.loads(manifest_path.read_bytes())
|
||||
observed = receipt['at']
|
||||
if (receipt.get('status') != 'collected_not_parsed_not_imported'
|
||||
or receipt.get('database_written') is not False
|
||||
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid reviewed snapshot receipt')
|
||||
receipts = receipt['sources']
|
||||
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
|
||||
raise ValueError('Unexpected snapshot source set')
|
||||
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
|
||||
for name, (url, expected) in PAGES.items():
|
||||
matches = [r for r in receipts if r['file'] == name]
|
||||
if len(matches) != 1:
|
||||
raise ValueError('Repeated snapshot receipt')
|
||||
r = matches[0]; p = root/name
|
||||
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||
raise ValueError('Unsafe or oversized snapshot')
|
||||
raw = p.read_bytes()
|
||||
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
|
||||
raise ValueError('Reviewed snapshot hash changed')
|
||||
if r['url'] != url or r['final_url'] != url:
|
||||
raise ValueError('Unexpected snapshot URL or redirect')
|
||||
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
|
||||
bodies[name] = raw.decode('utf-8')
|
||||
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
|
||||
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
|
||||
for entry in entries:
|
||||
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||
if entry['section'] == PREVIOUS:
|
||||
previous.append({**entry, 'entry_key': entry_key,
|
||||
'presentation_kind': 'previously_published_research',
|
||||
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
|
||||
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
|
||||
continue
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
|
||||
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
|
||||
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
|
||||
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
|
||||
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
|
||||
'metadata': {'official_listing': entry, 'source_evidence': evidence,
|
||||
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
|
||||
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
|
||||
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
|
||||
'abstract_availability': 'not_on_official_listing',
|
||||
'original_decision_signature_verified': False,
|
||||
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
|
||||
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
|
||||
'status': 'partial', 'missing_abstract_count': 90}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
sidecar = out/'previously-published-presentations.json'
|
||||
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
|
||||
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
|
||||
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
|
||||
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
|
||||
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
|
||||
import argparse
|
||||
from datetime import datetime, date
|
||||
import hashlib
|
||||
from html import unescape
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import clean
|
||||
|
||||
SOURCE = 'crossref_icaps_registry'
|
||||
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
|
||||
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
|
||||
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
|
||||
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
|
||||
TRACK = 'main_tracks_unclassified'
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def abstract_text(raw):
|
||||
if not isinstance(raw, str):
|
||||
raise ValueError('Missing registry abstract')
|
||||
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
|
||||
if not match or re.search(r'<[^>]*>', match[1]):
|
||||
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
|
||||
text = clean(unescape(match[1]))
|
||||
if not text:
|
||||
raise ValueError('Empty registry abstract')
|
||||
return text
|
||||
|
||||
|
||||
def registry_papers(snapshot):
|
||||
observed = snapshot['at']
|
||||
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
|
||||
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
|
||||
or snapshot.get('database_written') is not False
|
||||
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
|
||||
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
|
||||
raise ValueError('Unreviewed registry snapshot receipt')
|
||||
papers = []; excluded = []; seen = set()
|
||||
for row in snapshot['records']:
|
||||
doi = row.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen:
|
||||
raise ValueError('Missing or repeated registry DOI')
|
||||
seen.add(doi)
|
||||
if row.get('volume') == '35':
|
||||
excluded.append(doi)
|
||||
continue
|
||||
if (row.get('volume') != '36' or row.get('issue') != '1'
|
||||
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
|
||||
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
|
||||
or row.get('URL') != 'https://doi.org/'+doi
|
||||
or not isinstance(row.get('title'), list) or len(row['title']) != 1
|
||||
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
|
||||
raise ValueError('Unreviewed publication volume, DOI, type or title')
|
||||
day, = row['published']['date-parts']
|
||||
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
|
||||
raise ValueError('Unreviewed publication date')
|
||||
publication_date = date(*day).isoformat()
|
||||
authors = row.get('authors')
|
||||
if (not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
|
||||
or not a.get('given') or not a.get('family') for a in authors)):
|
||||
raise ValueError('Incomplete registry authors')
|
||||
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||
if (len(set(n.casefold() for n in names)) != len(names)
|
||||
or any('@' in n or 'anonymous' in n.casefold() for n in names)
|
||||
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||
raise ValueError('Unreviewed registry author order or identity')
|
||||
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
|
||||
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
|
||||
if len(papers) != 90 or len(excluded) != 1:
|
||||
raise ValueError('Incomplete reviewed volume census')
|
||||
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
path = Path(snapshots)/'crossref-filtered.json'
|
||||
if path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Unsafe or oversized registry snapshot')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != SNAPSHOT_SHA:
|
||||
raise ValueError('Reviewed registry snapshot hash changed')
|
||||
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
|
||||
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
|
||||
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
|
||||
for paper in papers:
|
||||
original = paper['original']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
|
||||
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
|
||||
'metadata': {'registry_fields': original, 'source_evidence': evidence,
|
||||
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
|
||||
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
|
||||
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
|
||||
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
|
||||
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
|
||||
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
|
||||
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
|
||||
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
||||
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
||||
import argparse
|
||||
from datetime import datetime, date
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.http import fetch
|
||||
|
||||
SOURCE = 'rss_proceedings'
|
||||
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
||||
HEADING = 'Robotics: Science and Systems XXII'
|
||||
PAGES = {'index.html': ROOT,
|
||||
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
||||
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
||||
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(values):
|
||||
if len(values) != 1:
|
||||
raise ValueError('Expected one reviewed HTML field')
|
||||
return values[0]
|
||||
|
||||
|
||||
def comparable(value):
|
||||
return unicodedata.normalize('NFC', clean(value))
|
||||
|
||||
|
||||
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
||||
policy = clean(Document(policy_html).root.text())
|
||||
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
||||
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
||||
if text not in policy:
|
||||
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
||||
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
||||
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
||||
nodes = list(Document(index_html).root.walk())
|
||||
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
||||
raise ValueError('Unexpected publisher edition')
|
||||
index = {}
|
||||
for row in (n for n in nodes if n.tag == 'tr'):
|
||||
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
||||
if not links:
|
||||
continue
|
||||
link = one(links); pid = int(link.attrs['href'][1:4])
|
||||
if pid in index:
|
||||
raise ValueError('Repeated publisher ID')
|
||||
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
||||
program = {}
|
||||
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
||||
raise ValueError('Unexpected accepted program')
|
||||
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
||||
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
||||
raise ValueError('Unexpected program row')
|
||||
pid = int(clean(cells[0].text()))
|
||||
if pid in program:
|
||||
raise ValueError('Repeated program ID')
|
||||
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
||||
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
||||
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
||||
if not preview or not full.startswith(preview):
|
||||
raise ValueError('Author preview is not a prefix of complete author evidence')
|
||||
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
||||
if links != [f'/2026/program/papers/{pid}/']:
|
||||
raise ValueError('Program display link changed')
|
||||
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
||||
'authors_preview': preview, 'session': clean(cells[1].text()),
|
||||
'program_url': 'https://roboticsconference.org'+links[0]}
|
||||
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
||||
raise ValueError('Reviewed 210-entry source census changed')
|
||||
return index, program
|
||||
|
||||
|
||||
def parse_paper(html, pid, index, program):
|
||||
nodes = list(Document(html).root.walk()); meta = {}
|
||||
for n in nodes:
|
||||
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
||||
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
||||
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
||||
day = one(meta.get('citation_publication_date', []))
|
||||
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
||||
raise ValueError('Unexpected publication date')
|
||||
date.fromisoformat(day.replace('/', '-'))
|
||||
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
||||
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
||||
or not authors or any(not clean(a) for a in authors)
|
||||
or comparable(title) != comparable(index['title'])
|
||||
or comparable(', '.join(authors)) != comparable(index['authors'])
|
||||
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
||||
raise ValueError('Publisher citation/index/visible metadata disagree')
|
||||
pdf = one(meta.get('citation_pdf_url', []))
|
||||
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
||||
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
||||
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
||||
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
||||
raise ValueError('BibTeX DOI/year mismatch')
|
||||
variants = []
|
||||
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
||||
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
||||
raise ValueError('Unreviewed complete author variant')
|
||||
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
||||
if comparable(title) != comparable(program['title']):
|
||||
if (pid, title, program['title']) != (89,
|
||||
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
||||
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
||||
raise ValueError('Unreviewed program title variant')
|
||||
variants.append('reviewed_program_title_variant_not_erratum')
|
||||
inside = False; parts = []; markers = []
|
||||
for n in (n for n in nodes if n.tag == 'p'):
|
||||
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
||||
if 'Abstract:' in labels:
|
||||
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
||||
if value: parts.append(value)
|
||||
elif 'Download:' in labels:
|
||||
inside = False; markers.append('end')
|
||||
elif inside:
|
||||
parts.append(clean(n.text()))
|
||||
if markers != ['start', 'end'] or not any(parts):
|
||||
raise ValueError('Missing or changed publisher abstract layout')
|
||||
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
||||
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
||||
|
||||
|
||||
def collect(out, *, snapshots=None, cache=None):
|
||||
if (snapshots is None) == (cache is None):
|
||||
raise ValueError('Specify snapshots or cache')
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.mkdir()
|
||||
evidence = {}; receipts = {}
|
||||
if snapshots is not None:
|
||||
root = Path(snapshots)
|
||||
try:
|
||||
for r in json.loads((root/'receipts.json').read_bytes()):
|
||||
name = r['file']
|
||||
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
||||
raise ValueError('Unsafe or repeated snapshot receipt')
|
||||
receipts[name] = r
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
def page(name, url):
|
||||
if snapshots is not None:
|
||||
path = root/name
|
||||
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
||||
raw = path.read_bytes(); r = receipts[name]
|
||||
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
||||
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
||||
raise ValueError('Snapshot hash/receipt mismatch')
|
||||
at = r['at']
|
||||
else:
|
||||
raw = fetch(url, cache, limit=8*1024*1024)
|
||||
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
||||
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
||||
at = sidecar['retrieved_at']
|
||||
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
||||
raise ValueError('Invalid source size/timezone')
|
||||
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
||||
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
||||
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
||||
return raw.decode('utf-8')
|
||||
try:
|
||||
pages = {name: page(name, url) for name, url in PAGES.items()}
|
||||
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
||||
records = []
|
||||
for pid in sorted(index):
|
||||
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
||||
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
||||
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
||||
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
||||
'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
||||
'observed_at': evidence[name]['observed_at'],
|
||||
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
||||
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
||||
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
||||
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
||||
'policy_evidence': evidence['cfp.html'],
|
||||
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
||||
records.append(row)
|
||||
path = out/'rss-proceedings-2026-main.jsonl'
|
||||
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
||||
path.write_bytes(raw)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
||||
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
||||
'status': 'partial', 'missing_abstract_count': 0}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
||||
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
||||
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||
temporary.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
p.add_argument('--out', required=True, type=Path)
|
||||
g = p.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
||||
args = p.parse_args(argv)
|
||||
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
||||
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Frozen SDM 2026 publication records and separate public program observations."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import date, datetime
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
SOURCE = 'crossref_sdm_registry'
|
||||
TRACK = 'proceedings_unclassified'
|
||||
BOOK = '10.1137/1.9781611979329'
|
||||
ISBN = '9781611979329'
|
||||
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
|
||||
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
|
||||
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
|
||||
SNAPSHOTS = {
|
||||
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
|
||||
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
|
||||
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
|
||||
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def load_snapshot(root, name):
|
||||
expected, key = SNAPSHOTS[name]
|
||||
directory = root/name; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != expected:
|
||||
raise ValueError('Reviewed snapshot manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
|
||||
raise ValueError('Snapshot timestamp requires timezone')
|
||||
files = {'manifest.json': raw}
|
||||
for receipt in manifest[key]:
|
||||
if 'file' not in receipt:
|
||||
# A recorded 403 is an access observation, not an empty data set.
|
||||
continue
|
||||
filename = receipt['file']; path = directory/filename
|
||||
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
|
||||
raise ValueError('Unsafe snapshot source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
if filename in files:
|
||||
raise ValueError('Repeated snapshot source file')
|
||||
files[filename] = body
|
||||
return manifest, files
|
||||
|
||||
|
||||
def registry_papers(message):
|
||||
items = message.get('items', [])
|
||||
if message.get('total-results') != 55 or len(items) != 55:
|
||||
raise ValueError('Incomplete exact-ISBN registry census')
|
||||
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
|
||||
raise ValueError('Unreviewed publication item types')
|
||||
papers = []; excluded = []; seen = set()
|
||||
for original in items:
|
||||
doi = original.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
|
||||
raise ValueError('Missing/repeated DOI or wrong ISBN')
|
||||
seen.add(doi)
|
||||
if doi in (BOOK, BOOK+'.fm'):
|
||||
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
|
||||
if (original['type'], original.get('title')) != expected:
|
||||
raise ValueError('Unreviewed excluded volume/front matter')
|
||||
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
|
||||
continue
|
||||
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
|
||||
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
|
||||
or original.get('URL') != 'https://doi.org/'+doi
|
||||
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
|
||||
or original.get('relation', {}).get('is-part-of') != [
|
||||
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
|
||||
raise ValueError('Wrong publication edition, publisher or parent DOI')
|
||||
title = one(original.get('title', []))
|
||||
if not isinstance(title, str) or not clean(title):
|
||||
raise ValueError('Missing registry title')
|
||||
flags = []
|
||||
if '<' in title or '>' in title:
|
||||
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
|
||||
if not match:
|
||||
raise ValueError('Unreviewed title markup')
|
||||
comparison_title = clean(match[1])
|
||||
flags.append('title_sup_asterisk_footnote_requires_review')
|
||||
else:
|
||||
comparison_title = clean(Document(title).root.text())
|
||||
displayed_title = clean(Document(title).root.text())
|
||||
authors = original.get('author')
|
||||
if (not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
|
||||
or not clean(a['given']) or not clean(a['family']) for a in authors)
|
||||
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||
raise ValueError('Incomplete registry authors or order evidence')
|
||||
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||
if any('@' in n or 'anonymous' in n.casefold() for n in names):
|
||||
raise ValueError('Unreviewed registry author identity')
|
||||
dates = {}
|
||||
for field in ('published', 'published-print', 'published-online', 'issued'):
|
||||
if field in original:
|
||||
parts = one(original[field]['date-parts'])
|
||||
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
|
||||
raise ValueError('Unreviewed registry publication date')
|
||||
dates[field] = date(*parts).isoformat()
|
||||
if not dates or original.get('abstract'):
|
||||
raise ValueError('Reviewed registry date/abstract availability changed')
|
||||
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
|
||||
'authors': names, 'review_flags': flags, 'publication_dates': dates})
|
||||
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
|
||||
raise ValueError('Incomplete reviewed 53-chapter DOI set')
|
||||
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
|
||||
|
||||
|
||||
class SessionParser(HTMLParser):
|
||||
"""The source omits closing dt/dd tags; each next dt starts a new event."""
|
||||
def __init__(self):
|
||||
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
|
||||
|
||||
def finish(self):
|
||||
if self.current is not None:
|
||||
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
|
||||
if not match or not clean(self.current['authors_literal']):
|
||||
raise ValueError('Incomplete contributed program event')
|
||||
self.current['time_literal'], self.current['title'] = match.groups()
|
||||
self.current['authors_literal'] = clean(self.current['authors_literal'])
|
||||
self.rows.append(self.current)
|
||||
self.current = None; self.strong = False; self.author = False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == 'dt':
|
||||
self.finish()
|
||||
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
|
||||
if self.current is not None:
|
||||
if tag == 'strong': self.strong = True
|
||||
elif tag == 'dd': self.author = True
|
||||
elif tag == 's': self.current['cancelled_presentation'] = True
|
||||
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
|
||||
self.current['abstract_links'].append(attrs['href'])
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'strong': self.strong = False
|
||||
elif tag == 'dl': self.finish()
|
||||
|
||||
def handle_data(self, text):
|
||||
if self.current is not None:
|
||||
if self.strong: self.current['timed_title'] += text
|
||||
elif self.author: self.current['authors_literal'] += text
|
||||
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
|
||||
|
||||
|
||||
def talk_fields(raw, session_url):
|
||||
nodes = list(Document(raw.decode('utf-8')).root.walk())
|
||||
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
|
||||
raise ValueError('Wrong program page edition')
|
||||
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
|
||||
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
|
||||
abstract = clean(abstract_node.text())
|
||||
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
|
||||
raise ValueError('Missing public program title/abstract')
|
||||
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
|
||||
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
|
||||
def session_key(url):
|
||||
u = urlsplit(url)
|
||||
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
|
||||
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
|
||||
raise ValueError('Foreign program session link')
|
||||
return query['sessioncode']
|
||||
if session_key(link) != session_key(session_url):
|
||||
raise ValueError('Program page belongs to a different session')
|
||||
authors = []
|
||||
for n in nodes:
|
||||
if n.tag == 'li':
|
||||
# Source rows contain name, affiliations, country and contact email.
|
||||
# Only the independently checked leading name is a metadata field.
|
||||
literal = clean(n.text()); name, separator, _ = literal.partition(',')
|
||||
if not separator or not name or '@' in name:
|
||||
raise ValueError('Unreviewed program author row')
|
||||
authors.append(name)
|
||||
if not authors:
|
||||
raise ValueError('Missing source-reported program authors')
|
||||
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
|
||||
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
|
||||
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
|
||||
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
|
||||
|
||||
|
||||
def program_observations(snapshots, papers):
|
||||
official, official_files = snapshots['sdm2026-official-research-v1']
|
||||
sessions, session_files = snapshots['sdm2026-program-research-v1']
|
||||
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
|
||||
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
|
||||
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
|
||||
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
|
||||
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
|
||||
raise ValueError('Contributed session membership disagrees with main program')
|
||||
receipts = {r['url']: r for r in pages['records']}
|
||||
if len(receipts) != len(pages['records']) or len(receipts) != 52:
|
||||
raise ValueError('Incomplete or repeated public talk receipts')
|
||||
rows = []; used = set()
|
||||
for session in sessions['sessions']:
|
||||
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
|
||||
for position, event in enumerate(parser.rows, 1):
|
||||
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
|
||||
event.update(record_kind='source_observation_not_import_record', session=session['title'],
|
||||
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
|
||||
if not urls and event['cancelled_presentation']:
|
||||
event.update(program_page=None, public_page_fields=None)
|
||||
else:
|
||||
url = one(urls)
|
||||
if url not in receipts or url in used:
|
||||
raise ValueError('Missing or repeated public program page')
|
||||
used.add(url); receipt = receipts[url]
|
||||
if receipt['session_url'] != session['url']:
|
||||
raise ValueError('Public page receipt session disagrees')
|
||||
fields = talk_fields(page_files[receipt['file']], session['url'])
|
||||
if fields['title'] != event['title']:
|
||||
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
|
||||
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
|
||||
candidates = []
|
||||
for paper in papers:
|
||||
if paper['comparison_title'].casefold() == event['title'].casefold():
|
||||
fields = event['public_page_fields']
|
||||
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
|
||||
'comparison': 'title_candidate_only_not_identity_approval',
|
||||
'registry_title_review_flags': paper['review_flags'],
|
||||
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
|
||||
event['registry_candidates'] = candidates
|
||||
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
|
||||
rows.append(event)
|
||||
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
|
||||
raise ValueError('Incomplete reviewed program event census')
|
||||
return rows
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
|
||||
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
|
||||
message = json.loads(registry_files['isbn-works.json'])['message']
|
||||
papers, excluded = registry_papers(message)
|
||||
observations = program_observations(loaded, papers)
|
||||
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||
for name, (manifest, files) in loaded.items():
|
||||
for filename, raw in files.items():
|
||||
rel = 'evidence/'+digest(raw)+Path(filename).suffix
|
||||
(out/rel).write_bytes(raw)
|
||||
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
|
||||
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
|
||||
rows = []
|
||||
for p in papers:
|
||||
original = p['original']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
|
||||
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
|
||||
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
|
||||
'observed_at': receipt['at'], 'evidence_url': QUERY,
|
||||
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
|
||||
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
|
||||
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
|
||||
'original_decision_signature_verified': False,
|
||||
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
|
||||
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
|
||||
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
|
||||
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
|
||||
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||
sidecar = out/'program-observations.jsonl'
|
||||
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
|
||||
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
|
||||
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
|
||||
'cancelled_presentations': 1, 'not_an_import_manifest': True},
|
||||
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
|
||||
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -4,7 +4,7 @@ import re
|
||||
from paper_atlas.io.publication_ids import mlsys_id
|
||||
|
||||
|
||||
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
|
||||
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
|
||||
|
||||
|
||||
def timestamp(value=None):
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,75 @@
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import corl2026 as c
|
||||
|
||||
|
||||
class CoRL2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
|
||||
def test_official_census_and_presentation_dedup(self):
|
||||
program = c.parse_program(self.bundle['official_html'])
|
||||
self.assertEqual(len(program), 687)
|
||||
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||
|
||||
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||
html = self.bundle['official_html']
|
||||
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||
self.assertIn(title, html)
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||
|
||||
def test_foreign_forum_source_is_rejected(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_unknown_profile_fields_are_rejected(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'][0]['authorids'] = ['profile']
|
||||
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_source_sets_must_agree(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'].pop()
|
||||
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||
official.mkdir(); archive.mkdir()
|
||||
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||
(official/'accepted.html').write_text('tampered')
|
||||
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||
self.assertTrue((root/'failed/failed.json').exists())
|
||||
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||
|
||||
def test_forum_identity_validation(self):
|
||||
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||
with self.assertRaises(ValueError):
|
||||
c.forum_id(bad)
|
||||
@@ -0,0 +1,95 @@
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icaps2026 as c
|
||||
|
||||
|
||||
class ICAPS2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
|
||||
def snapshots(self, root):
|
||||
root.mkdir()
|
||||
for name, html in self.bundle['pages'].items():
|
||||
(root/name).write_text(html)
|
||||
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
|
||||
return root
|
||||
|
||||
def test_full_listing_matches_independently_recorded_literals(self):
|
||||
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
|
||||
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
|
||||
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
|
||||
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
|
||||
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
|
||||
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
|
||||
|
||||
def test_incomplete_and_duplicate_listing_rejected(self):
|
||||
html = self.bundle['pages']['accepted.html']
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
|
||||
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
|
||||
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
|
||||
|
||||
def test_changed_edition_or_section_rejected(self):
|
||||
html = self.bundle['pages']['accepted.html']
|
||||
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
|
||||
html.replace(c.MAIN, 'Workshop Papers')]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_listing(changed)
|
||||
|
||||
def test_ambiguous_author_lists_not_guessed(self):
|
||||
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
|
||||
with self.assertRaises(ValueError):
|
||||
c.authors_from_literal(literal)
|
||||
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
|
||||
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
|
||||
|
||||
def test_collection_separates_presentations_and_unresolved_tracks(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
|
||||
report = c.collect(out, snapshots=src)
|
||||
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
|
||||
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
|
||||
previous = json.loads((out/report['previously_published_file']).read_text())
|
||||
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
|
||||
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
|
||||
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
|
||||
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
|
||||
self.assertEqual(len({r['external_id'] for r in rows}), 90)
|
||||
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(report['files'][0]['status'], 'partial')
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(out, snapshots=src)
|
||||
|
||||
def test_tampering_retains_failure_without_success_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src')
|
||||
(src/'accepted.html').write_text('tampered')
|
||||
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists())
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
|
||||
for kind in ('redirect', 'time', 'duplicate'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
|
||||
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
|
||||
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
|
||||
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
|
||||
p.write_text(json.dumps(r))
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_snapshot_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
@@ -0,0 +1,96 @@
|
||||
import copy
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icaps_registry2026 as c
|
||||
|
||||
|
||||
class ICAPSRegistry2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
|
||||
cls.snapshot = json.loads(cls.raw)
|
||||
|
||||
def test_exact_edition_not_publication_year_alone(self):
|
||||
papers, excluded = c.registry_papers(self.snapshot)
|
||||
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
|
||||
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
|
||||
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
|
||||
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
|
||||
|
||||
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
|
||||
report = c.collect(root/'out', snapshots=src)
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
|
||||
self.assertEqual({r['external_id'] for r in rows}, set(original))
|
||||
for r in rows:
|
||||
doi = r['external_id']; o = original[doi]
|
||||
self.assertEqual(r['metadata']['registry_fields'], o)
|
||||
self.assertEqual(r['title'], o['title'][0])
|
||||
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
|
||||
self.assertEqual(r['identifiers'], {'doi': doi})
|
||||
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
|
||||
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(r['track'], 'main_tracks_unclassified')
|
||||
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
|
||||
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
|
||||
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
|
||||
self.assertIn('Stephen F. Smith', changed['authors'])
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
|
||||
def test_math_markup_is_not_flattened_as_plain_numbers(self):
|
||||
self.assertEqual(c.abstract_text('<jats:p>A < B and C & D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
|
||||
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.abstract_text(raw)
|
||||
|
||||
def test_invalid_or_repeated_doi_and_edition_rejected(self):
|
||||
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
|
||||
with self.subTest(kind=kind):
|
||||
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
|
||||
if kind == 'doi': r['DOI'] = '10.9999/foreign'
|
||||
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
|
||||
if kind == 'volume': r['volume'] = '37'
|
||||
if kind == 'container': r['container-title'] = ['Other Conference']
|
||||
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
|
||||
with self.assertRaises(ValueError):
|
||||
c.registry_papers(s)
|
||||
|
||||
def test_missing_authors_and_changed_order_evidence_rejected(self):
|
||||
for kind in ('missing', 'sequence', 'profile'):
|
||||
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
|
||||
if kind == 'missing': r['authors'][0].pop('family')
|
||||
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
|
||||
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
|
||||
with self.assertRaises(ValueError):
|
||||
c.registry_papers(s)
|
||||
|
||||
def test_receipt_scope_and_timezone_rejected(self):
|
||||
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
|
||||
s = copy.deepcopy(self.snapshot); s[key] = value
|
||||
with self.assertRaises(ValueError):
|
||||
c.registry_papers(s)
|
||||
|
||||
def test_tamper_failure_retained_and_success_not_published(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
|
||||
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists())
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
@@ -0,0 +1,100 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from paper_atlas.collectors import rss2026 as rss
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
|
||||
|
||||
class Rss2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
|
||||
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
||||
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
|
||||
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
|
||||
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
|
||||
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
|
||||
|
||||
def lists(self,**changes):
|
||||
pages={**self.pages,**changes}
|
||||
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
|
||||
|
||||
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
|
||||
report=rss.collect(self.root/'out',snapshots=self.snap)
|
||||
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
|
||||
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
|
||||
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
|
||||
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
for row,gold in zip(rows,self.fixture['independent_gold']):
|
||||
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
|
||||
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
|
||||
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
|
||||
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
|
||||
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
|
||||
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
|
||||
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
|
||||
self.assertEqual(manifest['missing_abstract_count'],0)
|
||||
|
||||
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
|
||||
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
|
||||
self.assertEqual(len(shortened),6)
|
||||
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
|
||||
self.assertEqual(len(p['authors']),55)
|
||||
self.assertFalse(p['program_variants'])
|
||||
|
||||
def test_six_reviewed_name_variants_preserve_original_strings(self):
|
||||
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
|
||||
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
|
||||
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
|
||||
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
|
||||
|
||||
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
|
||||
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
|
||||
self.assertTrue(p['title'].startswith('RLinf-VLA'))
|
||||
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
|
||||
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
|
||||
|
||||
def test_unreviewed_author_or_title_variant_stops(self):
|
||||
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
|
||||
with self.assertRaisesRegex(ValueError,'Unreviewed'):
|
||||
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
|
||||
|
||||
def test_demo_policy_cannot_be_reused_from2025(self):
|
||||
with self.assertRaisesRegex(ValueError,'policy'):
|
||||
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
|
||||
with self.assertRaises(ValueError):
|
||||
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
|
||||
|
||||
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
|
||||
html=self.pages['p001.html']
|
||||
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
|
||||
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
|
||||
self.assertIn(old,html)
|
||||
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
|
||||
|
||||
def test_index_census_and_preview_tampering_stop(self):
|
||||
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
|
||||
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
|
||||
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
|
||||
|
||||
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
|
||||
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
|
||||
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
|
||||
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
|
||||
self.assertFalse((self.root/'out/collection-report.json').exists())
|
||||
self.assertTrue((self.root/'out/failed.json').exists())
|
||||
|
||||
def test_existing_output_and_unsafe_receipt_are_rejected(self):
|
||||
out=self.root/'existing';out.mkdir()
|
||||
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
|
||||
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
|
||||
(self.snap/'receipts.json').write_text(json.dumps(receipts))
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
|
||||
self.assertTrue((self.root/'unsafe/failed.json').exists())
|
||||
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
|
||||
@@ -0,0 +1,59 @@
|
||||
"""RSS publisher/DBLP observations share one event without losing old decisions."""
|
||||
from pathlib import Path
|
||||
import json
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from paper_atlas.db.submissions import identities_for_row
|
||||
from tests.test_aamas_publication_identity import row, ingest
|
||||
|
||||
DOI='10.15607/rss.2026.xxii.041'
|
||||
|
||||
def rss(source,external,**kwargs):
|
||||
return row(source,external,venue='RSS',year=2026,track='main',
|
||||
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
|
||||
|
||||
class RssPublicationIdentityTests(unittest.TestCase):
|
||||
def test_only_verified_accepted_source_adds_publication_identity(self):
|
||||
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
|
||||
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
|
||||
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
|
||||
self.assertEqual(identities_for_row({**r,'status':status}),[])
|
||||
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
|
||||
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
|
||||
|
||||
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
|
||||
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
|
||||
for pair in ((a,b),(b,a)):
|
||||
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);db=root/'papers.sqlite'
|
||||
for r in pair:ingest(db,root,r)
|
||||
ingest(db,root,a)
|
||||
with sqlite3.connect(db) as c:
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
|
||||
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
|
||||
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
|
||||
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||
|
||||
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);db=root/'papers.sqlite'
|
||||
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
|
||||
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
|
||||
ingest(db,root,old)
|
||||
with sqlite3.connect(db) as c:
|
||||
before=c.execute('SELECT * FROM decisions').fetchall()
|
||||
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
|
||||
ingest(db,root,new)
|
||||
with sqlite3.connect(db) as c:
|
||||
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||
@@ -0,0 +1,142 @@
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import sdm2026 as c
|
||||
|
||||
|
||||
class SDM2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
|
||||
|
||||
def snapshots(self, root):
|
||||
for name, text in self.fixture['files'].items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
|
||||
|
||||
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
|
||||
papers, excluded = c.registry_papers(self.message)
|
||||
self.assertEqual(len(papers), 53)
|
||||
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
|
||||
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
|
||||
marked = [p for p in papers if p['review_flags']]
|
||||
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
|
||||
for p in marked:
|
||||
self.assertTrue(p['title'].endswith('*'))
|
||||
self.assertIn('<sup>*</sup>', p['original']['title'][0])
|
||||
|
||||
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
report = c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
|
||||
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
|
||||
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
||||
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
|
||||
for row in rows:
|
||||
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
|
||||
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
|
||||
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
|
||||
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
|
||||
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
|
||||
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
|
||||
sidecar = report['program_observations']
|
||||
self.assertTrue(sidecar['not_an_import_manifest'])
|
||||
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
|
||||
self.assertEqual(len(observations), 53)
|
||||
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
|
||||
for observation in observations:
|
||||
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
|
||||
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
|
||||
cancelled, = [x for x in observations if x['cancelled_presentation']]
|
||||
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
|
||||
self.assertIsNone(cancelled['public_page_fields'])
|
||||
# The real source has conflicting author sets under the same title.
|
||||
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
|
||||
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
|
||||
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
|
||||
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
|
||||
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
|
||||
# Every copied source and manifest remains byte-identical, including CRLF.
|
||||
for evidence in report['evidence']:
|
||||
body = (root/'out'/evidence['output_file']).read_bytes()
|
||||
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
|
||||
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
|
||||
def test_all_program_events_match_independent_inventory(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root)
|
||||
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
|
||||
papers, _ = c.registry_papers(self.message)
|
||||
observations = c.program_observations(loaded, papers)
|
||||
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
|
||||
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
|
||||
for item in observations:
|
||||
original = gold[(item['session_url'], item['title'])]
|
||||
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
|
||||
self.assertEqual(item[key], original[key])
|
||||
|
||||
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
|
||||
files = self.fixture['files']
|
||||
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
|
||||
receipts = {r['url']: r for r in manifest['records']}
|
||||
self.assertEqual(len(self.fixture['page_gold']), 52)
|
||||
for gold in self.fixture['page_gold']:
|
||||
receipt = receipts[gold['program_url']]
|
||||
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
|
||||
fields = c.talk_fields(raw, receipt['session_url'])
|
||||
self.assertEqual(fields['title'], gold['title'])
|
||||
self.assertEqual(fields['source_reported_authors'], gold['authors'])
|
||||
self.assertEqual(fields['abstract'], gold['abstract'])
|
||||
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
|
||||
|
||||
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
|
||||
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
|
||||
with self.subTest(kind=kind):
|
||||
message = copy.deepcopy(self.message)
|
||||
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
|
||||
if kind == 'isbn': row['ISBN'] = ['9780000000000']
|
||||
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
|
||||
if kind == 'repeat': row['DOI'] = c.BOOK
|
||||
if kind == 'type': row['type'] = 'other'
|
||||
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
|
||||
if kind == 'author': row['author'][0]['sequence'] = 'additional'
|
||||
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
|
||||
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
|
||||
if kind == 'missing': message['items'].pop()
|
||||
with self.assertRaises(ValueError): c.registry_papers(message)
|
||||
|
||||
def test_foreign_session_and_changed_edition_rejected(self):
|
||||
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
|
||||
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
|
||||
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
|
||||
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
|
||||
with self.assertRaises(ValueError): c.talk_fields(changed, url)
|
||||
|
||||
def test_corruption_preserves_failure_without_success_report(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
|
||||
with self.assertRaisesRegex(ValueError, 'receipt/body'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertTrue((root/'out/failed.json').exists())
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_source_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
|
||||
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
Reference in New Issue
Block a user