Compare commits
25 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2f1b5c959a | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| 06467835c7 | |||
| f2753bc741 | |||
| 1ea17c810d | |||
| f800c6ecb4 | |||
| 28255743ef | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 | |||
| 59053fd4fb | |||
| b50136d5bc | |||
| 87a0689fcc | |||
| 9093a8ce25 | |||
| e19cb83cd6 | |||
| 4d2df2a826 | |||
| 19d03ced54 | |||
| c33f5422de | |||
| 1cfd44d8b6 | |||
| 5de7f6ff22 | |||
| 93f00d1a64 | |||
| 8b7bc6d871 | |||
| 7588a4e904 |
@@ -48,6 +48,106 @@ jobs:
|
|||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
|
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
eccv_official = root/'eccv2026-official'
|
||||||
|
eccv_papers = root/'eccv2026-paper-samples'
|
||||||
|
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||||
|
for name, html in eccv_fixture['official_pages'].items():
|
||||||
|
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, html in eccv_fixture['paper_pages'].items():
|
||||||
|
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||||
|
eccv_encoding = eccv_fixture['encoding_sample']
|
||||||
|
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||||
|
eccv_literal = eccv_fixture['literal_title_sample']
|
||||||
|
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||||
|
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||||
|
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||||
|
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||||
|
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||||
|
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||||
|
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||||
|
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||||
|
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||||
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||||
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||||
|
assert '&' in eccv_literal_row['title']
|
||||||
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
sdm_snapshots = root/'sdm2026-snapshots'
|
||||||
|
for name, text in sdm_fixture['files'].items():
|
||||||
|
target = sdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
||||||
|
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
||||||
|
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
||||||
|
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
||||||
|
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
||||||
|
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
||||||
|
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
||||||
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||||
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||||
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||||
|
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
icdm_snapshots = root/'icdm2026-snapshots'
|
||||||
|
for name, text in icdm_fixture['files'].items():
|
||||||
|
target = icdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||||
|
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||||
|
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||||
|
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||||
|
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||||
|
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||||
|
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||||
|
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||||
|
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||||
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
|
icaps_snapshots.mkdir()
|
||||||
|
for name, html in icaps_fixture['pages'].items():
|
||||||
|
(icaps_snapshots/name).write_text(html)
|
||||||
|
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
||||||
|
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
||||||
|
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
||||||
|
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
||||||
|
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
||||||
|
registry_snapshots = root/'icaps2026-registry-snapshots'
|
||||||
|
registry_snapshots.mkdir()
|
||||||
|
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
|
||||||
|
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
|
||||||
|
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
|
||||||
|
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
|
||||||
|
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||||
|
rss_snapshots = root/'rss2026-snapshots'
|
||||||
|
rss_snapshots.mkdir()
|
||||||
|
for name, html in rss_fixture['pages'].items():
|
||||||
|
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
corl_official = root/'corl2026-official'
|
||||||
|
corl_archive = root/'corl2026-archive'
|
||||||
|
corl_official.mkdir()
|
||||||
|
corl_archive.mkdir()
|
||||||
|
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||||
|
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||||
|
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||||
|
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||||
|
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||||
|
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||||
|
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||||
|
|||||||
@@ -118,3 +118,19 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
|||||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||||
|
|
||||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||||
|
|
||||||
|
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||||
|
|
||||||
|
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||||
|
|
||||||
|
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
||||||
|
|
||||||
|
### CoRL 2026 官方程序快照
|
||||||
|
|
||||||
|
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||||
|
|
||||||
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# CoRL 2026:官方程序与公开档案
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect corl2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||||
|
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||||
|
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||||
|
|
||||||
|
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||||
|
|
||||||
|
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||||
|
|
||||||
|
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||||
|
|
||||||
|
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||||
|
|
||||||
|
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||||
|
|
||||||
|
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# ECCV 2026:暂定节目与公开元数据
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect eccv2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||||
|
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||||
|
--max-new-pages 200 \
|
||||||
|
--out /data/paper-index/collected/eccv2026-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||||
|
|
||||||
|
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||||
|
|
||||||
|
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||||
|
|
||||||
|
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||||
|
|
||||||
|
## 来源边界
|
||||||
|
|
||||||
|
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||||
|
|
||||||
|
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||||
|
|
||||||
|
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||||
|
|
||||||
|
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||||
|
|
||||||
|
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||||
|
|
||||||
|
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||||
|
|
||||||
|
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
# ECCV 2026 来源核查记录
|
||||||
|
|
||||||
|
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||||
|
|
||||||
|
## 贡献类型与 track
|
||||||
|
|
||||||
|
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||||
|
|
||||||
|
| 官方 group 字段值 | 含义摘要 |
|
||||||
|
| --- | --- |
|
||||||
|
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||||
|
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||||
|
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||||
|
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||||
|
| Concept & Feasibility | 新概念及可行性证据 |
|
||||||
|
|
||||||
|
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||||
|
|
||||||
|
## 公开负面决定元数据
|
||||||
|
|
||||||
|
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||||
|
|
||||||
|
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||||
|
|
||||||
|
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||||
|
|
||||||
|
## 可恢复证据与后续入口
|
||||||
|
|
||||||
|
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||||
|
|
||||||
|
| 文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||||
|
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||||
|
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||||
|
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||||
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICAPS 2026 官方名单
|
||||||
|
|
||||||
|
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-program \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
|
||||||
|
|
||||||
|
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
|
||||||
|
|
||||||
|
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
|
||||||
|
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
|
||||||
|
|
||||||
|
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
|
||||||
|
|
||||||
|
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
|
||||||
|
|
||||||
|
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
|
||||||
|
|
||||||
|
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
|
||||||
|
|
||||||
|
## 独立注册元数据版本
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
|
||||||
|
|
||||||
|
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
|
||||||
|
|
||||||
|
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
|
||||||
|
|
||||||
|
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# ICDM2026 冻结日程适配器
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icdm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||||
|
|
||||||
|
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||||
|
|
||||||
|
## 两个明确来源轨道
|
||||||
|
|
||||||
|
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||||
|
|
||||||
|
| 输出文件 | 官方明确标签 | 记录数 |
|
||||||
|
| --- | --- | ---: |
|
||||||
|
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||||
|
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||||
|
|
||||||
|
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||||
|
|
||||||
|
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||||
|
|
||||||
|
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||||
|
|
||||||
|
## 始终保留范围限制
|
||||||
|
|
||||||
|
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||||
|
|
||||||
|
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||||
|
|
||||||
|
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||||
|
|
||||||
|
## 验证与恢复
|
||||||
|
|
||||||
|
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||||
|
|
||||||
|
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||||
|
|
||||||
|
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||||
|
|
||||||
|
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# RSS 2026 论文集来源
|
||||||
|
|
||||||
|
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-v1
|
||||||
|
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
|
||||||
|
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-replay-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
|
||||||
|
|
||||||
|
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
|
||||||
|
|
||||||
|
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
|
||||||
|
|
||||||
|
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
|
||||||
|
|
||||||
|
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
|
||||||
@@ -0,0 +1,78 @@
|
|||||||
|
# SDM 2026:生产规范化与身份核查记录
|
||||||
|
|
||||||
|
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
||||||
|
|
||||||
|
## 已核实的生产输出
|
||||||
|
|
||||||
|
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||||
|
|
||||||
|
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||||
|
|
||||||
|
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||||
|
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||||
|
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||||
|
|
||||||
|
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||||
|
|
||||||
|
| 证据文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||||
|
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||||
|
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||||
|
|
||||||
|
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||||
|
|
||||||
|
## 当前数据库只读预检
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||||
|
|
||||||
|
| 候选类别 | 记录数 | 含义 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||||
|
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||||
|
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||||
|
|
||||||
|
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||||
|
|
||||||
|
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||||
|
|
||||||
|
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||||
|
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||||
|
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||||
|
|
||||||
|
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||||
|
|
||||||
|
## 需要保护的较早拒稿历史
|
||||||
|
|
||||||
|
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||||
|
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||||
|
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||||
|
|
||||||
|
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||||
|
|
||||||
|
## 未精确匹配的补充检索
|
||||||
|
|
||||||
|
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||||
|
|
||||||
|
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||||
|
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||||
|
|
||||||
|
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||||
|
|
||||||
|
| 服务器恢复文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||||
|
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||||
|
|
||||||
|
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||||
|
|
||||||
|
## 下次恢复
|
||||||
|
|
||||||
|
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||||
|
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||||
|
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||||
|
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# SDM 2026:论文集与日程版本
|
||||||
|
|
||||||
|
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect sdm2026-proceedings \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/sdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
|
||||||
|
|
||||||
|
## 两种输出
|
||||||
|
|
||||||
|
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
|
||||||
|
|
||||||
|
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
|
||||||
|
|
||||||
|
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
|
||||||
|
|
||||||
|
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
|
||||||
|
|
||||||
|
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
|
||||||
|
|
||||||
|
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
|
||||||
|
|
||||||
|
## 公开拒稿与访问限制
|
||||||
|
|
||||||
|
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
|
||||||
|
|
||||||
|
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
|
||||||
|
|
||||||
|
## 来源恢复与后续核查
|
||||||
|
|
||||||
|
| 冻结目录 | manifest SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
|
||||||
|
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
|
||||||
|
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
|
||||||
|
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||||
|
|
||||||
|
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||||
|
|
||||||
|
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||||
@@ -23,9 +23,16 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||||
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||||
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'corl_program_archive'
|
||||||
|
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||||
|
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||||
|
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||||
|
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||||
|
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||||
|
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||||
|
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||||
|
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||||
|
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||||
|
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||||
|
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def forum_id(url):
|
||||||
|
if not isinstance(url, str):
|
||||||
|
raise ValueError('Forum URL must be text')
|
||||||
|
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||||
|
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||||
|
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||||
|
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||||
|
raise ValueError('Invalid original forum identity')
|
||||||
|
return q['id'][0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||||
|
if len(embeds) != 1:
|
||||||
|
raise ValueError('Expected one official embedded program')
|
||||||
|
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected program edition')
|
||||||
|
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||||
|
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||||
|
raise ValueError('Reviewed program census changed')
|
||||||
|
papers = {}; sessions = {}; occurrences = []
|
||||||
|
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||||
|
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||||
|
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||||
|
raise ValueError('Unexpected or repeated presentation session')
|
||||||
|
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||||
|
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||||
|
if len(headings) != 1:
|
||||||
|
raise ValueError('Ambiguous session heading')
|
||||||
|
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||||
|
seen = set()
|
||||||
|
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||||
|
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||||
|
if fid in seen or not title:
|
||||||
|
raise ValueError('Repeated or empty paper in session')
|
||||||
|
seen.add(fid)
|
||||||
|
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||||
|
if paper['title'] != title:
|
||||||
|
raise ValueError('Conflicting official titles share a forum')
|
||||||
|
event = {'session_id': sid, **sessions[sid]}
|
||||||
|
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||||
|
counts = Counter(kind for _, _, kind in occurrences)
|
||||||
|
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||||
|
raise ValueError('Incomplete official program')
|
||||||
|
for paper in papers.values():
|
||||||
|
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||||
|
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||||
|
raise ValueError('Invalid poster/talk multiplicity')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def join_metadata(program, archive):
|
||||||
|
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||||
|
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||||
|
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||||
|
or not isinstance(archive.get('papers'), list)):
|
||||||
|
raise ValueError('Unpinned public metadata provenance')
|
||||||
|
indexed = {}
|
||||||
|
for row in archive['papers']:
|
||||||
|
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||||
|
raise ValueError('Unexpected metadata/profile fields')
|
||||||
|
fid = forum_id(row.get('openreview'))
|
||||||
|
if fid != row.get('id') or fid in indexed:
|
||||||
|
raise ValueError('Repeated or mismatched archive forum')
|
||||||
|
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||||
|
raise ValueError('Field source belongs to another submission')
|
||||||
|
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||||
|
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||||
|
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||||
|
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||||
|
raise ValueError('Incomplete scholarly metadata')
|
||||||
|
indexed[fid] = row
|
||||||
|
if set(indexed) != set(program):
|
||||||
|
raise ValueError('Official and archive forum sets differ')
|
||||||
|
return indexed
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, archive_snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||||
|
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||||
|
if official.is_symlink() or filtered.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||||
|
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||||
|
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Missing official source receipt')
|
||||||
|
receipt = matches[0]; observed = receipt['at']
|
||||||
|
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||||
|
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid official receipt')
|
||||||
|
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
sources = {}
|
||||||
|
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||||
|
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||||
|
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||||
|
rows = []; variants = []
|
||||||
|
for fid in sorted(program):
|
||||||
|
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||||
|
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||||
|
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||||
|
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||||
|
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||||
|
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||||
|
'observed_at': observed, 'tags': [], 'metadata': {
|
||||||
|
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||||
|
'source_evidence': sources, 'archive_revision': REVISION,
|
||||||
|
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||||
|
if len(variants) != 34:
|
||||||
|
raise ValueError('Reviewed title-variant census changed')
|
||||||
|
path = out/'corl-program-archive-2026-main.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||||
|
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--out', type=Path, required=True)
|
||||||
|
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||||
|
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import fcntl
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||||
|
|
||||||
|
SOURCE = 'eccv_program'
|
||||||
|
ORIGIN = 'https://eccv.ecva.net'
|
||||||
|
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||||
|
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||||
|
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected one reviewed metadata field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, value):
|
||||||
|
return value in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||||
|
raise ValueError('Unexpected ECCV program edition')
|
||||||
|
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||||
|
raise ValueError('Reviewed provisional-publication notice changed')
|
||||||
|
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||||
|
papers = {}; occurrences = 0
|
||||||
|
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 3:
|
||||||
|
raise ValueError('Unexpected program columns')
|
||||||
|
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||||
|
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||||
|
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||||
|
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||||
|
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||||
|
if not row['title'] or not authors:
|
||||||
|
raise ValueError('Incomplete public program fields')
|
||||||
|
occurrences += 1
|
||||||
|
if pid in papers:
|
||||||
|
if papers[pid]['listing'] != row:
|
||||||
|
raise ValueError('Conflicting entries share a program ID')
|
||||||
|
papers[pid]['row_ordinals'].append(occurrences)
|
||||||
|
else:
|
||||||
|
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||||
|
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||||
|
raise ValueError('Incomplete reviewed program census')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(raw, url, program):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||||
|
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||||
|
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||||
|
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||||
|
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||||
|
authors = ld.get('author')
|
||||||
|
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||||
|
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||||
|
or not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||||
|
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||||
|
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||||
|
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||||
|
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||||
|
# Decode only to test agreement with visible text, never to rewrite it.
|
||||||
|
def agrees_with_display(value, expected):
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
for _ in range(2):
|
||||||
|
value = unescape(value)
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
raw_names = [a['name'] for a in authors]
|
||||||
|
names = [clean(name) for name in raw_names]
|
||||||
|
if not agrees_with_display(ld['name'], title):
|
||||||
|
raise ValueError('Public visible and structured titles disagree')
|
||||||
|
if ' ⋅ '.join(names) != authors_literal:
|
||||||
|
visible_names = authors_literal.split(' ⋅ ')
|
||||||
|
if len(visible_names) != len(raw_names) or any(
|
||||||
|
not agrees_with_display(raw_name, visible_name)
|
||||||
|
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
names = visible_names
|
||||||
|
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||||
|
if raw_names != names or ld['name'] != title:
|
||||||
|
flags.append('jsonld_display_text_normalized_requires_review')
|
||||||
|
if title != program['listing']['title']:
|
||||||
|
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||||
|
if authors_literal != program['listing']['authors_literal']:
|
||||||
|
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||||
|
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||||
|
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||||
|
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||||
|
|
||||||
|
|
||||||
|
def official_snapshot(root):
|
||||||
|
root = Path(root); manifest = root/'manifest.json'
|
||||||
|
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed official snapshot manifest changed')
|
||||||
|
m = json.loads(manifest.read_bytes())
|
||||||
|
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Official snapshot requires a timezone')
|
||||||
|
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||||
|
raise ValueError('Official source set changed')
|
||||||
|
pages = {}; evidence = {}
|
||||||
|
for r in m['sources']:
|
||||||
|
name = r['file']; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe official snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||||
|
raise ValueError('Invalid official source receipt')
|
||||||
|
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||||
|
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||||
|
return m, pages, evidence, program
|
||||||
|
|
||||||
|
|
||||||
|
def paper_snapshots(root, program):
|
||||||
|
root = Path(root); path = root/'manifest.json'
|
||||||
|
if root.is_symlink() or path.is_symlink():
|
||||||
|
raise ValueError('Paper snapshot symlinks rejected')
|
||||||
|
if path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Oversized paper manifest')
|
||||||
|
m = json.loads(path.read_bytes()); result = {}
|
||||||
|
for r in m['records']:
|
||||||
|
pid = r['paper_id']
|
||||||
|
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||||
|
raise ValueError('Foreign or repeated paper snapshot')
|
||||||
|
p = root/r['file']
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe paper snapshot')
|
||||||
|
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||||
|
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||||
|
if (paper_snapshots_dir is None) == (cache is None):
|
||||||
|
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||||
|
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||||
|
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
lock = None
|
||||||
|
try:
|
||||||
|
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||||
|
directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in pages.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||||
|
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||||
|
client = None
|
||||||
|
if cache is not None:
|
||||||
|
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||||
|
raise ValueError('Cache lock symlinks rejected')
|
||||||
|
lock = (cache/'.collector.lock').open('a+')
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
client = PublicHTMLCache(cache, ORIGIN)
|
||||||
|
rows = []; missing = []; fetched = 0; reused = 0
|
||||||
|
for pid in sorted(program, key=int):
|
||||||
|
p = program[pid]
|
||||||
|
if snapshots is not None:
|
||||||
|
response = snapshots.get(pid)
|
||||||
|
else:
|
||||||
|
response = client.cached(p['url'])
|
||||||
|
if response is not None:
|
||||||
|
reused += 1
|
||||||
|
elif max_new_pages is None or fetched < max_new_pages:
|
||||||
|
response = client.get(p['url']); fetched += 1
|
||||||
|
if response is None:
|
||||||
|
missing.append(pid); continue
|
||||||
|
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||||
|
keyword = p['listing']['keywords_literal']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||||
|
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||||
|
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||||
|
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||||
|
'evidence_url': ACCEPTED,
|
||||||
|
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||||
|
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||||
|
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||||
|
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||||
|
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||||
|
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||||
|
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||||
|
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||||
|
if len(rows) % 10 == 0:
|
||||||
|
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||||
|
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||||
|
files = []
|
||||||
|
if rows:
|
||||||
|
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||||
|
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||||
|
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||||
|
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||||
|
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||||
|
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||||
|
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if lock is not None:
|
||||||
|
lock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--official-snapshots', required=True)
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument('--paper-snapshots')
|
||||||
|
group.add_argument('--cache')
|
||||||
|
parser.add_argument('--max-new-pages', type=int)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||||
|
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icaps_official_program'
|
||||||
|
ROOT = 'https://icaps26.icaps-conference.org/'
|
||||||
|
PAGES = {
|
||||||
|
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
|
||||||
|
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
|
||||||
|
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
|
||||||
|
}
|
||||||
|
MAIN = 'Papers accepted to the Main Tracks'
|
||||||
|
PREVIOUS = 'Previously Published Papers'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def authors_from_literal(literal):
|
||||||
|
# This frozen listing uses commas between authors and a final "and".
|
||||||
|
# Preserve the complete literal too; never infer person-name aliases.
|
||||||
|
if literal.count(' and ') != 1 or ', and ' in literal:
|
||||||
|
raise ValueError('Unreviewed author-list punctuation')
|
||||||
|
first, last = literal.rsplit(' and ', 1)
|
||||||
|
authors = first.split(', ') + [last]
|
||||||
|
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
|
||||||
|
raise ValueError('Incomplete or ambiguous author list')
|
||||||
|
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
|
||||||
|
raise ValueError('Author literal cannot be reconstructed')
|
||||||
|
return authors
|
||||||
|
|
||||||
|
|
||||||
|
def parse_listing(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
|
||||||
|
if headings != ['List of Papers Accepted to ICAPS 2026']:
|
||||||
|
raise ValueError('Unexpected official conference edition')
|
||||||
|
section = None; sections = []; rows = []; seen = set()
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'h2':
|
||||||
|
section = clean(n.text()); sections.append(section)
|
||||||
|
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
|
||||||
|
continue
|
||||||
|
if section not in (MAIN, PREVIOUS):
|
||||||
|
raise ValueError('Paper belongs to an unreviewed section')
|
||||||
|
# Require the reviewed title and complete-author blocks; fail on layout drift.
|
||||||
|
blocks = [b for b in n.children if isinstance(b, Node)]
|
||||||
|
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
|
||||||
|
raise ValueError('Changed paper metadata layout')
|
||||||
|
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
|
||||||
|
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
|
||||||
|
raise ValueError('Changed paper title/link layout')
|
||||||
|
title = clean(titles[0].text()); literal = clean(blocks[1].text())
|
||||||
|
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
|
||||||
|
raise ValueError('Empty or repeated paper title')
|
||||||
|
seen.add(title.casefold())
|
||||||
|
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
|
||||||
|
'authors_literal': literal})
|
||||||
|
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
|
||||||
|
raise ValueError('Incomplete reviewed official listing')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
root = Path(snapshots)
|
||||||
|
manifest_path = root/'manifest.json'
|
||||||
|
if manifest_path.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
receipt = json.loads(manifest_path.read_bytes())
|
||||||
|
observed = receipt['at']
|
||||||
|
if (receipt.get('status') != 'collected_not_parsed_not_imported'
|
||||||
|
or receipt.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid reviewed snapshot receipt')
|
||||||
|
receipts = receipt['sources']
|
||||||
|
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
|
||||||
|
raise ValueError('Unexpected snapshot source set')
|
||||||
|
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (url, expected) in PAGES.items():
|
||||||
|
matches = [r for r in receipts if r['file'] == name]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Repeated snapshot receipt')
|
||||||
|
r = matches[0]; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
if r['url'] != url or r['final_url'] != url:
|
||||||
|
raise ValueError('Unexpected snapshot URL or redirect')
|
||||||
|
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
|
||||||
|
bodies[name] = raw.decode('utf-8')
|
||||||
|
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
|
||||||
|
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
|
||||||
|
for entry in entries:
|
||||||
|
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if entry['section'] == PREVIOUS:
|
||||||
|
previous.append({**entry, 'entry_key': entry_key,
|
||||||
|
'presentation_kind': 'previously_published_research',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
|
||||||
|
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
|
||||||
|
continue
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
|
||||||
|
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
|
||||||
|
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
|
||||||
|
'metadata': {'official_listing': entry, 'source_evidence': evidence,
|
||||||
|
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
|
||||||
|
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
|
||||||
|
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
|
||||||
|
'abstract_availability': 'not_on_official_listing',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
|
||||||
|
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
|
||||||
|
'status': 'partial', 'missing_abstract_count': 90}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'previously-published-presentations.json'
|
||||||
|
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
|
||||||
|
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
|
||||||
|
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
|
||||||
|
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
|
||||||
|
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icaps_registry'
|
||||||
|
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
|
||||||
|
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
|
||||||
|
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
|
||||||
|
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def abstract_text(raw):
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise ValueError('Missing registry abstract')
|
||||||
|
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
|
||||||
|
if not match or re.search(r'<[^>]*>', match[1]):
|
||||||
|
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
|
||||||
|
text = clean(unescape(match[1]))
|
||||||
|
if not text:
|
||||||
|
raise ValueError('Empty registry abstract')
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(snapshot):
|
||||||
|
observed = snapshot['at']
|
||||||
|
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
|
||||||
|
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
|
||||||
|
or snapshot.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
|
||||||
|
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
|
||||||
|
raise ValueError('Unreviewed registry snapshot receipt')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for row in snapshot['records']:
|
||||||
|
doi = row.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if row.get('volume') == '35':
|
||||||
|
excluded.append(doi)
|
||||||
|
continue
|
||||||
|
if (row.get('volume') != '36' or row.get('issue') != '1'
|
||||||
|
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
|
||||||
|
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
|
||||||
|
or row.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or not isinstance(row.get('title'), list) or len(row['title']) != 1
|
||||||
|
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
|
||||||
|
raise ValueError('Unreviewed publication volume, DOI, type or title')
|
||||||
|
day, = row['published']['date-parts']
|
||||||
|
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
|
||||||
|
raise ValueError('Unreviewed publication date')
|
||||||
|
publication_date = date(*day).isoformat()
|
||||||
|
authors = row.get('authors')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
|
||||||
|
or not a.get('given') or not a.get('family') for a in authors)):
|
||||||
|
raise ValueError('Incomplete registry authors')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if (len(set(n.casefold() for n in names)) != len(names)
|
||||||
|
or any('@' in n or 'anonymous' in n.casefold() for n in names)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Unreviewed registry author order or identity')
|
||||||
|
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
|
||||||
|
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
|
||||||
|
if len(papers) != 90 or len(excluded) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed volume census')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
path = Path(snapshots)/'crossref-filtered.json'
|
||||||
|
if path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized registry snapshot')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != SNAPSHOT_SHA:
|
||||||
|
raise ValueError('Reviewed registry snapshot hash changed')
|
||||||
|
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
|
||||||
|
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
|
||||||
|
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
|
||||||
|
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'source_evidence': evidence,
|
||||||
|
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
|
||||||
|
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
|
||||||
|
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
|
||||||
|
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
|
||||||
|
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
|
||||||
|
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
|
||||||
|
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icdm_program'
|
||||||
|
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||||
|
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||||
|
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||||
|
PAGES = {
|
||||||
|
'detailed-sessions': PROGRAM,
|
||||||
|
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||||
|
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, name):
|
||||||
|
return name in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def field(node, name):
|
||||||
|
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||||
|
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe source manifest')
|
||||||
|
raw = manifest.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Frozen manifest hash changed')
|
||||||
|
data = json.loads(raw)
|
||||||
|
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||||
|
raise ValueError('Wrong source edition or incomplete research run')
|
||||||
|
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||||
|
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||||
|
raise ValueError('Incomplete ICDM source set')
|
||||||
|
files = {'manifest.json': raw}; indexed = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
filename = receipt['file']; path = folder/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||||
|
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body)
|
||||||
|
or not receipt['content_type'].lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body mismatch')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated source filename')
|
||||||
|
files[filename] = body; indexed[receipt['label']] = receipt
|
||||||
|
return indexed, files
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8'))
|
||||||
|
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||||
|
text = clean(main.text())
|
||||||
|
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||||
|
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||||
|
raise ValueError('Wrong edition or unreviewed program scope')
|
||||||
|
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||||
|
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||||
|
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||||
|
raise ValueError('Incomplete or repeated session census')
|
||||||
|
records = []; seen = set()
|
||||||
|
for session in sessions:
|
||||||
|
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||||
|
form = session.attrs.get('data-format')
|
||||||
|
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||||
|
or form not in ('long', 'short')):
|
||||||
|
raise ValueError('Invalid session ID, track or presentation label')
|
||||||
|
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||||
|
if field(session, 'sd-track') != label:
|
||||||
|
raise ValueError('Visible track and source attribute disagree')
|
||||||
|
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||||
|
if count is None or count[1].lower() != form:
|
||||||
|
raise ValueError('Visible presentation label disagrees')
|
||||||
|
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||||
|
if len(rows) != int(count[2]):
|
||||||
|
raise ValueError('Session paper count changed')
|
||||||
|
note = field(session, 'sd-table-note')
|
||||||
|
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||||
|
raise ValueError('Unreviewed presentation-note semantics')
|
||||||
|
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||||
|
'title': field(session, 'sd-session-title'),
|
||||||
|
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||||
|
for row in rows:
|
||||||
|
pid = row.attrs['data-paper-id']
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||||
|
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||||
|
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||||
|
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||||
|
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||||
|
or field(row, 'sd-paper-id') != pid):
|
||||||
|
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||||
|
seen.add(pid)
|
||||||
|
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||||
|
authors = [name.strip() for name in authors_literal.split(';')]
|
||||||
|
if not title or any(not name for name in authors):
|
||||||
|
raise ValueError('Missing program title or author')
|
||||||
|
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||||
|
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||||
|
'session': context.copy(), 'source_position': len(records)+1})
|
||||||
|
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||||
|
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||||
|
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||||
|
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
receipts, files = load_snapshot(snapshots)
|
||||||
|
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
outputs = []
|
||||||
|
for track in ('research', 'applied'):
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['session']['track_literal'] != track:
|
||||||
|
continue
|
||||||
|
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||||
|
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||||
|
'evidence_url': url,
|
||||||
|
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||||
|
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||||
|
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||||
|
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||||
|
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||||
|
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||||
|
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||||
|
'paper_type': None,
|
||||||
|
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||||
|
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||||
|
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||||
|
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||||
|
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||||
|
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||||
|
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||||
|
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||||
|
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True, type=Path)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
||||||
|
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
SOURCE = 'rss_proceedings'
|
||||||
|
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
||||||
|
HEADING = 'Robotics: Science and Systems XXII'
|
||||||
|
PAGES = {'index.html': ROOT,
|
||||||
|
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
||||||
|
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
||||||
|
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(values):
|
||||||
|
if len(values) != 1:
|
||||||
|
raise ValueError('Expected one reviewed HTML field')
|
||||||
|
return values[0]
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(value):
|
||||||
|
return unicodedata.normalize('NFC', clean(value))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
||||||
|
policy = clean(Document(policy_html).root.text())
|
||||||
|
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
||||||
|
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
||||||
|
if text not in policy:
|
||||||
|
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
||||||
|
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
||||||
|
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
||||||
|
nodes = list(Document(index_html).root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
||||||
|
raise ValueError('Unexpected publisher edition')
|
||||||
|
index = {}
|
||||||
|
for row in (n for n in nodes if n.tag == 'tr'):
|
||||||
|
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
||||||
|
if not links:
|
||||||
|
continue
|
||||||
|
link = one(links); pid = int(link.attrs['href'][1:4])
|
||||||
|
if pid in index:
|
||||||
|
raise ValueError('Repeated publisher ID')
|
||||||
|
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
||||||
|
program = {}
|
||||||
|
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected accepted program')
|
||||||
|
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
||||||
|
raise ValueError('Unexpected program row')
|
||||||
|
pid = int(clean(cells[0].text()))
|
||||||
|
if pid in program:
|
||||||
|
raise ValueError('Repeated program ID')
|
||||||
|
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
||||||
|
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
||||||
|
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
||||||
|
if not preview or not full.startswith(preview):
|
||||||
|
raise ValueError('Author preview is not a prefix of complete author evidence')
|
||||||
|
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
||||||
|
if links != [f'/2026/program/papers/{pid}/']:
|
||||||
|
raise ValueError('Program display link changed')
|
||||||
|
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
||||||
|
'authors_preview': preview, 'session': clean(cells[1].text()),
|
||||||
|
'program_url': 'https://roboticsconference.org'+links[0]}
|
||||||
|
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
||||||
|
raise ValueError('Reviewed 210-entry source census changed')
|
||||||
|
return index, program
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(html, pid, index, program):
|
||||||
|
nodes = list(Document(html).root.walk()); meta = {}
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
||||||
|
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
||||||
|
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
||||||
|
day = one(meta.get('citation_publication_date', []))
|
||||||
|
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
||||||
|
raise ValueError('Unexpected publication date')
|
||||||
|
date.fromisoformat(day.replace('/', '-'))
|
||||||
|
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
||||||
|
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
||||||
|
or not authors or any(not clean(a) for a in authors)
|
||||||
|
or comparable(title) != comparable(index['title'])
|
||||||
|
or comparable(', '.join(authors)) != comparable(index['authors'])
|
||||||
|
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
||||||
|
raise ValueError('Publisher citation/index/visible metadata disagree')
|
||||||
|
pdf = one(meta.get('citation_pdf_url', []))
|
||||||
|
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
||||||
|
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
||||||
|
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
||||||
|
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
||||||
|
raise ValueError('BibTeX DOI/year mismatch')
|
||||||
|
variants = []
|
||||||
|
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
||||||
|
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
||||||
|
raise ValueError('Unreviewed complete author variant')
|
||||||
|
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
||||||
|
if comparable(title) != comparable(program['title']):
|
||||||
|
if (pid, title, program['title']) != (89,
|
||||||
|
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
||||||
|
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
||||||
|
raise ValueError('Unreviewed program title variant')
|
||||||
|
variants.append('reviewed_program_title_variant_not_erratum')
|
||||||
|
inside = False; parts = []; markers = []
|
||||||
|
for n in (n for n in nodes if n.tag == 'p'):
|
||||||
|
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
||||||
|
if 'Abstract:' in labels:
|
||||||
|
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
||||||
|
if value: parts.append(value)
|
||||||
|
elif 'Download:' in labels:
|
||||||
|
inside = False; markers.append('end')
|
||||||
|
elif inside:
|
||||||
|
parts.append(clean(n.text()))
|
||||||
|
if markers != ['start', 'end'] or not any(parts):
|
||||||
|
raise ValueError('Missing or changed publisher abstract layout')
|
||||||
|
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
||||||
|
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots=None, cache=None):
|
||||||
|
if (snapshots is None) == (cache is None):
|
||||||
|
raise ValueError('Specify snapshots or cache')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
||||||
|
out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
out.mkdir()
|
||||||
|
evidence = {}; receipts = {}
|
||||||
|
if snapshots is not None:
|
||||||
|
root = Path(snapshots)
|
||||||
|
try:
|
||||||
|
for r in json.loads((root/'receipts.json').read_bytes()):
|
||||||
|
name = r['file']
|
||||||
|
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
||||||
|
raise ValueError('Unsafe or repeated snapshot receipt')
|
||||||
|
receipts[name] = r
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
def page(name, url):
|
||||||
|
if snapshots is not None:
|
||||||
|
path = root/name
|
||||||
|
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
||||||
|
raw = path.read_bytes(); r = receipts[name]
|
||||||
|
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
||||||
|
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
||||||
|
raise ValueError('Snapshot hash/receipt mismatch')
|
||||||
|
at = r['at']
|
||||||
|
else:
|
||||||
|
raw = fetch(url, cache, limit=8*1024*1024)
|
||||||
|
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
||||||
|
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
||||||
|
at = sidecar['retrieved_at']
|
||||||
|
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Invalid source size/timezone')
|
||||||
|
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
||||||
|
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
||||||
|
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
||||||
|
return raw.decode('utf-8')
|
||||||
|
try:
|
||||||
|
pages = {name: page(name, url) for name, url in PAGES.items()}
|
||||||
|
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
||||||
|
records = []
|
||||||
|
for pid in sorted(index):
|
||||||
|
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
||||||
|
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
||||||
|
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
||||||
|
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
||||||
|
'observed_at': evidence[name]['observed_at'],
|
||||||
|
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
||||||
|
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
||||||
|
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
||||||
|
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
||||||
|
'policy_evidence': evidence['cfp.html'],
|
||||||
|
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
||||||
|
records.append(row)
|
||||||
|
path = out/'rss-proceedings-2026-main.jsonl'
|
||||||
|
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
||||||
|
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
||||||
|
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
p.add_argument('--out', required=True, type=Path)
|
||||||
|
g = p.add_mutually_exclusive_group(required=True)
|
||||||
|
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
"""Frozen SDM 2026 publication records and separate public program observations."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import date, datetime
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_sdm_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
BOOK = '10.1137/1.9781611979329'
|
||||||
|
ISBN = '9781611979329'
|
||||||
|
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
|
||||||
|
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
|
||||||
|
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
|
||||||
|
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
|
||||||
|
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
|
||||||
|
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root, name):
|
||||||
|
expected, key = SNAPSHOTS[name]
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed snapshot manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
|
||||||
|
raise ValueError('Snapshot timestamp requires timezone')
|
||||||
|
files = {'manifest.json': raw}
|
||||||
|
for receipt in manifest[key]:
|
||||||
|
if 'file' not in receipt:
|
||||||
|
# A recorded 403 is an access observation, not an empty data set.
|
||||||
|
continue
|
||||||
|
filename = receipt['file']; path = directory/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated snapshot source file')
|
||||||
|
files[filename] = body
|
||||||
|
return manifest, files
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(message):
|
||||||
|
items = message.get('items', [])
|
||||||
|
if message.get('total-results') != 55 or len(items) != 55:
|
||||||
|
raise ValueError('Incomplete exact-ISBN registry census')
|
||||||
|
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
|
||||||
|
raise ValueError('Unreviewed publication item types')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for original in items:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
|
||||||
|
raise ValueError('Missing/repeated DOI or wrong ISBN')
|
||||||
|
seen.add(doi)
|
||||||
|
if doi in (BOOK, BOOK+'.fm'):
|
||||||
|
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
|
||||||
|
if (original['type'], original.get('title')) != expected:
|
||||||
|
raise ValueError('Unreviewed excluded volume/front matter')
|
||||||
|
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
|
||||||
|
continue
|
||||||
|
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
|
||||||
|
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
|
||||||
|
or original.get('relation', {}).get('is-part-of') != [
|
||||||
|
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
|
||||||
|
raise ValueError('Wrong publication edition, publisher or parent DOI')
|
||||||
|
title = one(original.get('title', []))
|
||||||
|
if not isinstance(title, str) or not clean(title):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
flags = []
|
||||||
|
if '<' in title or '>' in title:
|
||||||
|
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
|
||||||
|
if not match:
|
||||||
|
raise ValueError('Unreviewed title markup')
|
||||||
|
comparison_title = clean(match[1])
|
||||||
|
flags.append('title_sup_asterisk_footnote_requires_review')
|
||||||
|
else:
|
||||||
|
comparison_title = clean(Document(title).root.text())
|
||||||
|
displayed_title = clean(Document(title).root.text())
|
||||||
|
authors = original.get('author')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
|
||||||
|
or not clean(a['given']) or not clean(a['family']) for a in authors)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Incomplete registry authors or order evidence')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if any('@' in n or 'anonymous' in n.casefold() for n in names):
|
||||||
|
raise ValueError('Unreviewed registry author identity')
|
||||||
|
dates = {}
|
||||||
|
for field in ('published', 'published-print', 'published-online', 'issued'):
|
||||||
|
if field in original:
|
||||||
|
parts = one(original[field]['date-parts'])
|
||||||
|
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
|
||||||
|
raise ValueError('Unreviewed registry publication date')
|
||||||
|
dates[field] = date(*parts).isoformat()
|
||||||
|
if not dates or original.get('abstract'):
|
||||||
|
raise ValueError('Reviewed registry date/abstract availability changed')
|
||||||
|
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
|
||||||
|
'authors': names, 'review_flags': flags, 'publication_dates': dates})
|
||||||
|
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
|
||||||
|
raise ValueError('Incomplete reviewed 53-chapter DOI set')
|
||||||
|
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
|
||||||
|
|
||||||
|
|
||||||
|
class SessionParser(HTMLParser):
|
||||||
|
"""The source omits closing dt/dd tags; each next dt starts a new event."""
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def finish(self):
|
||||||
|
if self.current is not None:
|
||||||
|
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
|
||||||
|
if not match or not clean(self.current['authors_literal']):
|
||||||
|
raise ValueError('Incomplete contributed program event')
|
||||||
|
self.current['time_literal'], self.current['title'] = match.groups()
|
||||||
|
self.current['authors_literal'] = clean(self.current['authors_literal'])
|
||||||
|
self.rows.append(self.current)
|
||||||
|
self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
attrs = dict(attrs)
|
||||||
|
if tag == 'dt':
|
||||||
|
self.finish()
|
||||||
|
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
|
||||||
|
if self.current is not None:
|
||||||
|
if tag == 'strong': self.strong = True
|
||||||
|
elif tag == 'dd': self.author = True
|
||||||
|
elif tag == 's': self.current['cancelled_presentation'] = True
|
||||||
|
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
|
||||||
|
self.current['abstract_links'].append(attrs['href'])
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if tag == 'strong': self.strong = False
|
||||||
|
elif tag == 'dl': self.finish()
|
||||||
|
|
||||||
|
def handle_data(self, text):
|
||||||
|
if self.current is not None:
|
||||||
|
if self.strong: self.current['timed_title'] += text
|
||||||
|
elif self.author: self.current['authors_literal'] += text
|
||||||
|
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
|
||||||
|
|
||||||
|
|
||||||
|
def talk_fields(raw, session_url):
|
||||||
|
nodes = list(Document(raw.decode('utf-8')).root.walk())
|
||||||
|
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
|
||||||
|
raise ValueError('Wrong program page edition')
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
|
||||||
|
raise ValueError('Missing public program title/abstract')
|
||||||
|
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
|
||||||
|
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
|
||||||
|
def session_key(url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
|
||||||
|
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
|
||||||
|
raise ValueError('Foreign program session link')
|
||||||
|
return query['sessioncode']
|
||||||
|
if session_key(link) != session_key(session_url):
|
||||||
|
raise ValueError('Program page belongs to a different session')
|
||||||
|
authors = []
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'li':
|
||||||
|
# Source rows contain name, affiliations, country and contact email.
|
||||||
|
# Only the independently checked leading name is a metadata field.
|
||||||
|
literal = clean(n.text()); name, separator, _ = literal.partition(',')
|
||||||
|
if not separator or not name or '@' in name:
|
||||||
|
raise ValueError('Unreviewed program author row')
|
||||||
|
authors.append(name)
|
||||||
|
if not authors:
|
||||||
|
raise ValueError('Missing source-reported program authors')
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
|
||||||
|
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
|
||||||
|
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
|
||||||
|
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(snapshots, papers):
|
||||||
|
official, official_files = snapshots['sdm2026-official-research-v1']
|
||||||
|
sessions, session_files = snapshots['sdm2026-program-research-v1']
|
||||||
|
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
|
||||||
|
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
|
||||||
|
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
|
||||||
|
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
|
||||||
|
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
|
||||||
|
raise ValueError('Contributed session membership disagrees with main program')
|
||||||
|
receipts = {r['url']: r for r in pages['records']}
|
||||||
|
if len(receipts) != len(pages['records']) or len(receipts) != 52:
|
||||||
|
raise ValueError('Incomplete or repeated public talk receipts')
|
||||||
|
rows = []; used = set()
|
||||||
|
for session in sessions['sessions']:
|
||||||
|
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
|
||||||
|
for position, event in enumerate(parser.rows, 1):
|
||||||
|
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
|
||||||
|
event.update(record_kind='source_observation_not_import_record', session=session['title'],
|
||||||
|
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
|
||||||
|
if not urls and event['cancelled_presentation']:
|
||||||
|
event.update(program_page=None, public_page_fields=None)
|
||||||
|
else:
|
||||||
|
url = one(urls)
|
||||||
|
if url not in receipts or url in used:
|
||||||
|
raise ValueError('Missing or repeated public program page')
|
||||||
|
used.add(url); receipt = receipts[url]
|
||||||
|
if receipt['session_url'] != session['url']:
|
||||||
|
raise ValueError('Public page receipt session disagrees')
|
||||||
|
fields = talk_fields(page_files[receipt['file']], session['url'])
|
||||||
|
if fields['title'] != event['title']:
|
||||||
|
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
|
||||||
|
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
|
||||||
|
candidates = []
|
||||||
|
for paper in papers:
|
||||||
|
if paper['comparison_title'].casefold() == event['title'].casefold():
|
||||||
|
fields = event['public_page_fields']
|
||||||
|
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
|
||||||
|
'comparison': 'title_candidate_only_not_identity_approval',
|
||||||
|
'registry_title_review_flags': paper['review_flags'],
|
||||||
|
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
|
||||||
|
event['registry_candidates'] = candidates
|
||||||
|
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
|
||||||
|
rows.append(event)
|
||||||
|
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed program event census')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
|
||||||
|
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
|
||||||
|
message = json.loads(registry_files['isbn-works.json'])['message']
|
||||||
|
papers, excluded = registry_papers(message)
|
||||||
|
observations = program_observations(loaded, papers)
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (manifest, files) in loaded.items():
|
||||||
|
for filename, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(filename).suffix
|
||||||
|
(out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
|
||||||
|
rows = []
|
||||||
|
for p in papers:
|
||||||
|
original = p['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
|
||||||
|
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
|
||||||
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
|
||||||
|
'observed_at': receipt['at'], 'evidence_url': QUERY,
|
||||||
|
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
|
||||||
|
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
|
||||||
|
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
|
||||||
|
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
|
||||||
|
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
|
||||||
|
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'program-observations.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
|
||||||
|
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
|
||||||
|
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
|
||||||
|
'cancelled_presentations': 1, 'not_an_import_manifest': True},
|
||||||
|
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def checked_html(raw, receipt, url, limit):
|
||||||
|
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||||
|
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||||
|
or receipt.get('sha256') != digest(raw)
|
||||||
|
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||||
|
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||||
|
raise ValueError('Access challenge; stop without bypass')
|
||||||
|
return raw, receipt
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *args, **kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCache:
|
||||||
|
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||||
|
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||||
|
self.root = Path(directory)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self.root.is_symlink():
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
self.origin = origin
|
||||||
|
self.limit = limit
|
||||||
|
self.delay = delay
|
||||||
|
self.opener = urllib.request.build_opener(NoRedirect())
|
||||||
|
|
||||||
|
def paths(self, url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||||
|
raise ValueError('Unapproved public metadata origin')
|
||||||
|
key = digest(url.encode())
|
||||||
|
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||||
|
|
||||||
|
def cached(self, url):
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||||
|
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
if any(p.exists() for p in parts):
|
||||||
|
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||||
|
if failure.exists():
|
||||||
|
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||||
|
if body.exists() != receipt.exists():
|
||||||
|
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||||
|
if not body.exists():
|
||||||
|
return None
|
||||||
|
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||||
|
raise ValueError('Oversized cache entry')
|
||||||
|
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
cached = self.cached(url)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
try:
|
||||||
|
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||||
|
with self.opener.open(request, timeout=30) as response:
|
||||||
|
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||||
|
'content_type': response.headers.get('Content-Type', '')}
|
||||||
|
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||||
|
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||||
|
raw = response.read(self.limit+1)
|
||||||
|
if len(raw) > self.limit:
|
||||||
|
raise ValueError('Metadata response exceeds size limit')
|
||||||
|
expected = response.headers.get('Content-Length')
|
||||||
|
if expected is not None and int(expected) != len(raw):
|
||||||
|
raise ConnectionError('Incomplete metadata response')
|
||||||
|
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||||
|
checked_html(raw, rec, url, self.limit)
|
||||||
|
except (urllib.error.HTTPError, ValueError) as error:
|
||||||
|
if isinstance(error, urllib.error.HTTPError):
|
||||||
|
status = error.code
|
||||||
|
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||||
|
error.close()
|
||||||
|
else:
|
||||||
|
status = None
|
||||||
|
retry_after = None
|
||||||
|
# Access errors remain sticky until the operator verifies availability.
|
||||||
|
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||||
|
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||||
|
raise
|
||||||
|
part = body.with_suffix('.html.part')
|
||||||
|
part.write_bytes(raw); part.replace(body)
|
||||||
|
part = receipt.with_suffix('.json.part')
|
||||||
|
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||||
|
time.sleep(self.delay)
|
||||||
|
return raw, rec
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,75 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import corl2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class CoRL2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def test_official_census_and_presentation_dedup(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html'])
|
||||||
|
self.assertEqual(len(program), 687)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||||
|
|
||||||
|
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||||
|
html = self.bundle['official_html']
|
||||||
|
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||||
|
self.assertIn(title, html)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||||
|
|
||||||
|
def test_foreign_forum_source_is_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_unknown_profile_fields_are_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['authorids'] = ['profile']
|
||||||
|
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_source_sets_must_agree(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'].pop()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||||
|
official.mkdir(); archive.mkdir()
|
||||||
|
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||||
|
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||||
|
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||||
|
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||||
|
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||||
|
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||||
|
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||||
|
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
(official/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertTrue((root/'failed/failed.json').exists())
|
||||||
|
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_forum_identity_validation(self):
|
||||||
|
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.forum_id(bad)
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import eccv2026 as c
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||||
|
|
||||||
|
|
||||||
|
class ECCV2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||||
|
for name, raw in self.bundle['official_pages'].items():
|
||||||
|
(official/name).write_bytes(raw.encode('utf-8'))
|
||||||
|
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, raw in self.bundle['paper_pages'].items():
|
||||||
|
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||||
|
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||||
|
return official, paper
|
||||||
|
|
||||||
|
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||||
|
self.assertEqual(len(self.program), 2834)
|
||||||
|
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||||
|
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||||
|
item = self.program[gold['paper_id']]
|
||||||
|
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||||
|
self.assertIn(ordinal, item['row_ordinals'])
|
||||||
|
|
||||||
|
def test_conflicting_duplicate_not_silently_folded(self):
|
||||||
|
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||||
|
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||||
|
|
||||||
|
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||||
|
html = self.bundle['official_pages']['accepted.html']
|
||||||
|
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||||
|
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_public_page_fields_match_independent_sample_review(self):
|
||||||
|
for gold in self.bundle['paper_gold']['records']:
|
||||||
|
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||||
|
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||||
|
|
||||||
|
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||||
|
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_paper(changed, url, self.program['4048'])
|
||||||
|
|
||||||
|
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||||
|
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||||
|
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||||
|
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||||
|
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||||
|
|
||||||
|
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root)
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||||
|
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||||
|
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
|
||||||
|
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||||
|
for r in self.bundle['paper_manifest']['records']:
|
||||||
|
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||||
|
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||||
|
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||||
|
|
||||||
|
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_structured_math_markup_requires_explicit_review(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']
|
||||||
|
self.assertIn('<p>Proactive tamper', raw)
|
||||||
|
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||||
|
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||||
|
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||||
|
|
||||||
|
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||||
|
import hashlib
|
||||||
|
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||||
|
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(len(raw), receipt['bytes'])
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
for receipt in self.bundle['paper_manifest']['records']:
|
||||||
|
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
|
||||||
|
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||||
|
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||||
|
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
|
# Real changes to the author or an extra encoding layer remain errors.
|
||||||
|
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||||
|
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||||
|
|
||||||
|
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertIn('&', fields['title'])
|
||||||
|
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||||
|
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||||
|
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||||
|
|
||||||
|
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCacheTests(unittest.TestCase):
|
||||||
|
def receipt(self, raw):
|
||||||
|
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||||
|
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||||
|
|
||||||
|
def test_old_cache_reused_without_network_or_expiry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||||
|
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||||
|
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||||
|
for kind in ('hash','orphan','part'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||||
|
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||||
|
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||||
|
c.opener = MagicMock()
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||||
|
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||||
|
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||||
|
self.assertEqual(c.opener.open.call_count,1)
|
||||||
|
|
||||||
|
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||||
|
for kind in ('redirect','type','challenge','size'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||||
|
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||||
|
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||||
|
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||||
|
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||||
|
if kind=='size':r.read.return_value=b'x'*101
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
self.assertFalse(c.paths(URL)[0].exists())
|
||||||
|
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||||
|
|
||||||
|
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||||
|
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||||
|
|
||||||
|
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||||
|
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||||
|
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
|
||||||
|
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = URL
|
||||||
|
raw = b'<html>public metadata</html>'
|
||||||
|
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||||
|
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||||
|
self.assertEqual(c.opener.open.call_count, 1)
|
||||||
|
body, receipt, _ = c.paths(URL)
|
||||||
|
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||||
|
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||||
|
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||||
|
|
||||||
|
def test_changed_url_is_rejected_before_reading_body(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||||
|
response.headers = {'Content-Type':'text/html'}
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
response.read.assert_not_called()
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPS2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
root.mkdir()
|
||||||
|
for name, html in self.bundle['pages'].items():
|
||||||
|
(root/name).write_text(html)
|
||||||
|
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
|
||||||
|
return root
|
||||||
|
|
||||||
|
def test_full_listing_matches_independently_recorded_literals(self):
|
||||||
|
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
|
||||||
|
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
|
||||||
|
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
|
||||||
|
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
|
||||||
|
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
|
||||||
|
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
|
||||||
|
|
||||||
|
def test_incomplete_and_duplicate_listing_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
|
||||||
|
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
|
||||||
|
|
||||||
|
def test_changed_edition_or_section_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
|
||||||
|
html.replace(c.MAIN, 'Workshop Papers')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(changed)
|
||||||
|
|
||||||
|
def test_ambiguous_author_lists_not_guessed(self):
|
||||||
|
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.authors_from_literal(literal)
|
||||||
|
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
|
||||||
|
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
|
||||||
|
|
||||||
|
def test_collection_separates_presentations_and_unresolved_tracks(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
|
||||||
|
report = c.collect(out, snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
previous = json.loads((out/report['previously_published_file']).read_text())
|
||||||
|
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
|
||||||
|
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
|
||||||
|
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 90)
|
||||||
|
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['files'][0]['status'], 'partial')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, snapshots=src)
|
||||||
|
|
||||||
|
def test_tampering_retains_failure_without_success_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src')
|
||||||
|
(src/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
|
||||||
|
for kind in ('redirect', 'time', 'duplicate'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
|
||||||
|
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
|
||||||
|
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
|
||||||
|
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
|
||||||
|
p.write_text(json.dumps(r))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_snapshot_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps_registry2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPSRegistry2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
|
||||||
|
cls.snapshot = json.loads(cls.raw)
|
||||||
|
|
||||||
|
def test_exact_edition_not_publication_year_alone(self):
|
||||||
|
papers, excluded = c.registry_papers(self.snapshot)
|
||||||
|
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
|
||||||
|
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
|
||||||
|
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
|
||||||
|
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
|
||||||
|
|
||||||
|
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, set(original))
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; o = original[doi]
|
||||||
|
self.assertEqual(r['metadata']['registry_fields'], o)
|
||||||
|
self.assertEqual(r['title'], o['title'][0])
|
||||||
|
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
|
||||||
|
self.assertEqual(r['identifiers'], {'doi': doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
|
||||||
|
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(r['track'], 'main_tracks_unclassified')
|
||||||
|
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
|
||||||
|
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
|
||||||
|
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
|
||||||
|
self.assertIn('Stephen F. Smith', changed['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
|
||||||
|
def test_math_markup_is_not_flattened_as_plain_numbers(self):
|
||||||
|
self.assertEqual(c.abstract_text('<jats:p>A < B and C & D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
|
||||||
|
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.abstract_text(raw)
|
||||||
|
|
||||||
|
def test_invalid_or_repeated_doi_and_edition_rejected(self):
|
||||||
|
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
|
||||||
|
if kind == 'doi': r['DOI'] = '10.9999/foreign'
|
||||||
|
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
|
||||||
|
if kind == 'volume': r['volume'] = '37'
|
||||||
|
if kind == 'container': r['container-title'] = ['Other Conference']
|
||||||
|
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_missing_authors_and_changed_order_evidence_rejected(self):
|
||||||
|
for kind in ('missing', 'sequence', 'profile'):
|
||||||
|
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
|
||||||
|
if kind == 'missing': r['authors'][0].pop('family')
|
||||||
|
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_receipt_scope_and_timezone_rejected(self):
|
||||||
|
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
|
||||||
|
s = copy.deepcopy(self.snapshot); s[key] = value
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_tamper_failure_retained_and_success_not_published(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||||
|
|
||||||
|
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||||
|
result = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(result), len(self.gold['entries']))
|
||||||
|
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||||
|
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||||
|
self.assertEqual(row[field], gold[field])
|
||||||
|
self.assertEqual(row['source_position'], position)
|
||||||
|
for field, value in gold['session'].items():
|
||||||
|
self.assertEqual(row['session'][field], value)
|
||||||
|
|
||||||
|
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||||
|
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||||
|
all_rows = []
|
||||||
|
for manifest in report['files']:
|
||||||
|
path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||||
|
self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in raw.splitlines()]
|
||||||
|
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||||
|
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||||
|
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||||
|
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||||
|
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||||
|
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||||
|
all_rows.extend(rows)
|
||||||
|
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||||
|
for item in report['evidence']:
|
||||||
|
raw = (root/'out'/item['output_file']).read_bytes()
|
||||||
|
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||||
|
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||||
|
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_stale_or_different_conference_rejected(self):
|
||||||
|
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||||
|
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_track_attribute_must_agree_with_visible_label(self):
|
||||||
|
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'track'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||||
|
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||||
|
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||||
|
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||||
|
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_empty_author_is_not_silently_dropped(self):
|
||||||
|
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'author'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||||
|
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||||
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertFalse((root/'out').exists())
|
||||||
|
|
||||||
|
def test_snapshot_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||||
|
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from paper_atlas.collectors import rss2026 as rss
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
|
||||||
|
|
||||||
|
class Rss2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
|
||||||
|
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
|
||||||
|
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
|
||||||
|
|
||||||
|
def lists(self,**changes):
|
||||||
|
pages={**self.pages,**changes}
|
||||||
|
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
|
||||||
|
|
||||||
|
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
|
||||||
|
report=rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
|
||||||
|
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
for row,gold in zip(rows,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
|
||||||
|
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
|
||||||
|
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
|
||||||
|
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
|
||||||
|
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
|
||||||
|
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
|
||||||
|
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'],0)
|
||||||
|
|
||||||
|
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
|
||||||
|
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
|
||||||
|
self.assertEqual(len(shortened),6)
|
||||||
|
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
|
||||||
|
self.assertEqual(len(p['authors']),55)
|
||||||
|
self.assertFalse(p['program_variants'])
|
||||||
|
|
||||||
|
def test_six_reviewed_name_variants_preserve_original_strings(self):
|
||||||
|
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
|
||||||
|
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
|
||||||
|
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
|
||||||
|
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
|
||||||
|
|
||||||
|
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
|
||||||
|
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
|
||||||
|
self.assertTrue(p['title'].startswith('RLinf-VLA'))
|
||||||
|
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
|
||||||
|
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
|
||||||
|
|
||||||
|
def test_unreviewed_author_or_title_variant_stops(self):
|
||||||
|
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unreviewed'):
|
||||||
|
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
|
||||||
|
|
||||||
|
def test_demo_policy_cannot_be_reused_from2025(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'policy'):
|
||||||
|
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
|
||||||
|
|
||||||
|
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
|
||||||
|
html=self.pages['p001.html']
|
||||||
|
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
|
||||||
|
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
|
||||||
|
self.assertIn(old,html)
|
||||||
|
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
|
||||||
|
|
||||||
|
def test_index_census_and_preview_tampering_stop(self):
|
||||||
|
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
|
||||||
|
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
|
||||||
|
|
||||||
|
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
|
||||||
|
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
|
||||||
|
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
|
||||||
|
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertFalse((self.root/'out/collection-report.json').exists())
|
||||||
|
self.assertTrue((self.root/'out/failed.json').exists())
|
||||||
|
|
||||||
|
def test_existing_output_and_unsafe_receipt_are_rejected(self):
|
||||||
|
out=self.root/'existing';out.mkdir()
|
||||||
|
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
|
||||||
|
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(receipts))
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
|
||||||
|
self.assertTrue((self.root/'unsafe/failed.json').exists())
|
||||||
|
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import sdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class SDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
|
||||||
|
papers, excluded = c.registry_papers(self.message)
|
||||||
|
self.assertEqual(len(papers), 53)
|
||||||
|
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
|
||||||
|
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
|
||||||
|
marked = [p for p in papers if p['review_flags']]
|
||||||
|
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
|
||||||
|
for p in marked:
|
||||||
|
self.assertTrue(p['title'].endswith('*'))
|
||||||
|
self.assertIn('<sup>*</sup>', p['original']['title'][0])
|
||||||
|
|
||||||
|
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
|
||||||
|
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
|
||||||
|
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
|
||||||
|
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
|
||||||
|
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
|
||||||
|
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
|
||||||
|
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
|
||||||
|
sidecar = report['program_observations']
|
||||||
|
self.assertTrue(sidecar['not_an_import_manifest'])
|
||||||
|
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(len(observations), 53)
|
||||||
|
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
|
||||||
|
for observation in observations:
|
||||||
|
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
|
||||||
|
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
|
||||||
|
cancelled, = [x for x in observations if x['cancelled_presentation']]
|
||||||
|
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
|
||||||
|
self.assertIsNone(cancelled['public_page_fields'])
|
||||||
|
# The real source has conflicting author sets under the same title.
|
||||||
|
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
|
||||||
|
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
|
||||||
|
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
|
||||||
|
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
|
||||||
|
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
|
||||||
|
# Every copied source and manifest remains byte-identical, including CRLF.
|
||||||
|
for evidence in report['evidence']:
|
||||||
|
body = (root/'out'/evidence['output_file']).read_bytes()
|
||||||
|
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
|
||||||
|
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_all_program_events_match_independent_inventory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root)
|
||||||
|
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
|
||||||
|
papers, _ = c.registry_papers(self.message)
|
||||||
|
observations = c.program_observations(loaded, papers)
|
||||||
|
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
|
||||||
|
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
|
||||||
|
for item in observations:
|
||||||
|
original = gold[(item['session_url'], item['title'])]
|
||||||
|
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
|
||||||
|
self.assertEqual(item[key], original[key])
|
||||||
|
|
||||||
|
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
|
||||||
|
files = self.fixture['files']
|
||||||
|
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
|
||||||
|
receipts = {r['url']: r for r in manifest['records']}
|
||||||
|
self.assertEqual(len(self.fixture['page_gold']), 52)
|
||||||
|
for gold in self.fixture['page_gold']:
|
||||||
|
receipt = receipts[gold['program_url']]
|
||||||
|
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
|
||||||
|
fields = c.talk_fields(raw, receipt['session_url'])
|
||||||
|
self.assertEqual(fields['title'], gold['title'])
|
||||||
|
self.assertEqual(fields['source_reported_authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract'])
|
||||||
|
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
|
||||||
|
|
||||||
|
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
|
||||||
|
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
message = copy.deepcopy(self.message)
|
||||||
|
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
|
||||||
|
if kind == 'isbn': row['ISBN'] = ['9780000000000']
|
||||||
|
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
|
||||||
|
if kind == 'repeat': row['DOI'] = c.BOOK
|
||||||
|
if kind == 'type': row['type'] = 'other'
|
||||||
|
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
|
||||||
|
if kind == 'author': row['author'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
|
||||||
|
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
|
||||||
|
if kind == 'missing': message['items'].pop()
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(message)
|
||||||
|
|
||||||
|
def test_foreign_session_and_changed_edition_rejected(self):
|
||||||
|
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
|
||||||
|
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
|
||||||
|
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
|
||||||
|
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
|
||||||
|
with self.assertRaises(ValueError): c.talk_fields(changed, url)
|
||||||
|
|
||||||
|
def test_corruption_preserves_failure_without_success_report(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'receipt/body'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
|
||||||
|
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
Reference in New Issue
Block a user