Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f2753bc741 | |||
| 1ea17c810d | |||
| f800c6ecb4 | |||
| 28255743ef | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 | |||
| 59053fd4fb | |||
| b50136d5bc | |||
| 87a0689fcc | |||
| 19d03ced54 | |||
| c33f5422de |
@@ -49,6 +49,51 @@ jobs:
|
|||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
|
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
eccv_official = root/'eccv2026-official'
|
||||||
|
eccv_papers = root/'eccv2026-paper-samples'
|
||||||
|
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||||
|
for name, html in eccv_fixture['official_pages'].items():
|
||||||
|
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, html in eccv_fixture['paper_pages'].items():
|
||||||
|
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||||
|
eccv_encoding = eccv_fixture['encoding_sample']
|
||||||
|
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||||
|
eccv_literal = eccv_fixture['literal_title_sample']
|
||||||
|
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||||
|
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||||
|
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||||
|
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||||
|
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||||
|
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||||
|
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||||
|
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||||
|
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||||
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||||
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||||
|
assert '&' in eccv_literal_row['title']
|
||||||
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
sdm_snapshots = root/'sdm2026-snapshots'
|
||||||
|
for name, text in sdm_fixture['files'].items():
|
||||||
|
target = sdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
||||||
|
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
||||||
|
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
||||||
|
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
||||||
|
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
||||||
|
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
||||||
|
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
||||||
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||||
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||||
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
icaps_snapshots = root/'icaps2026-snapshots'
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
icaps_snapshots.mkdir()
|
icaps_snapshots.mkdir()
|
||||||
@@ -75,6 +120,19 @@ jobs:
|
|||||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
corl_official = root/'corl2026-official'
|
||||||
|
corl_archive = root/'corl2026-archive'
|
||||||
|
corl_official.mkdir()
|
||||||
|
corl_archive.mkdir()
|
||||||
|
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||||
|
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||||
|
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||||
|
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||||
|
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||||
|
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||||
|
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||||
|
|||||||
@@ -124,3 +124,11 @@ RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 de
|
|||||||
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||||
|
|
||||||
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
||||||
|
|
||||||
|
### CoRL 2026 官方程序快照
|
||||||
|
|
||||||
|
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||||
|
|
||||||
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# CoRL 2026:官方程序与公开档案
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect corl2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||||
|
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||||
|
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||||
|
|
||||||
|
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||||
|
|
||||||
|
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||||
|
|
||||||
|
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||||
|
|
||||||
|
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||||
|
|
||||||
|
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||||
|
|
||||||
|
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# ECCV 2026:暂定节目与公开元数据
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect eccv2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||||
|
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||||
|
--max-new-pages 200 \
|
||||||
|
--out /data/paper-index/collected/eccv2026-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||||
|
|
||||||
|
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||||
|
|
||||||
|
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||||
|
|
||||||
|
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||||
|
|
||||||
|
## 来源边界
|
||||||
|
|
||||||
|
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||||
|
|
||||||
|
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||||
|
|
||||||
|
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||||
|
|
||||||
|
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||||
|
|
||||||
|
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||||
|
|
||||||
|
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||||
|
|
||||||
|
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||||
@@ -0,0 +1,39 @@
|
|||||||
|
# ECCV 2026 来源核查记录
|
||||||
|
|
||||||
|
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||||
|
|
||||||
|
## 贡献类型与 track
|
||||||
|
|
||||||
|
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||||
|
|
||||||
|
| 官方 group 字段值 | 含义摘要 |
|
||||||
|
| --- | --- |
|
||||||
|
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||||
|
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||||
|
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||||
|
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||||
|
| Concept & Feasibility | 新概念及可行性证据 |
|
||||||
|
|
||||||
|
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||||
|
|
||||||
|
## 公开负面决定元数据
|
||||||
|
|
||||||
|
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||||
|
|
||||||
|
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||||
|
|
||||||
|
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||||
|
|
||||||
|
## 可恢复证据与后续入口
|
||||||
|
|
||||||
|
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||||
|
|
||||||
|
| 文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||||
|
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||||
|
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||||
|
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||||
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
# SDM 2026:论文集与日程版本
|
||||||
|
|
||||||
|
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect sdm2026-proceedings \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/sdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
|
||||||
|
|
||||||
|
## 两种输出
|
||||||
|
|
||||||
|
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
|
||||||
|
|
||||||
|
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
|
||||||
|
|
||||||
|
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
|
||||||
|
|
||||||
|
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
|
||||||
|
|
||||||
|
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
|
||||||
|
|
||||||
|
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
|
||||||
|
|
||||||
|
## 公开拒稿与访问限制
|
||||||
|
|
||||||
|
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
|
||||||
|
|
||||||
|
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
|
||||||
|
|
||||||
|
## 来源恢复与后续核查
|
||||||
|
|
||||||
|
| 冻结目录 | manifest SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
|
||||||
|
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
|
||||||
|
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
|
||||||
|
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||||
|
|
||||||
|
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||||
@@ -24,11 +24,14 @@ COMMANDS = {
|
|||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||||
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||||
|
|||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'corl_program_archive'
|
||||||
|
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||||
|
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||||
|
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||||
|
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||||
|
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||||
|
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||||
|
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||||
|
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||||
|
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||||
|
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||||
|
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def forum_id(url):
|
||||||
|
if not isinstance(url, str):
|
||||||
|
raise ValueError('Forum URL must be text')
|
||||||
|
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||||
|
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||||
|
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||||
|
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||||
|
raise ValueError('Invalid original forum identity')
|
||||||
|
return q['id'][0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||||
|
if len(embeds) != 1:
|
||||||
|
raise ValueError('Expected one official embedded program')
|
||||||
|
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected program edition')
|
||||||
|
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||||
|
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||||
|
raise ValueError('Reviewed program census changed')
|
||||||
|
papers = {}; sessions = {}; occurrences = []
|
||||||
|
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||||
|
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||||
|
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||||
|
raise ValueError('Unexpected or repeated presentation session')
|
||||||
|
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||||
|
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||||
|
if len(headings) != 1:
|
||||||
|
raise ValueError('Ambiguous session heading')
|
||||||
|
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||||
|
seen = set()
|
||||||
|
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||||
|
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||||
|
if fid in seen or not title:
|
||||||
|
raise ValueError('Repeated or empty paper in session')
|
||||||
|
seen.add(fid)
|
||||||
|
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||||
|
if paper['title'] != title:
|
||||||
|
raise ValueError('Conflicting official titles share a forum')
|
||||||
|
event = {'session_id': sid, **sessions[sid]}
|
||||||
|
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||||
|
counts = Counter(kind for _, _, kind in occurrences)
|
||||||
|
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||||
|
raise ValueError('Incomplete official program')
|
||||||
|
for paper in papers.values():
|
||||||
|
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||||
|
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||||
|
raise ValueError('Invalid poster/talk multiplicity')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def join_metadata(program, archive):
|
||||||
|
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||||
|
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||||
|
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||||
|
or not isinstance(archive.get('papers'), list)):
|
||||||
|
raise ValueError('Unpinned public metadata provenance')
|
||||||
|
indexed = {}
|
||||||
|
for row in archive['papers']:
|
||||||
|
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||||
|
raise ValueError('Unexpected metadata/profile fields')
|
||||||
|
fid = forum_id(row.get('openreview'))
|
||||||
|
if fid != row.get('id') or fid in indexed:
|
||||||
|
raise ValueError('Repeated or mismatched archive forum')
|
||||||
|
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||||
|
raise ValueError('Field source belongs to another submission')
|
||||||
|
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||||
|
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||||
|
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||||
|
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||||
|
raise ValueError('Incomplete scholarly metadata')
|
||||||
|
indexed[fid] = row
|
||||||
|
if set(indexed) != set(program):
|
||||||
|
raise ValueError('Official and archive forum sets differ')
|
||||||
|
return indexed
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, archive_snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||||
|
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||||
|
if official.is_symlink() or filtered.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||||
|
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||||
|
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Missing official source receipt')
|
||||||
|
receipt = matches[0]; observed = receipt['at']
|
||||||
|
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||||
|
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid official receipt')
|
||||||
|
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
sources = {}
|
||||||
|
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||||
|
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||||
|
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||||
|
rows = []; variants = []
|
||||||
|
for fid in sorted(program):
|
||||||
|
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||||
|
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||||
|
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||||
|
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||||
|
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||||
|
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||||
|
'observed_at': observed, 'tags': [], 'metadata': {
|
||||||
|
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||||
|
'source_evidence': sources, 'archive_revision': REVISION,
|
||||||
|
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||||
|
if len(variants) != 34:
|
||||||
|
raise ValueError('Reviewed title-variant census changed')
|
||||||
|
path = out/'corl-program-archive-2026-main.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||||
|
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--out', type=Path, required=True)
|
||||||
|
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||||
|
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import fcntl
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||||
|
|
||||||
|
SOURCE = 'eccv_program'
|
||||||
|
ORIGIN = 'https://eccv.ecva.net'
|
||||||
|
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||||
|
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||||
|
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected one reviewed metadata field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, value):
|
||||||
|
return value in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||||
|
raise ValueError('Unexpected ECCV program edition')
|
||||||
|
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||||
|
raise ValueError('Reviewed provisional-publication notice changed')
|
||||||
|
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||||
|
papers = {}; occurrences = 0
|
||||||
|
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 3:
|
||||||
|
raise ValueError('Unexpected program columns')
|
||||||
|
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||||
|
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||||
|
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||||
|
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||||
|
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||||
|
if not row['title'] or not authors:
|
||||||
|
raise ValueError('Incomplete public program fields')
|
||||||
|
occurrences += 1
|
||||||
|
if pid in papers:
|
||||||
|
if papers[pid]['listing'] != row:
|
||||||
|
raise ValueError('Conflicting entries share a program ID')
|
||||||
|
papers[pid]['row_ordinals'].append(occurrences)
|
||||||
|
else:
|
||||||
|
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||||
|
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||||
|
raise ValueError('Incomplete reviewed program census')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(raw, url, program):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||||
|
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||||
|
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||||
|
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||||
|
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||||
|
authors = ld.get('author')
|
||||||
|
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||||
|
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||||
|
or not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||||
|
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||||
|
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||||
|
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||||
|
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||||
|
# Decode only to test agreement with visible text, never to rewrite it.
|
||||||
|
def agrees_with_display(value, expected):
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
for _ in range(2):
|
||||||
|
value = unescape(value)
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
raw_names = [a['name'] for a in authors]
|
||||||
|
names = [clean(name) for name in raw_names]
|
||||||
|
if not agrees_with_display(ld['name'], title):
|
||||||
|
raise ValueError('Public visible and structured titles disagree')
|
||||||
|
if ' ⋅ '.join(names) != authors_literal:
|
||||||
|
visible_names = authors_literal.split(' ⋅ ')
|
||||||
|
if len(visible_names) != len(raw_names) or any(
|
||||||
|
not agrees_with_display(raw_name, visible_name)
|
||||||
|
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
names = visible_names
|
||||||
|
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||||
|
if raw_names != names or ld['name'] != title:
|
||||||
|
flags.append('jsonld_display_text_normalized_requires_review')
|
||||||
|
if title != program['listing']['title']:
|
||||||
|
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||||
|
if authors_literal != program['listing']['authors_literal']:
|
||||||
|
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||||
|
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||||
|
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||||
|
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||||
|
|
||||||
|
|
||||||
|
def official_snapshot(root):
|
||||||
|
root = Path(root); manifest = root/'manifest.json'
|
||||||
|
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed official snapshot manifest changed')
|
||||||
|
m = json.loads(manifest.read_bytes())
|
||||||
|
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Official snapshot requires a timezone')
|
||||||
|
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||||
|
raise ValueError('Official source set changed')
|
||||||
|
pages = {}; evidence = {}
|
||||||
|
for r in m['sources']:
|
||||||
|
name = r['file']; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe official snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||||
|
raise ValueError('Invalid official source receipt')
|
||||||
|
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||||
|
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||||
|
return m, pages, evidence, program
|
||||||
|
|
||||||
|
|
||||||
|
def paper_snapshots(root, program):
|
||||||
|
root = Path(root); path = root/'manifest.json'
|
||||||
|
if root.is_symlink() or path.is_symlink():
|
||||||
|
raise ValueError('Paper snapshot symlinks rejected')
|
||||||
|
if path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Oversized paper manifest')
|
||||||
|
m = json.loads(path.read_bytes()); result = {}
|
||||||
|
for r in m['records']:
|
||||||
|
pid = r['paper_id']
|
||||||
|
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||||
|
raise ValueError('Foreign or repeated paper snapshot')
|
||||||
|
p = root/r['file']
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe paper snapshot')
|
||||||
|
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||||
|
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||||
|
if (paper_snapshots_dir is None) == (cache is None):
|
||||||
|
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||||
|
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||||
|
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
lock = None
|
||||||
|
try:
|
||||||
|
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||||
|
directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in pages.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||||
|
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||||
|
client = None
|
||||||
|
if cache is not None:
|
||||||
|
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||||
|
raise ValueError('Cache lock symlinks rejected')
|
||||||
|
lock = (cache/'.collector.lock').open('a+')
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
client = PublicHTMLCache(cache, ORIGIN)
|
||||||
|
rows = []; missing = []; fetched = 0; reused = 0
|
||||||
|
for pid in sorted(program, key=int):
|
||||||
|
p = program[pid]
|
||||||
|
if snapshots is not None:
|
||||||
|
response = snapshots.get(pid)
|
||||||
|
else:
|
||||||
|
response = client.cached(p['url'])
|
||||||
|
if response is not None:
|
||||||
|
reused += 1
|
||||||
|
elif max_new_pages is None or fetched < max_new_pages:
|
||||||
|
response = client.get(p['url']); fetched += 1
|
||||||
|
if response is None:
|
||||||
|
missing.append(pid); continue
|
||||||
|
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||||
|
keyword = p['listing']['keywords_literal']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||||
|
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||||
|
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||||
|
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||||
|
'evidence_url': ACCEPTED,
|
||||||
|
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||||
|
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||||
|
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||||
|
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||||
|
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||||
|
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||||
|
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||||
|
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||||
|
if len(rows) % 10 == 0:
|
||||||
|
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||||
|
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||||
|
files = []
|
||||||
|
if rows:
|
||||||
|
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||||
|
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||||
|
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||||
|
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||||
|
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||||
|
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||||
|
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if lock is not None:
|
||||||
|
lock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--official-snapshots', required=True)
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument('--paper-snapshots')
|
||||||
|
group.add_argument('--cache')
|
||||||
|
parser.add_argument('--max-new-pages', type=int)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||||
|
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
"""Frozen SDM 2026 publication records and separate public program observations."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import date, datetime
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_sdm_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
BOOK = '10.1137/1.9781611979329'
|
||||||
|
ISBN = '9781611979329'
|
||||||
|
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
|
||||||
|
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
|
||||||
|
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
|
||||||
|
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
|
||||||
|
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
|
||||||
|
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root, name):
|
||||||
|
expected, key = SNAPSHOTS[name]
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed snapshot manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
|
||||||
|
raise ValueError('Snapshot timestamp requires timezone')
|
||||||
|
files = {'manifest.json': raw}
|
||||||
|
for receipt in manifest[key]:
|
||||||
|
if 'file' not in receipt:
|
||||||
|
# A recorded 403 is an access observation, not an empty data set.
|
||||||
|
continue
|
||||||
|
filename = receipt['file']; path = directory/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated snapshot source file')
|
||||||
|
files[filename] = body
|
||||||
|
return manifest, files
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(message):
|
||||||
|
items = message.get('items', [])
|
||||||
|
if message.get('total-results') != 55 or len(items) != 55:
|
||||||
|
raise ValueError('Incomplete exact-ISBN registry census')
|
||||||
|
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
|
||||||
|
raise ValueError('Unreviewed publication item types')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for original in items:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
|
||||||
|
raise ValueError('Missing/repeated DOI or wrong ISBN')
|
||||||
|
seen.add(doi)
|
||||||
|
if doi in (BOOK, BOOK+'.fm'):
|
||||||
|
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
|
||||||
|
if (original['type'], original.get('title')) != expected:
|
||||||
|
raise ValueError('Unreviewed excluded volume/front matter')
|
||||||
|
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
|
||||||
|
continue
|
||||||
|
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
|
||||||
|
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
|
||||||
|
or original.get('relation', {}).get('is-part-of') != [
|
||||||
|
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
|
||||||
|
raise ValueError('Wrong publication edition, publisher or parent DOI')
|
||||||
|
title = one(original.get('title', []))
|
||||||
|
if not isinstance(title, str) or not clean(title):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
flags = []
|
||||||
|
if '<' in title or '>' in title:
|
||||||
|
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
|
||||||
|
if not match:
|
||||||
|
raise ValueError('Unreviewed title markup')
|
||||||
|
comparison_title = clean(match[1])
|
||||||
|
flags.append('title_sup_asterisk_footnote_requires_review')
|
||||||
|
else:
|
||||||
|
comparison_title = clean(Document(title).root.text())
|
||||||
|
displayed_title = clean(Document(title).root.text())
|
||||||
|
authors = original.get('author')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
|
||||||
|
or not clean(a['given']) or not clean(a['family']) for a in authors)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Incomplete registry authors or order evidence')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if any('@' in n or 'anonymous' in n.casefold() for n in names):
|
||||||
|
raise ValueError('Unreviewed registry author identity')
|
||||||
|
dates = {}
|
||||||
|
for field in ('published', 'published-print', 'published-online', 'issued'):
|
||||||
|
if field in original:
|
||||||
|
parts = one(original[field]['date-parts'])
|
||||||
|
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
|
||||||
|
raise ValueError('Unreviewed registry publication date')
|
||||||
|
dates[field] = date(*parts).isoformat()
|
||||||
|
if not dates or original.get('abstract'):
|
||||||
|
raise ValueError('Reviewed registry date/abstract availability changed')
|
||||||
|
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
|
||||||
|
'authors': names, 'review_flags': flags, 'publication_dates': dates})
|
||||||
|
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
|
||||||
|
raise ValueError('Incomplete reviewed 53-chapter DOI set')
|
||||||
|
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
|
||||||
|
|
||||||
|
|
||||||
|
class SessionParser(HTMLParser):
|
||||||
|
"""The source omits closing dt/dd tags; each next dt starts a new event."""
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def finish(self):
|
||||||
|
if self.current is not None:
|
||||||
|
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
|
||||||
|
if not match or not clean(self.current['authors_literal']):
|
||||||
|
raise ValueError('Incomplete contributed program event')
|
||||||
|
self.current['time_literal'], self.current['title'] = match.groups()
|
||||||
|
self.current['authors_literal'] = clean(self.current['authors_literal'])
|
||||||
|
self.rows.append(self.current)
|
||||||
|
self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
attrs = dict(attrs)
|
||||||
|
if tag == 'dt':
|
||||||
|
self.finish()
|
||||||
|
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
|
||||||
|
if self.current is not None:
|
||||||
|
if tag == 'strong': self.strong = True
|
||||||
|
elif tag == 'dd': self.author = True
|
||||||
|
elif tag == 's': self.current['cancelled_presentation'] = True
|
||||||
|
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
|
||||||
|
self.current['abstract_links'].append(attrs['href'])
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if tag == 'strong': self.strong = False
|
||||||
|
elif tag == 'dl': self.finish()
|
||||||
|
|
||||||
|
def handle_data(self, text):
|
||||||
|
if self.current is not None:
|
||||||
|
if self.strong: self.current['timed_title'] += text
|
||||||
|
elif self.author: self.current['authors_literal'] += text
|
||||||
|
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
|
||||||
|
|
||||||
|
|
||||||
|
def talk_fields(raw, session_url):
|
||||||
|
nodes = list(Document(raw.decode('utf-8')).root.walk())
|
||||||
|
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
|
||||||
|
raise ValueError('Wrong program page edition')
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
|
||||||
|
raise ValueError('Missing public program title/abstract')
|
||||||
|
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
|
||||||
|
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
|
||||||
|
def session_key(url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
|
||||||
|
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
|
||||||
|
raise ValueError('Foreign program session link')
|
||||||
|
return query['sessioncode']
|
||||||
|
if session_key(link) != session_key(session_url):
|
||||||
|
raise ValueError('Program page belongs to a different session')
|
||||||
|
authors = []
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'li':
|
||||||
|
# Source rows contain name, affiliations, country and contact email.
|
||||||
|
# Only the independently checked leading name is a metadata field.
|
||||||
|
literal = clean(n.text()); name, separator, _ = literal.partition(',')
|
||||||
|
if not separator or not name or '@' in name:
|
||||||
|
raise ValueError('Unreviewed program author row')
|
||||||
|
authors.append(name)
|
||||||
|
if not authors:
|
||||||
|
raise ValueError('Missing source-reported program authors')
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
|
||||||
|
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
|
||||||
|
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
|
||||||
|
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(snapshots, papers):
|
||||||
|
official, official_files = snapshots['sdm2026-official-research-v1']
|
||||||
|
sessions, session_files = snapshots['sdm2026-program-research-v1']
|
||||||
|
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
|
||||||
|
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
|
||||||
|
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
|
||||||
|
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
|
||||||
|
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
|
||||||
|
raise ValueError('Contributed session membership disagrees with main program')
|
||||||
|
receipts = {r['url']: r for r in pages['records']}
|
||||||
|
if len(receipts) != len(pages['records']) or len(receipts) != 52:
|
||||||
|
raise ValueError('Incomplete or repeated public talk receipts')
|
||||||
|
rows = []; used = set()
|
||||||
|
for session in sessions['sessions']:
|
||||||
|
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
|
||||||
|
for position, event in enumerate(parser.rows, 1):
|
||||||
|
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
|
||||||
|
event.update(record_kind='source_observation_not_import_record', session=session['title'],
|
||||||
|
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
|
||||||
|
if not urls and event['cancelled_presentation']:
|
||||||
|
event.update(program_page=None, public_page_fields=None)
|
||||||
|
else:
|
||||||
|
url = one(urls)
|
||||||
|
if url not in receipts or url in used:
|
||||||
|
raise ValueError('Missing or repeated public program page')
|
||||||
|
used.add(url); receipt = receipts[url]
|
||||||
|
if receipt['session_url'] != session['url']:
|
||||||
|
raise ValueError('Public page receipt session disagrees')
|
||||||
|
fields = talk_fields(page_files[receipt['file']], session['url'])
|
||||||
|
if fields['title'] != event['title']:
|
||||||
|
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
|
||||||
|
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
|
||||||
|
candidates = []
|
||||||
|
for paper in papers:
|
||||||
|
if paper['comparison_title'].casefold() == event['title'].casefold():
|
||||||
|
fields = event['public_page_fields']
|
||||||
|
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
|
||||||
|
'comparison': 'title_candidate_only_not_identity_approval',
|
||||||
|
'registry_title_review_flags': paper['review_flags'],
|
||||||
|
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
|
||||||
|
event['registry_candidates'] = candidates
|
||||||
|
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
|
||||||
|
rows.append(event)
|
||||||
|
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed program event census')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
|
||||||
|
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
|
||||||
|
message = json.loads(registry_files['isbn-works.json'])['message']
|
||||||
|
papers, excluded = registry_papers(message)
|
||||||
|
observations = program_observations(loaded, papers)
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (manifest, files) in loaded.items():
|
||||||
|
for filename, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(filename).suffix
|
||||||
|
(out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
|
||||||
|
rows = []
|
||||||
|
for p in papers:
|
||||||
|
original = p['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
|
||||||
|
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
|
||||||
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
|
||||||
|
'observed_at': receipt['at'], 'evidence_url': QUERY,
|
||||||
|
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
|
||||||
|
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
|
||||||
|
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
|
||||||
|
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
|
||||||
|
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
|
||||||
|
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'program-observations.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
|
||||||
|
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
|
||||||
|
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
|
||||||
|
'cancelled_presentations': 1, 'not_an_import_manifest': True},
|
||||||
|
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def checked_html(raw, receipt, url, limit):
|
||||||
|
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||||
|
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||||
|
or receipt.get('sha256') != digest(raw)
|
||||||
|
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||||
|
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||||
|
raise ValueError('Access challenge; stop without bypass')
|
||||||
|
return raw, receipt
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *args, **kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCache:
|
||||||
|
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||||
|
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||||
|
self.root = Path(directory)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self.root.is_symlink():
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
self.origin = origin
|
||||||
|
self.limit = limit
|
||||||
|
self.delay = delay
|
||||||
|
self.opener = urllib.request.build_opener(NoRedirect())
|
||||||
|
|
||||||
|
def paths(self, url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||||
|
raise ValueError('Unapproved public metadata origin')
|
||||||
|
key = digest(url.encode())
|
||||||
|
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||||
|
|
||||||
|
def cached(self, url):
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||||
|
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
if any(p.exists() for p in parts):
|
||||||
|
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||||
|
if failure.exists():
|
||||||
|
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||||
|
if body.exists() != receipt.exists():
|
||||||
|
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||||
|
if not body.exists():
|
||||||
|
return None
|
||||||
|
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||||
|
raise ValueError('Oversized cache entry')
|
||||||
|
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
cached = self.cached(url)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
try:
|
||||||
|
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||||
|
with self.opener.open(request, timeout=30) as response:
|
||||||
|
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||||
|
'content_type': response.headers.get('Content-Type', '')}
|
||||||
|
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||||
|
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||||
|
raw = response.read(self.limit+1)
|
||||||
|
if len(raw) > self.limit:
|
||||||
|
raise ValueError('Metadata response exceeds size limit')
|
||||||
|
expected = response.headers.get('Content-Length')
|
||||||
|
if expected is not None and int(expected) != len(raw):
|
||||||
|
raise ConnectionError('Incomplete metadata response')
|
||||||
|
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||||
|
checked_html(raw, rec, url, self.limit)
|
||||||
|
except (urllib.error.HTTPError, ValueError) as error:
|
||||||
|
if isinstance(error, urllib.error.HTTPError):
|
||||||
|
status = error.code
|
||||||
|
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||||
|
error.close()
|
||||||
|
else:
|
||||||
|
status = None
|
||||||
|
retry_after = None
|
||||||
|
# Access errors remain sticky until the operator verifies availability.
|
||||||
|
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||||
|
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||||
|
raise
|
||||||
|
part = body.with_suffix('.html.part')
|
||||||
|
part.write_bytes(raw); part.replace(body)
|
||||||
|
part = receipt.with_suffix('.json.part')
|
||||||
|
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||||
|
time.sleep(self.delay)
|
||||||
|
return raw, rec
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,75 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import corl2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class CoRL2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def test_official_census_and_presentation_dedup(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html'])
|
||||||
|
self.assertEqual(len(program), 687)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||||
|
|
||||||
|
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||||
|
html = self.bundle['official_html']
|
||||||
|
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||||
|
self.assertIn(title, html)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||||
|
|
||||||
|
def test_foreign_forum_source_is_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_unknown_profile_fields_are_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['authorids'] = ['profile']
|
||||||
|
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_source_sets_must_agree(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'].pop()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||||
|
official.mkdir(); archive.mkdir()
|
||||||
|
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||||
|
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||||
|
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||||
|
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||||
|
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||||
|
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||||
|
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||||
|
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
(official/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertTrue((root/'failed/failed.json').exists())
|
||||||
|
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_forum_identity_validation(self):
|
||||||
|
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.forum_id(bad)
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import eccv2026 as c
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||||
|
|
||||||
|
|
||||||
|
class ECCV2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||||
|
for name, raw in self.bundle['official_pages'].items():
|
||||||
|
(official/name).write_bytes(raw.encode('utf-8'))
|
||||||
|
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, raw in self.bundle['paper_pages'].items():
|
||||||
|
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||||
|
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||||
|
return official, paper
|
||||||
|
|
||||||
|
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||||
|
self.assertEqual(len(self.program), 2834)
|
||||||
|
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||||
|
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||||
|
item = self.program[gold['paper_id']]
|
||||||
|
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||||
|
self.assertIn(ordinal, item['row_ordinals'])
|
||||||
|
|
||||||
|
def test_conflicting_duplicate_not_silently_folded(self):
|
||||||
|
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||||
|
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||||
|
|
||||||
|
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||||
|
html = self.bundle['official_pages']['accepted.html']
|
||||||
|
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||||
|
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_public_page_fields_match_independent_sample_review(self):
|
||||||
|
for gold in self.bundle['paper_gold']['records']:
|
||||||
|
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||||
|
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||||
|
|
||||||
|
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||||
|
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_paper(changed, url, self.program['4048'])
|
||||||
|
|
||||||
|
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||||
|
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||||
|
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||||
|
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||||
|
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||||
|
|
||||||
|
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root)
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||||
|
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||||
|
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
|
||||||
|
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||||
|
for r in self.bundle['paper_manifest']['records']:
|
||||||
|
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||||
|
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||||
|
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||||
|
|
||||||
|
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_structured_math_markup_requires_explicit_review(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']
|
||||||
|
self.assertIn('<p>Proactive tamper', raw)
|
||||||
|
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||||
|
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||||
|
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||||
|
|
||||||
|
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||||
|
import hashlib
|
||||||
|
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||||
|
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(len(raw), receipt['bytes'])
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
for receipt in self.bundle['paper_manifest']['records']:
|
||||||
|
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
|
||||||
|
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||||
|
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||||
|
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
|
# Real changes to the author or an extra encoding layer remain errors.
|
||||||
|
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||||
|
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||||
|
|
||||||
|
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertIn('&', fields['title'])
|
||||||
|
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||||
|
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||||
|
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||||
|
|
||||||
|
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCacheTests(unittest.TestCase):
|
||||||
|
def receipt(self, raw):
|
||||||
|
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||||
|
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||||
|
|
||||||
|
def test_old_cache_reused_without_network_or_expiry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||||
|
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||||
|
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||||
|
for kind in ('hash','orphan','part'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||||
|
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||||
|
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||||
|
c.opener = MagicMock()
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||||
|
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||||
|
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||||
|
self.assertEqual(c.opener.open.call_count,1)
|
||||||
|
|
||||||
|
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||||
|
for kind in ('redirect','type','challenge','size'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||||
|
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||||
|
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||||
|
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||||
|
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||||
|
if kind=='size':r.read.return_value=b'x'*101
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
self.assertFalse(c.paths(URL)[0].exists())
|
||||||
|
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||||
|
|
||||||
|
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||||
|
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||||
|
|
||||||
|
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||||
|
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||||
|
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
|
||||||
|
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = URL
|
||||||
|
raw = b'<html>public metadata</html>'
|
||||||
|
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||||
|
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||||
|
self.assertEqual(c.opener.open.call_count, 1)
|
||||||
|
body, receipt, _ = c.paths(URL)
|
||||||
|
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||||
|
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||||
|
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||||
|
|
||||||
|
def test_changed_url_is_rejected_before_reading_body(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||||
|
response.headers = {'Content-Type':'text/html'}
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
response.read.assert_not_called()
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import sdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class SDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
|
||||||
|
papers, excluded = c.registry_papers(self.message)
|
||||||
|
self.assertEqual(len(papers), 53)
|
||||||
|
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
|
||||||
|
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
|
||||||
|
marked = [p for p in papers if p['review_flags']]
|
||||||
|
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
|
||||||
|
for p in marked:
|
||||||
|
self.assertTrue(p['title'].endswith('*'))
|
||||||
|
self.assertIn('<sup>*</sup>', p['original']['title'][0])
|
||||||
|
|
||||||
|
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
|
||||||
|
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
|
||||||
|
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
|
||||||
|
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
|
||||||
|
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
|
||||||
|
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
|
||||||
|
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
|
||||||
|
sidecar = report['program_observations']
|
||||||
|
self.assertTrue(sidecar['not_an_import_manifest'])
|
||||||
|
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(len(observations), 53)
|
||||||
|
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
|
||||||
|
for observation in observations:
|
||||||
|
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
|
||||||
|
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
|
||||||
|
cancelled, = [x for x in observations if x['cancelled_presentation']]
|
||||||
|
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
|
||||||
|
self.assertIsNone(cancelled['public_page_fields'])
|
||||||
|
# The real source has conflicting author sets under the same title.
|
||||||
|
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
|
||||||
|
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
|
||||||
|
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
|
||||||
|
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
|
||||||
|
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
|
||||||
|
# Every copied source and manifest remains byte-identical, including CRLF.
|
||||||
|
for evidence in report['evidence']:
|
||||||
|
body = (root/'out'/evidence['output_file']).read_bytes()
|
||||||
|
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
|
||||||
|
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_all_program_events_match_independent_inventory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root)
|
||||||
|
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
|
||||||
|
papers, _ = c.registry_papers(self.message)
|
||||||
|
observations = c.program_observations(loaded, papers)
|
||||||
|
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
|
||||||
|
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
|
||||||
|
for item in observations:
|
||||||
|
original = gold[(item['session_url'], item['title'])]
|
||||||
|
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
|
||||||
|
self.assertEqual(item[key], original[key])
|
||||||
|
|
||||||
|
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
|
||||||
|
files = self.fixture['files']
|
||||||
|
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
|
||||||
|
receipts = {r['url']: r for r in manifest['records']}
|
||||||
|
self.assertEqual(len(self.fixture['page_gold']), 52)
|
||||||
|
for gold in self.fixture['page_gold']:
|
||||||
|
receipt = receipts[gold['program_url']]
|
||||||
|
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
|
||||||
|
fields = c.talk_fields(raw, receipt['session_url'])
|
||||||
|
self.assertEqual(fields['title'], gold['title'])
|
||||||
|
self.assertEqual(fields['source_reported_authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract'])
|
||||||
|
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
|
||||||
|
|
||||||
|
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
|
||||||
|
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
message = copy.deepcopy(self.message)
|
||||||
|
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
|
||||||
|
if kind == 'isbn': row['ISBN'] = ['9780000000000']
|
||||||
|
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
|
||||||
|
if kind == 'repeat': row['DOI'] = c.BOOK
|
||||||
|
if kind == 'type': row['type'] = 'other'
|
||||||
|
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
|
||||||
|
if kind == 'author': row['author'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
|
||||||
|
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
|
||||||
|
if kind == 'missing': message['items'].pop()
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(message)
|
||||||
|
|
||||||
|
def test_foreign_session_and_changed_edition_rejected(self):
|
||||||
|
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
|
||||||
|
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
|
||||||
|
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
|
||||||
|
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
|
||||||
|
with self.assertRaises(ValueError): c.talk_fields(changed, url)
|
||||||
|
|
||||||
|
def test_corruption_preserves_failure_without_success_report(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'receipt/body'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
|
||||||
|
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
Reference in New Issue
Block a user