Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 33bb378369 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 2f1b5c959a |
@@ -12,7 +12,7 @@ jobs:
|
|||||||
- name: Verify metadata storage and query commands
|
- name: Verify metadata storage and query commands
|
||||||
run: |
|
run: |
|
||||||
python3 -B - <<'PY'
|
python3 -B - <<'PY'
|
||||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
sha = os.environ['ARCHIVE_SHA']
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||||
raise SystemExit('Invalid source commit')
|
raise SystemExit('Invalid source commit')
|
||||||
@@ -94,6 +94,21 @@ jobs:
|
|||||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||||
|
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
icdm_snapshots = root/'icdm2026-snapshots'
|
||||||
|
for name, text in icdm_fixture['files'].items():
|
||||||
|
target = icdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||||
|
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||||
|
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||||
|
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||||
|
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||||
|
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||||
|
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||||
|
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||||
|
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
icaps_snapshots = root/'icaps2026-snapshots'
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
icaps_snapshots.mkdir()
|
icaps_snapshots.mkdir()
|
||||||
@@ -120,6 +135,21 @@ jobs:
|
|||||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
recsys_snapshots = root/'recsys2026-snapshots'
|
||||||
|
recsys_snapshots.mkdir()
|
||||||
|
for name,encoded in recsys_fixture['files'].items():
|
||||||
|
path = recsys_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||||
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||||
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||||
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||||
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||||
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
|
|||||||
@@ -132,3 +132,7 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
|||||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# ICDM2026 冻结日程适配器
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icdm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||||
|
|
||||||
|
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||||
|
|
||||||
|
## 两个明确来源轨道
|
||||||
|
|
||||||
|
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||||
|
|
||||||
|
| 输出文件 | 官方明确标签 | 记录数 |
|
||||||
|
| --- | --- | ---: |
|
||||||
|
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||||
|
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||||
|
|
||||||
|
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||||
|
|
||||||
|
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||||
|
|
||||||
|
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||||
|
|
||||||
|
## 始终保留范围限制
|
||||||
|
|
||||||
|
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||||
|
|
||||||
|
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||||
|
|
||||||
|
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||||
|
|
||||||
|
## 验证与恢复
|
||||||
|
|
||||||
|
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||||
|
|
||||||
|
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||||
|
|
||||||
|
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||||
|
|
||||||
|
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# RecSys 2026 公开录用贡献采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect recsys2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||||
|
|
||||||
|
## 数据范围与重复项
|
||||||
|
|
||||||
|
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||||
|
|
||||||
|
| track | 观察数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| long | 65 |
|
||||||
|
| short | 29 |
|
||||||
|
| past_present_future | 13 |
|
||||||
|
| reproducibility | 4 |
|
||||||
|
| resource | 7 |
|
||||||
|
| industry | 83 |
|
||||||
|
| research_practice_notes | 34 |
|
||||||
|
| demo | 18 |
|
||||||
|
| doctoral_symposium | 10 |
|
||||||
|
|
||||||
|
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||||
|
|
||||||
|
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||||
|
|
||||||
|
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||||
|
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||||
|
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||||
|
|
||||||
|
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||||
|
|
||||||
|
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||||
|
|
||||||
|
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||||
|
|
||||||
|
## 输出与恢复
|
||||||
|
|
||||||
|
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||||
|
|
||||||
|
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||||
|
|
||||||
|
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||||
|
|
||||||
|
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||||
@@ -25,9 +25,11 @@ COMMANDS = {
|
|||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
|||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icdm_program'
|
||||||
|
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||||
|
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||||
|
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||||
|
PAGES = {
|
||||||
|
'detailed-sessions': PROGRAM,
|
||||||
|
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||||
|
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, name):
|
||||||
|
return name in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def field(node, name):
|
||||||
|
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||||
|
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe source manifest')
|
||||||
|
raw = manifest.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Frozen manifest hash changed')
|
||||||
|
data = json.loads(raw)
|
||||||
|
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||||
|
raise ValueError('Wrong source edition or incomplete research run')
|
||||||
|
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||||
|
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||||
|
raise ValueError('Incomplete ICDM source set')
|
||||||
|
files = {'manifest.json': raw}; indexed = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
filename = receipt['file']; path = folder/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||||
|
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body)
|
||||||
|
or not receipt['content_type'].lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body mismatch')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated source filename')
|
||||||
|
files[filename] = body; indexed[receipt['label']] = receipt
|
||||||
|
return indexed, files
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8'))
|
||||||
|
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||||
|
text = clean(main.text())
|
||||||
|
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||||
|
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||||
|
raise ValueError('Wrong edition or unreviewed program scope')
|
||||||
|
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||||
|
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||||
|
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||||
|
raise ValueError('Incomplete or repeated session census')
|
||||||
|
records = []; seen = set()
|
||||||
|
for session in sessions:
|
||||||
|
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||||
|
form = session.attrs.get('data-format')
|
||||||
|
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||||
|
or form not in ('long', 'short')):
|
||||||
|
raise ValueError('Invalid session ID, track or presentation label')
|
||||||
|
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||||
|
if field(session, 'sd-track') != label:
|
||||||
|
raise ValueError('Visible track and source attribute disagree')
|
||||||
|
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||||
|
if count is None or count[1].lower() != form:
|
||||||
|
raise ValueError('Visible presentation label disagrees')
|
||||||
|
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||||
|
if len(rows) != int(count[2]):
|
||||||
|
raise ValueError('Session paper count changed')
|
||||||
|
note = field(session, 'sd-table-note')
|
||||||
|
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||||
|
raise ValueError('Unreviewed presentation-note semantics')
|
||||||
|
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||||
|
'title': field(session, 'sd-session-title'),
|
||||||
|
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||||
|
for row in rows:
|
||||||
|
pid = row.attrs['data-paper-id']
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||||
|
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||||
|
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||||
|
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||||
|
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||||
|
or field(row, 'sd-paper-id') != pid):
|
||||||
|
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||||
|
seen.add(pid)
|
||||||
|
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||||
|
authors = [name.strip() for name in authors_literal.split(';')]
|
||||||
|
if not title or any(not name for name in authors):
|
||||||
|
raise ValueError('Missing program title or author')
|
||||||
|
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||||
|
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||||
|
'session': context.copy(), 'source_position': len(records)+1})
|
||||||
|
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||||
|
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||||
|
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||||
|
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
receipts, files = load_snapshot(snapshots)
|
||||||
|
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
outputs = []
|
||||||
|
for track in ('research', 'applied'):
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['session']['track_literal'] != track:
|
||||||
|
continue
|
||||||
|
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||||
|
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||||
|
'evidence_url': url,
|
||||||
|
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||||
|
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||||
|
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||||
|
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||||
|
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||||
|
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||||
|
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||||
|
'paper_type': None,
|
||||||
|
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||||
|
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||||
|
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||||
|
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||||
|
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||||
|
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||||
|
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||||
|
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||||
|
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True, type=Path)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'recsys_program'
|
||||||
|
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||||
|
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||||
|
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||||
|
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||||
|
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||||
|
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||||
|
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||||
|
('doctoral_symposium', 'Doctoral', 10)]
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot body')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||||
|
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
return receipt, {'manifest.json': raw, name: body}, body
|
||||||
|
|
||||||
|
|
||||||
|
def body_text(node, doi_url):
|
||||||
|
if not isinstance(node, Node):
|
||||||
|
return node
|
||||||
|
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||||
|
return ''
|
||||||
|
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||||
|
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||||
|
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||||
|
raise ValueError('Wrong conference edition')
|
||||||
|
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||||
|
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||||
|
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||||
|
raise ValueError('Unreviewed track menu')
|
||||||
|
observations = []; seen = set()
|
||||||
|
for tab, (track, label, count) in enumerate(TRACKS):
|
||||||
|
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||||
|
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||||
|
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||||
|
if len(rows) != count:
|
||||||
|
raise ValueError('Incomplete reviewed track census')
|
||||||
|
for position, row in enumerate(rows, 1):
|
||||||
|
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||||
|
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||||
|
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||||
|
raise ValueError('Missing or repeated program locator')
|
||||||
|
seen.add(locator)
|
||||||
|
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||||
|
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||||
|
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||||
|
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||||
|
if (type_literal,abbreviation) != expected_type:
|
||||||
|
raise ValueError('Unreviewed source contribution type')
|
||||||
|
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||||
|
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||||
|
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||||
|
if not authors_literal.startswith('by '):
|
||||||
|
raise ValueError('Missing program author prefix')
|
||||||
|
authors_literal = authors_literal[3:]
|
||||||
|
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||||
|
if not title or any(not n or '@' in n for n in authors):
|
||||||
|
raise ValueError('Missing title or program author')
|
||||||
|
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||||
|
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||||
|
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||||
|
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||||
|
raise ValueError('Wrong publication DOI family')
|
||||||
|
abstract = clean(body_text(body, doi_url))
|
||||||
|
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||||
|
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||||
|
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||||
|
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||||
|
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||||
|
'track': track, 'track_label': label})
|
||||||
|
counts = Counter(r['doi'] for r in observations)
|
||||||
|
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||||
|
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||||
|
raise ValueError('Unreviewed DOI observation census')
|
||||||
|
for item in observations:
|
||||||
|
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
(evidence/name).write_bytes(raw)
|
||||||
|
outputs = []
|
||||||
|
for track,label,count in TRACKS:
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['track'] != track:
|
||||||
|
continue
|
||||||
|
doi = item['doi']; url = URL+'#'+item['locator']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||||
|
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||||
|
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||||
|
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||||
|
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||||
|
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified':False,
|
||||||
|
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||||
|
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||||
|
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||||
|
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||||
|
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||||
|
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||||
|
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||||
|
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||||
|
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||||
|
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,113 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||||
|
|
||||||
|
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||||
|
result = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(result), len(self.gold['entries']))
|
||||||
|
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||||
|
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||||
|
self.assertEqual(row[field], gold[field])
|
||||||
|
self.assertEqual(row['source_position'], position)
|
||||||
|
for field, value in gold['session'].items():
|
||||||
|
self.assertEqual(row['session'][field], value)
|
||||||
|
|
||||||
|
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||||
|
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||||
|
all_rows = []
|
||||||
|
for manifest in report['files']:
|
||||||
|
path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||||
|
self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in raw.splitlines()]
|
||||||
|
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||||
|
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||||
|
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||||
|
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||||
|
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||||
|
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||||
|
all_rows.extend(rows)
|
||||||
|
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||||
|
for item in report['evidence']:
|
||||||
|
raw = (root/'out'/item['output_file']).read_bytes()
|
||||||
|
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||||
|
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||||
|
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_stale_or_different_conference_rejected(self):
|
||||||
|
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||||
|
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_track_attribute_must_agree_with_visible_label(self):
|
||||||
|
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'track'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||||
|
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||||
|
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||||
|
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||||
|
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_empty_author_is_not_silently_dropped(self):
|
||||||
|
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'author'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||||
|
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||||
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertFalse((root/'out').exists())
|
||||||
|
|
||||||
|
def test_snapshot_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||||
|
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import base64
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import recsys2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class RecSys2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||||
|
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||||
|
|
||||||
|
def test_all263_observations_against_independent_gold(self):
|
||||||
|
items=c.parse_program(self.body)
|
||||||
|
keys=self.fixture['independent_gold'][0].keys()
|
||||||
|
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||||
|
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||||
|
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||||
|
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||||
|
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||||
|
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||||
|
|
||||||
|
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||||
|
report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||||
|
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||||
|
for manifest in report['files']:
|
||||||
|
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
for r in rows:
|
||||||
|
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||||
|
for key in expected:self.assertEqual(item[key],expected[key])
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||||
|
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_edition_and_track_label_drift_rejected(self):
|
||||||
|
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||||
|
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||||
|
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_and_root_rejected(self):
|
||||||
|
for kind in ('source','root'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||||
|
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||||
Reference in New Issue
Block a user