Merge main and preserve independent collector command registration
Metadata integrity checks / metadata (push) Successful in 3m20s
Metadata integrity checks / metadata (push) Successful in 3m20s
This commit is contained in:
+30
-15
@@ -77,21 +77,6 @@ jobs:
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
recsys_snapshots = root/'recsys2026-snapshots'
|
||||
recsys_snapshots.mkdir()
|
||||
for name,encoded in recsys_fixture['files'].items():
|
||||
path = recsys_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
@@ -109,6 +94,21 @@ jobs:
|
||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
icdm_snapshots = root/'icdm2026-snapshots'
|
||||
for name, text in icdm_fixture['files'].items():
|
||||
target = icdm_snapshots/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(text.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
@@ -135,6 +135,21 @@ jobs:
|
||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
recsys_snapshots = root/'recsys2026-snapshots'
|
||||
recsys_snapshots.mkdir()
|
||||
for name,encoded in recsys_fixture['files'].items():
|
||||
path = recsys_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
corl_official = root/'corl2026-official'
|
||||
corl_archive = root/'corl2026-archive'
|
||||
|
||||
@@ -131,6 +131,8 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
||||
|
||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# 会议覆盖与公开决定核查
|
||||
|
||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||
|
||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||
|
||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# 2020 起补漏状态:2026-11-17
|
||||
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页补充新的生产结果,不回写 [2026-11-10 恢复清单](backfill-handoff-2026-11-10.md) 或更早快照。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
|
||||
## 最新数据库盘点
|
||||
|
||||
只读快照 `2026-11-17 09:38:31.677743+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 270 个至少有一条投稿记录,8 个仍为空,均为 2026 年:**CIKM、ECCV、ICASSP、ICDM、IROS、ISMIR、MM、RecSys**。本次 SDM2026 新增记录使空槽从 9 减到 8。
|
||||
|
||||
这只说明有无记录,**不表示 270 个槽位已经逐篇、逐 track、摘要或公开拒稿覆盖完整**;不能据此宣布第 14 步完成。2020–2025 没有整年空槽,仍有逐来源完整性和公开负面决定缺口。SDM2026 仅导入 25/53 条,状态为 partial,同样不能称为整届补齐。
|
||||
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/active-scope-census-20261117-v17/summary.json | bfb1db8b50619c3756abb91ad0f73fa58ff998c7db58a216661342f12047749f |
|
||||
| collected/active-scope-census-20261117-v17/snapshot.json | b5389205f8b34f7ed57946416f49c253e0aed4a05e45eb818801f18fca6a8cb4 |
|
||||
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||
|
||||
## 已推进的两批来源
|
||||
|
||||
- **SDM2026**:53 个正式注册章节、53 个独立日程观察和 52 个公开摘要页已冻结并规范化。25 条发表记录已在 PostgreSQL 独立验收,新增论文实体为 0;28 条待身份/版本核查。完整旧历史和 5,653 篇受保护论文的资料、来源、标签和既有标识保留。见 [生产验收与恢复](../sdm2026-identity-review.md)。这批队列已经消费,禁止重放。
|
||||
- **ECCV2026**:2,834 个唯一初步日程条目及公开摘要已全量采集并独立验收;新的只读身份预检和 1,997 个候选历史证据包已保存,尚未入库。初步日程不等于已核验出版社最终论文集;负面决定元数据仍受匿名访问限制。见 [来源状态](../eccv2026-source-status.md)。
|
||||
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时具有较早 ICLR 拒稿与后来的 SDM 发表事实。旧的两次 ICLR 投稿及四条决定来源观察仍保留,没有被较晚 accepted 覆盖;多个来源观察不直接累加成多次拒稿。
|
||||
|
||||
## 其余七个会议的最新入口核查
|
||||
|
||||
已对下表七个会议的 14 个明确官方 URL 保存原始响应和回执:13 个 HTTP 200,IROS 官方论文索引 HTTP 403。所有请求均匿名,不绕过访问限制,没有下载论文 PDF;保存入口不是完成逐篇采集或负面决定普查。
|
||||
|
||||
| 会议 | 已保存的明确官方入口 | 尚未完成 |
|
||||
| --- | --- | --- |
|
||||
| ICDM | [详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)、[研究轨道政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)、[应用轨道政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm) | 官方页面称为线下展示列表;需要核对逐篇范围、研究/应用轨道及演讲长短标签,不能把展示长短自动等同于论文类型;摘要和正式出版身份仍待核查 |
|
||||
| ICASSP | [录用标题清单](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[作者指南](https://2026.ieeeicassp.org/author-guidelines/) | 取得逐篇完整作者/摘要/出版记录,区别会议投稿与期刊展示,不能只导入标题或从缺席推断拒稿 |
|
||||
| IROS | [官方论文索引](https://2026.ieee-iros.org/program/paper_index/) | 当前服务器返回 403,保留缺口并停止该入口;第三方浏览器页面不能替代独立核验的官方原文 |
|
||||
| ISMIR | [官网](https://ismir2026.ismir.net/)、[官方虚拟平台](https://ismir2026program.ismir.net/)、[作者指南](https://ismir2026.ismir.net/authors/author-guidelines) | 核查平台公开静态数据与正式开放论文集的逐篇映射;首页可读取不代表论文目录已经取得 |
|
||||
| CIKM | [录用页](https://cikm2026.diag.uniroma1.it/accepted-papers/)、[日程页](https://cikm2026.diag.uniroma1.it/program/) | 网页提取的录用页仍显示 TBD;需要检查原文引用的节目数据与独立正式出版目录,不能当成没有论文 |
|
||||
| RecSys | [官网](https://recsys.acm.org/recsys26/)、[投稿说明](https://recsys.acm.org/recsys26/call/) | 继续取得可核验的逐篇正式记录,并分开主轨、工业、复现、demo、challenge/workshop |
|
||||
| MM | [官网](https://2026.acmmm.org/) | 继续核对官方完整日程和出版目录,分开研究论文与 grand challenge、workshop 等;首页响应不证明覆盖完整 |
|
||||
|
||||
冻结目录:`/data/paper-index/collected/remaining2026-official-leads-v1`。manifest SHA-256:`0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。恢复包:`/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256:`0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。HTTP 成功仅是入口可读性,不能作为可导入记录数量。
|
||||
|
||||
## 第 14 步仍需做什么
|
||||
|
||||
1. 取得并规范化其余可公开取得的完整逐篇来源,记录应有范围、排除项和版本;对缺失、受限或尚未公开的入口保存明确证据,不能伪造“已补齐”。优先推进当前可读的 ICDM 详细日程、ICASSP 清单及其他会议的正式目录。
|
||||
2. 完成来源与年度、track、摘要完整性的对应核查;已经有若干记录的年份也要看目录差集,不能只消除空槽。
|
||||
3. 继续近期和历史年度实际公开的 rejected、desk_rejected、withdrawn 来源核查,特别是 UAI2024–2026 的 opt-in 拒稿、EMNLP2023 的实际公开范围以及其他会议的逐年限定政策。公开评审和公开决定不同;没有找到不等于零条或未曾拒稿。
|
||||
4. 通过身份核查的记录再执行新快照守护、恢复备份和增量导入,保留较早决定。深层作品版本、跨会历史及类型/标签协调继续列为第 15 步;不能为减少空槽创建重复论文,也不能只盯住少量身份疑难项而停止其他来源采集。
|
||||
|
||||
第 16 步仍是所有导入结束后的独立全库去重与完整审计,第 17 步是最终长期运行/恢复和交付整理。上述任务均未因单批验收或 CI 成功而完成;总体 goal 继续保持 active。
|
||||
@@ -37,3 +37,23 @@ group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/R
|
||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||
|
||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||
|
||||
|
||||
## 2026-11-17 全量采集与只读预检
|
||||
|
||||
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||
|
||||
| 服务器证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||
|
||||
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||
|
||||
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||
|
||||
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
# ICDM2026 冻结日程适配器
|
||||
|
||||
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||
|
||||
```sh
|
||||
paper-atlas collect icdm2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||
```
|
||||
|
||||
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||
|
||||
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||
|
||||
## 两个明确来源轨道
|
||||
|
||||
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||
|
||||
| 输出文件 | 官方明确标签 | 记录数 |
|
||||
| --- | --- | ---: |
|
||||
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||
|
||||
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||
|
||||
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||
|
||||
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||
|
||||
## 始终保留范围限制
|
||||
|
||||
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||
|
||||
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||
|
||||
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||
|
||||
## 验证与恢复
|
||||
|
||||
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||
|
||||
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||
|
||||
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||
|
||||
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||
@@ -1,6 +1,6 @@
|
||||
# SDM 2026:生产规范化与身份核查记录
|
||||
|
||||
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
||||
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||
|
||||
## 已核实的生产输出
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||
|
||||
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
|
||||
## 未精确匹配的补充检索
|
||||
|
||||
@@ -70,9 +70,28 @@
|
||||
|
||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||
|
||||
## 25 条生产导入验收
|
||||
|
||||
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||
|
||||
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||
|
||||
| 生产证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||
|
||||
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||
|
||||
## 下次恢复
|
||||
|
||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||
|
||||
+2
-1
@@ -25,10 +25,11 @@ COMMANDS = {
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'icdm_program'
|
||||
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||
PAGES = {
|
||||
'detailed-sessions': PROGRAM,
|
||||
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, name):
|
||||
return name in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def field(node, name):
|
||||
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||
raise ValueError('Unsafe source manifest')
|
||||
raw = manifest.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Frozen manifest hash changed')
|
||||
data = json.loads(raw)
|
||||
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||
raise ValueError('Wrong source edition or incomplete research run')
|
||||
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||
raise ValueError('Incomplete ICDM source set')
|
||||
files = {'manifest.json': raw}; indexed = {}
|
||||
for receipt in receipts:
|
||||
filename = receipt['file']; path = folder/filename
|
||||
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body)
|
||||
or not receipt['content_type'].lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Source receipt/body mismatch')
|
||||
if filename in files:
|
||||
raise ValueError('Repeated source filename')
|
||||
files[filename] = body; indexed[receipt['label']] = receipt
|
||||
return indexed, files
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8'))
|
||||
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||
text = clean(main.text())
|
||||
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||
raise ValueError('Wrong edition or unreviewed program scope')
|
||||
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||
raise ValueError('Incomplete or repeated session census')
|
||||
records = []; seen = set()
|
||||
for session in sessions:
|
||||
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||
form = session.attrs.get('data-format')
|
||||
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||
or form not in ('long', 'short')):
|
||||
raise ValueError('Invalid session ID, track or presentation label')
|
||||
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||
if field(session, 'sd-track') != label:
|
||||
raise ValueError('Visible track and source attribute disagree')
|
||||
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||
if count is None or count[1].lower() != form:
|
||||
raise ValueError('Visible presentation label disagrees')
|
||||
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||
if len(rows) != int(count[2]):
|
||||
raise ValueError('Session paper count changed')
|
||||
note = field(session, 'sd-table-note')
|
||||
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||
raise ValueError('Unreviewed presentation-note semantics')
|
||||
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||
'title': field(session, 'sd-session-title'),
|
||||
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||
for row in rows:
|
||||
pid = row.attrs['data-paper-id']
|
||||
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||
or field(row, 'sd-paper-id') != pid):
|
||||
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||
seen.add(pid)
|
||||
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||
authors = [name.strip() for name in authors_literal.split(';')]
|
||||
if not title or any(not name for name in authors):
|
||||
raise ValueError('Missing program title or author')
|
||||
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||
'session': context.copy(), 'source_position': len(records)+1})
|
||||
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||
return records
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
receipts, files = load_snapshot(snapshots)
|
||||
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in files.items():
|
||||
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||
outputs = []
|
||||
for track in ('research', 'applied'):
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['session']['track_literal'] != track:
|
||||
continue
|
||||
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||
'evidence_url': url,
|
||||
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||
'paper_type': None,
|
||||
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||
temporary.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True, type=Path)
|
||||
parser.add_argument('--out', required=True, type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,113 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icdm2026 as c
|
||||
|
||||
|
||||
class ICDM2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||
cls.gold = cls.fixture['independent_inventory']
|
||||
|
||||
def snapshots(self, root):
|
||||
for name, text in self.fixture['files'].items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||
|
||||
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||
result = c.parse_program(self.raw)
|
||||
self.assertEqual(len(result), len(self.gold['entries']))
|
||||
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||
self.assertEqual(row[field], gold[field])
|
||||
self.assertEqual(row['source_position'], position)
|
||||
for field, value in gold['session'].items():
|
||||
self.assertEqual(row['session'][field], value)
|
||||
|
||||
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
report = c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||
all_rows = []
|
||||
for manifest in report['files']:
|
||||
path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||
self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(line) for line in raw.splitlines()]
|
||||
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||
for row in rows:
|
||||
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||
all_rows.extend(rows)
|
||||
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||
for item in report['evidence']:
|
||||
raw = (root/'out'/item['output_file']).read_bytes()
|
||||
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
|
||||
def test_stale_or_different_conference_rejected(self):
|
||||
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_track_attribute_must_agree_with_visible_label(self):
|
||||
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'track'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new, 1))
|
||||
|
||||
def test_empty_author_is_not_silently_dropped(self):
|
||||
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'author'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertFalse((root/'out').exists())
|
||||
|
||||
def test_snapshot_source_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
Reference in New Issue
Block a user