Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| 8c147f5e6e |
@@ -46,6 +46,26 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
mm_snapshots = root/'mm2026-snapshots'
|
||||||
|
for name, body in mm_fixture['files'].items():
|
||||||
|
path = mm_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||||
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||||
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||||
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||||
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||||
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
import hashlib
|
||||||
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||||
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||||
|
row = mm_rows[gold['paper_id']]
|
||||||
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||||
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||||
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||||
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
@@ -166,6 +186,21 @@ jobs:
|
|||||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
ismir_snapshots = root/'ismir2026-snapshots'
|
||||||
|
ismir_snapshots.mkdir()
|
||||||
|
for name,encoded in ismir_fixture['files'].items():
|
||||||
|
target = ismir_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||||
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||||
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||||
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||||
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||||
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||||
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
|
|||||||
@@ -131,6 +131,8 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
|||||||
|
|
||||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||||
|
|
||||||
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||||
|
|
||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
@@ -138,3 +140,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
|||||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# ISMIR 2026 官方节目与摘要采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect ismir2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录需要三个冻结目录:
|
||||||
|
|
||||||
|
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||||
|
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||||
|
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||||
|
|
||||||
|
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||||
|
|
||||||
|
## 来源与版本
|
||||||
|
|
||||||
|
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||||
|
|
||||||
|
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||||
|
|
||||||
|
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||||
|
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||||
|
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||||
|
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||||
|
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||||
|
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||||
|
|
||||||
|
## 公开评审与拒稿边界
|
||||||
|
|
||||||
|
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||||
|
|
||||||
|
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||||
|
|
||||||
|
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||||
|
|
||||||
|
## 恢复与验证
|
||||||
|
|
||||||
|
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||||
|
|
||||||
|
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||||
|
|
||||||
|
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||||
|
|
||||||
|
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||||
@@ -73,4 +73,20 @@
|
|||||||
|
|
||||||
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||||
|
|
||||||
本批尚未形成生产导入队列。下一步需要可复用采集器保留每个节目 ID 的全部展示上下文、明确贡献类型与来源边界,单列缺失字段,再做作品/出版身份及既往投稿审查。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
## 可复用采集命令
|
||||||
|
|
||||||
|
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/mm2026-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||||
|
|
||||||
|
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||||
|
|
||||||
|
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||||
|
|
||||||
|
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||||
@@ -31,6 +32,7 @@ COMMANDS = {
|
|||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'ismir_program'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||||
|
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||||
|
'research_completed_not_coverage_or_import_approval', {
|
||||||
|
'ismir-accepted.html': ACCEPTED,
|
||||||
|
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||||
|
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||||
|
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||||
|
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||||
|
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||||
|
}
|
||||||
|
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def title_key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshots(root):
|
||||||
|
root = Path(root)
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ValueError('Unsafe snapshot root')
|
||||||
|
result = {}
|
||||||
|
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
files = {'manifest.json': raw}; receipts = {}
|
||||||
|
for filename, url in urls.items():
|
||||||
|
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||||
|
path = directory/filename
|
||||||
|
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body disagree')
|
||||||
|
files[filename] = body; receipts[filename] = receipt
|
||||||
|
result[name] = (files, receipts)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(literal):
|
||||||
|
# Institution names can contain semicolons; split only outside parentheses.
|
||||||
|
parts = []; depth = 0; start = 0
|
||||||
|
for index, char in enumerate(literal):
|
||||||
|
if char == '(': depth += 1
|
||||||
|
elif char == ')':
|
||||||
|
depth -= 1
|
||||||
|
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||||
|
elif char == ';' and depth == 0:
|
||||||
|
parts.append(literal[start:index]); start = index+1
|
||||||
|
if depth:
|
||||||
|
raise ValueError('Unbalanced author affiliation')
|
||||||
|
parts.append(literal[start:])
|
||||||
|
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||||
|
if any(not name for name in names):
|
||||||
|
raise ValueError('Missing accepted-list author')
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_listing(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); tables = []
|
||||||
|
for table in doc.root.walk():
|
||||||
|
if table.tag != 'table': continue
|
||||||
|
rows = []
|
||||||
|
for row in table.walk():
|
||||||
|
if row.tag == 'tr':
|
||||||
|
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||||
|
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||||
|
tables.append(rows)
|
||||||
|
rows = one(tables)[1:]; result = {}
|
||||||
|
if len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete official accepted-list census')
|
||||||
|
for row in rows:
|
||||||
|
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||||
|
raise ValueError('Missing or ambiguous accepted-list title')
|
||||||
|
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(rows, listing):
|
||||||
|
if not isinstance(rows, list) or len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete public program census')
|
||||||
|
observations = []; seen = set(); used = set()
|
||||||
|
for original in rows:
|
||||||
|
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||||
|
raise ValueError('Invalid public program record')
|
||||||
|
pid = original.get('id'); content = original.get('content', {})
|
||||||
|
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||||
|
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||||
|
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||||
|
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||||
|
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||||
|
raise ValueError('Missing program title, author sequence or abstract')
|
||||||
|
key = title_key(title)
|
||||||
|
if key not in listing or key in used:
|
||||||
|
raise ValueError('Program/accepted-list title membership mismatch')
|
||||||
|
used.add(key); accepted = listing[key]
|
||||||
|
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||||
|
session = content.get('session'); reviews = original.get('reviews')
|
||||||
|
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||||
|
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||||
|
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||||
|
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||||
|
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||||
|
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||||
|
or any(not isinstance(v, str) for v in reviews.values())
|
||||||
|
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||||
|
raise ValueError('Unreviewed opt-in review structure')
|
||||||
|
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||||
|
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||||
|
raise ValueError('Review publication preference/body disagree')
|
||||||
|
flags = []
|
||||||
|
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||||
|
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||||
|
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||||
|
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||||
|
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||||
|
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||||
|
raise ValueError('Unreviewed source-version or opt-in census')
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
sources = load_snapshots(snapshots)
|
||||||
|
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||||
|
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||||
|
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||||
|
raise ValueError('Unreviewed public program data path or poster link template')
|
||||||
|
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, (files, _) in sources.items():
|
||||||
|
directory = evidence/name; directory.mkdir()
|
||||||
|
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for observation in observations:
|
||||||
|
original = observation['original']; content = original['content']; pid = original['id']
|
||||||
|
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||||
|
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||||
|
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||||
|
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||||
|
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||||
|
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||||
|
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||||
|
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||||
|
'original_decision_signature_verified':False,'paper_type':None,
|
||||||
|
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||||
|
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||||
|
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||||
|
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||||
|
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||||
|
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||||
|
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||||
|
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||||
|
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||||
|
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||||
|
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||||
|
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
SOURCE = 'mm_program'
|
||||||
|
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||||
|
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||||
|
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||||
|
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||||
|
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||||
|
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||||
|
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||||
|
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||||
|
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||||
|
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||||
|
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||||
|
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def safe_read(path, limit):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||||
|
raise ValueError('Unsafe snapshot path or size')
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||||
|
raise ValueError('Unreviewed manifest scope')
|
||||||
|
evidence = {'manifest.json':raw}
|
||||||
|
for receipt in manifest['entries']:
|
||||||
|
name = receipt['file']
|
||||||
|
if Path(name).name != name or name in evidence:
|
||||||
|
raise ValueError('Unsafe or duplicate receipt name')
|
||||||
|
body = safe_read(directory/name, 12*1024**2)
|
||||||
|
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['content_type'].startswith('application/json')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if name == 'program-data.json':
|
||||||
|
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||||
|
raise ValueError('Wrong program source')
|
||||||
|
elif receipt['http_status'] != 403:
|
||||||
|
raise ValueError('Unreviewed public-access result')
|
||||||
|
evidence[name] = body
|
||||||
|
if 'program-data.json' not in evidence:
|
||||||
|
raise ValueError('Missing program source')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def without_emails(value):
|
||||||
|
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [without_emails(v) for v in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def program_locations(node, pointer=''):
|
||||||
|
"""Independent structural guard against silently skipping a new nesting form."""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
if 'paper_id' in node:
|
||||||
|
yield pointer
|
||||||
|
for key,value in node.items():
|
||||||
|
escaped = key.replace('~','~0').replace('/','~1')
|
||||||
|
yield from program_locations(value, pointer+'/'+escaped)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for i,value in enumerate(node):
|
||||||
|
yield from program_locations(value, pointer+'/'+str(i))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
data = json.loads(raw)
|
||||||
|
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||||
|
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||||
|
or len(data.get('entries',[])) != 157):
|
||||||
|
raise ValueError('Unreviewed program edition or census')
|
||||||
|
records = {}; sessions = []; seen = []
|
||||||
|
|
||||||
|
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||||
|
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||||
|
raise ValueError('Unreviewed program fields')
|
||||||
|
for key in ('paper_id','title','abstract','track'):
|
||||||
|
if not isinstance(paper[key],str):
|
||||||
|
raise ValueError('Wrong program field type')
|
||||||
|
for key in ('authors','affiliations'):
|
||||||
|
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||||
|
raise ValueError('Wrong author or affiliation list')
|
||||||
|
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||||
|
raise ValueError('Missing identity/title or unreviewed source track')
|
||||||
|
core = {k:paper[k] for k in CORE}
|
||||||
|
pid = paper['paper_id']; seen.append(pointer)
|
||||||
|
if pid not in records:
|
||||||
|
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||||
|
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||||
|
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||||
|
'observed_status':'listed_in_official_program',
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'approved_for_import':False,'missing_fields':missing,
|
||||||
|
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||||
|
'contexts':[]}
|
||||||
|
elif {k:records[pid][k] for k in CORE} != core:
|
||||||
|
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||||
|
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||||
|
'subsession_pointer':subsession_pointer,
|
||||||
|
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||||
|
|
||||||
|
for i,entry in enumerate(data['entries']):
|
||||||
|
pointer = '/entries/'+str(i)
|
||||||
|
if entry['track'] != '':
|
||||||
|
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||||
|
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||||
|
session.update(source_pointer=pointer,subsessions=[])
|
||||||
|
for j,paper in enumerate(entry.get('papers') or []):
|
||||||
|
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||||
|
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||||
|
sp = pointer+'/subsessions/'+str(j)
|
||||||
|
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||||
|
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||||
|
for k,item in enumerate(sub.get('items') or []):
|
||||||
|
ip = sp+'/items/'+str(k)
|
||||||
|
if 'paper_id' in item:
|
||||||
|
observe(item,ip,pointer,sp)
|
||||||
|
else:
|
||||||
|
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||||
|
session['subsessions'].append(saved)
|
||||||
|
sessions.append(session)
|
||||||
|
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||||
|
raise ValueError('Unhandled or duplicated program location')
|
||||||
|
items = list(records.values())
|
||||||
|
if (len(seen) != 4326 or len(items) != 2143
|
||||||
|
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||||
|
or sum(not r['abstract'] for r in items) != 216
|
||||||
|
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||||
|
raise ValueError('Program count or missing-field census changed')
|
||||||
|
return items,sessions
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; use a fresh directory')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = load_snapshot(snapshots)
|
||||||
|
rows,sessions = parse_program(evidence['program-data.json'])
|
||||||
|
(out/'evidence').mkdir()
|
||||||
|
for name,body in evidence.items():
|
||||||
|
(out/'evidence'/name).write_bytes(body)
|
||||||
|
outputs = []
|
||||||
|
for track,count in TRACK_COUNTS.items():
|
||||||
|
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||||
|
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||||
|
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||||
|
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||||
|
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||||
|
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||||
|
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||||
|
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||||
|
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||||
|
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||||
|
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||||
|
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||||
|
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||||
|
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||||
|
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||||
|
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||||
|
part = out/'collection-report.json.part'
|
||||||
|
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,92 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import ismir2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ISMIR2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||||
|
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||||
|
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||||
|
|
||||||
|
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||||
|
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||||
|
self.assertEqual(len(observations),140)
|
||||||
|
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||||
|
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||||
|
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||||
|
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||||
|
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||||
|
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||||
|
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||||
|
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||||
|
|
||||||
|
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||||
|
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||||
|
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||||
|
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||||
|
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||||
|
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||||
|
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||||
|
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||||
|
rows=copy.deepcopy(self.program)
|
||||||
|
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||||
|
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||||
|
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||||
|
if kind=='missing_row':rows.pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||||
|
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||||
|
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_source_optin_no_cannot_publish_review_text(self):
|
||||||
|
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||||
|
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||||
|
|
||||||
|
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||||
|
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||||
|
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||||
|
with self.assertRaises(ValueError):c.author_names(raw)
|
||||||
|
|
||||||
|
def test_ambiguous_official_title_is_not_fused(self):
|
||||||
|
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||||
|
|
||||||
|
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||||
|
for kind in ('manifest','body','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MM2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||||
|
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||||
|
cls.data = json.loads(cls.raw)
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name,body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def parse_changed(self, data):
|
||||||
|
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||||
|
|
||||||
|
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||||
|
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||||
|
actual = {r['paper_id']:r for r in rows}
|
||||||
|
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||||
|
for gold in self.gold['inventory']:
|
||||||
|
r = actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||||
|
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||||
|
|
||||||
|
def test_every_presentation_and_session_context_is_preserved(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
for r in rows:
|
||||||
|
for context in r['contexts']:
|
||||||
|
node = self.data
|
||||||
|
for key in context['source_pointer'].split('/')[1:]:
|
||||||
|
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||||
|
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||||
|
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||||
|
for i,session in enumerate(sessions):
|
||||||
|
original = self.data['entries'][i]
|
||||||
|
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||||
|
self.assertEqual(session['track'],'')
|
||||||
|
for key in original.keys()-{'papers','subsessions'}:
|
||||||
|
self.assertEqual(session[key],original[key])
|
||||||
|
for j,sub in enumerate(original.get('subsessions') or []):
|
||||||
|
saved = session['subsessions'][j]
|
||||||
|
self.assertEqual(saved['title'],sub['title'])
|
||||||
|
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||||
|
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||||
|
|
||||||
|
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||||
|
rows,_ = c.parse_program(self.raw)
|
||||||
|
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||||
|
for r in rows:
|
||||||
|
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['publication_identity_verified'])
|
||||||
|
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||||
|
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||||
|
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||||
|
missing = [r for r in rows if not r['authors']]
|
||||||
|
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||||
|
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||||
|
|
||||||
|
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src')
|
||||||
|
report=c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||||
|
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||||
|
for name,body in self.files.items():
|
||||||
|
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||||
|
n=0
|
||||||
|
for item in report['files']:
|
||||||
|
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||||
|
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||||
|
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||||
|
self.assertEqual(n,2143)
|
||||||
|
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
first=data['entries'][0]['papers'][0]
|
||||||
|
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||||
|
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
moved=data['entries'][0]['papers'].pop()
|
||||||
|
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||||
|
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||||
|
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||||
|
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||||
|
elif kind=='paper_track':paper['track']='New Track'
|
||||||
|
elif kind=='field':paper['doi']='unreviewed'
|
||||||
|
elif kind=='title':paper['title']=''
|
||||||
|
else:paper['authors']='Unparsed author string'
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_edition_or_census_change_requires_review(self):
|
||||||
|
for kind in ('edition','count'):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||||
|
else:data['entries'][0]['papers'].pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||||
|
for kind in ('manifest','program','probe','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||||
|
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||||
|
path=directory/name
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':
|
||||||
|
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||||
|
rows,sessions=c.parse_program(self.raw)
|
||||||
|
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||||
|
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||||
|
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||||
Reference in New Issue
Block a user