Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| afb3b939ca | |||
| 8c147f5e6e | |||
| 6961710e96 | |||
| f4be9de547 | |||
| 33bb378369 | |||
| 5bdf9f0fe5 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 564821d883 | |||
| f139c7e49d |
@@ -12,7 +12,7 @@ jobs:
|
|||||||
- name: Verify metadata storage and query commands
|
- name: Verify metadata storage and query commands
|
||||||
run: |
|
run: |
|
||||||
python3 -B - <<'PY'
|
python3 -B - <<'PY'
|
||||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
sha = os.environ['ARCHIVE_SHA']
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||||
raise SystemExit('Invalid source commit')
|
raise SystemExit('Invalid source commit')
|
||||||
@@ -46,6 +46,33 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
mm_snapshots = root/'mm2026-snapshots'
|
||||||
|
for name, body in mm_fixture['files'].items():
|
||||||
|
path = mm_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||||
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||||
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||||
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||||
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||||
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
import hashlib
|
||||||
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||||
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||||
|
row = mm_rows[gold['paper_id']]
|
||||||
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||||
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||||
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||||
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||||
|
if service_file.is_file():
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||||
|
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||||
|
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||||
|
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
@@ -77,6 +104,22 @@ jobs:
|
|||||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||||
assert '&' in eccv_literal_row['title']
|
assert '&' in eccv_literal_row['title']
|
||||||
|
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
icassp_snapshots = root/'icassp2026-snapshots'
|
||||||
|
icassp_snapshots.mkdir()
|
||||||
|
for name, encoded in icassp_fixture['files'].items():
|
||||||
|
target = icassp_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||||
|
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||||
|
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||||
|
assert icassp_report['single_name_author_fields'] == 23
|
||||||
|
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||||
|
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||||
|
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
sdm_snapshots = root/'sdm2026-snapshots'
|
sdm_snapshots = root/'sdm2026-snapshots'
|
||||||
for name, text in sdm_fixture['files'].items():
|
for name, text in sdm_fixture['files'].items():
|
||||||
@@ -135,6 +178,36 @@ jobs:
|
|||||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
recsys_snapshots = root/'recsys2026-snapshots'
|
||||||
|
recsys_snapshots.mkdir()
|
||||||
|
for name,encoded in recsys_fixture['files'].items():
|
||||||
|
path = recsys_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||||
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||||
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||||
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||||
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||||
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
ismir_snapshots = root/'ismir2026-snapshots'
|
||||||
|
ismir_snapshots.mkdir()
|
||||||
|
for name,encoded in ismir_fixture['files'].items():
|
||||||
|
target = ismir_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||||
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||||
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||||
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||||
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||||
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||||
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
|
|||||||
@@ -131,6 +131,14 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
|||||||
|
|
||||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||||
|
|
||||||
|
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||||
|
|
||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# 会议覆盖与公开决定核查
|
# 会议覆盖与公开决定核查
|
||||||
|
|
||||||
|
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||||
|
|
||||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||||
|
|
||||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||||
@@ -187,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
|||||||
|
|
||||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||||
|
|
||||||
|
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||||
|
|
||||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||||
|
|
||||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||||
@@ -196,3 +200,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
|||||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||||
|
|
||||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||||
|
|
||||||
|
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||||
|
|||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# 2020 起补漏状态:2026-11-17
|
||||||
|
|
||||||
|
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||||
|
|
||||||
|
## 最新数据库盘点
|
||||||
|
|
||||||
|
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||||
|
|
||||||
|
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||||
|
|
||||||
|
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||||
|
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||||
|
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||||
|
|
||||||
|
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||||
|
|
||||||
|
## 已独立接受的近期补录
|
||||||
|
|
||||||
|
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||||
|
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||||
|
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||||
|
|
||||||
|
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||||
|
|
||||||
|
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||||
|
|
||||||
|
## 已采集、尚待关联或规范化的来源
|
||||||
|
|
||||||
|
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||||
|
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||||
|
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||||
|
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
||||||
|
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||||
|
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||||
|
|
||||||
|
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||||
|
|
||||||
|
## 公开拒稿与后续工作
|
||||||
|
|
||||||
|
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||||
|
|
||||||
|
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||||
|
|
||||||
|
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||||
@@ -37,3 +37,23 @@ group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/R
|
|||||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-11-17 全量采集与只读预检
|
||||||
|
|
||||||
|
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||||
|
|
||||||
|
| 服务器证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||||
|
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||||
|
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||||
|
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||||
|
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||||
|
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||||
|
|
||||||
|
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||||
|
|
||||||
|
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||||
|
|
||||||
|
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICASSP 2026 注册元数据采集
|
||||||
|
|
||||||
|
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||||
|
|
||||||
|
## 输入与使用
|
||||||
|
|
||||||
|
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||||
|
|
||||||
|
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icassp2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||||
|
|
||||||
|
## 数据范围
|
||||||
|
|
||||||
|
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||||
|
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||||
|
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||||
|
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||||
|
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||||
|
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||||
|
|
||||||
|
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||||
|
|
||||||
|
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||||
|
|
||||||
|
## 证据与恢复
|
||||||
|
|
||||||
|
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||||
|
|
||||||
|
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||||
|
|
||||||
|
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||||
|
|
||||||
|
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# ISMIR 2026 官方节目与摘要采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect ismir2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录需要三个冻结目录:
|
||||||
|
|
||||||
|
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||||
|
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||||
|
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||||
|
|
||||||
|
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||||
|
|
||||||
|
## 来源与版本
|
||||||
|
|
||||||
|
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||||
|
|
||||||
|
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||||
|
|
||||||
|
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||||
|
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||||
|
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||||
|
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||||
|
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||||
|
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||||
|
|
||||||
|
## 公开评审与拒稿边界
|
||||||
|
|
||||||
|
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||||
|
|
||||||
|
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||||
|
|
||||||
|
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||||
|
|
||||||
|
## 恢复与验证
|
||||||
|
|
||||||
|
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||||
|
|
||||||
|
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||||
|
|
||||||
|
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||||
|
|
||||||
|
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||||
@@ -0,0 +1,110 @@
|
|||||||
|
# ACM MM 2026 官方节目来源状态
|
||||||
|
|
||||||
|
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||||
|
|
||||||
|
## 已取得的数据
|
||||||
|
|
||||||
|
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||||
|
|
||||||
|
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||||
|
|
||||||
|
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||||
|
|
||||||
|
| 来源 track 原文 | 不同节目 ID 数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| MRAC | 35 |
|
||||||
|
| SUMAC | 9 |
|
||||||
|
| NeuroMM | 10 |
|
||||||
|
| IMPACT | 8 |
|
||||||
|
| RichMediaGAI | 10 |
|
||||||
|
| MMSports | 16 |
|
||||||
|
| MUWS | 5 |
|
||||||
|
| EReL-MIR | 7 |
|
||||||
|
| GreenMM | 10 |
|
||||||
|
| DFF | 12 |
|
||||||
|
| Main Track | 1703 |
|
||||||
|
| Grand Challenge | 97 |
|
||||||
|
| Doctoral Symposium | 4 |
|
||||||
|
| Datasets | 50 |
|
||||||
|
| Interactive Art | 23 |
|
||||||
|
| Reproducibility | 1 |
|
||||||
|
| Demo and Video | 27 |
|
||||||
|
| Open Source | 14 |
|
||||||
|
| Industry Talk | 7 |
|
||||||
|
| BNI | 25 |
|
||||||
|
| QoEVMA | 4 |
|
||||||
|
| HUMA | 14 |
|
||||||
|
| WIPAI | 8 |
|
||||||
|
| AMI | 11 |
|
||||||
|
| Mulsemedia | 9 |
|
||||||
|
| UAVM | 11 |
|
||||||
|
| MCHM | 7 |
|
||||||
|
| JAV-CG | 6 |
|
||||||
|
|
||||||
|
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||||
|
|
||||||
|
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||||
|
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||||
|
|
||||||
|
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||||
|
|
||||||
|
## 公开负面决定的访问边界
|
||||||
|
|
||||||
|
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||||
|
|
||||||
|
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
原始目录(均相对 `/data/paper-index/collected/`):
|
||||||
|
|
||||||
|
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||||
|
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||||
|
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||||
|
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||||
|
|
||||||
|
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||||
|
|
||||||
|
| 证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||||
|
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||||
|
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||||
|
|
||||||
|
## 可复用采集命令
|
||||||
|
|
||||||
|
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/mm2026-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||||
|
|
||||||
|
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||||
|
|
||||||
|
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||||
|
|
||||||
|
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||||
|
|
||||||
|
## 直接审计节目身份与既往历史
|
||||||
|
|
||||||
|
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||||
|
paper-atlas audit source-identities \
|
||||||
|
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||||
|
|
||||||
|
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||||
|
|
||||||
|
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# RecSys 2026 公开录用贡献采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect recsys2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||||
|
|
||||||
|
## 数据范围与重复项
|
||||||
|
|
||||||
|
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||||
|
|
||||||
|
| track | 观察数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| long | 65 |
|
||||||
|
| short | 29 |
|
||||||
|
| past_present_future | 13 |
|
||||||
|
| reproducibility | 4 |
|
||||||
|
| resource | 7 |
|
||||||
|
| industry | 83 |
|
||||||
|
| research_practice_notes | 34 |
|
||||||
|
| demo | 18 |
|
||||||
|
| doctoral_symposium | 10 |
|
||||||
|
|
||||||
|
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||||
|
|
||||||
|
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||||
|
|
||||||
|
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||||
|
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||||
|
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||||
|
|
||||||
|
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||||
|
|
||||||
|
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||||
|
|
||||||
|
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||||
|
|
||||||
|
## 输出与恢复
|
||||||
|
|
||||||
|
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||||
|
|
||||||
|
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||||
|
|
||||||
|
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||||
|
|
||||||
|
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
## 2026-11-17 PostgreSQL 补录验收
|
||||||
|
|
||||||
|
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||||
|
|
||||||
|
| track | 已导入观察 / 来源应有观察 |
|
||||||
|
| --- | ---: |
|
||||||
|
| demo | 1 / 18 |
|
||||||
|
| doctoral_symposium | 2 / 10 |
|
||||||
|
| industry | 18 / 83 |
|
||||||
|
| long | 13 / 65 |
|
||||||
|
| past_present_future | 2 / 13 |
|
||||||
|
| reproducibility | 1 / 4 |
|
||||||
|
| research_practice_notes | 4 / 34 |
|
||||||
|
| resource | 2 / 7 |
|
||||||
|
| short | 4 / 29 |
|
||||||
|
|
||||||
|
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||||
|
|
||||||
|
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||||
|
|
||||||
|
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||||
|
|
||||||
|
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||||
|
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||||
|
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||||
|
|
||||||
|
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||||
|
|
||||||
|
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# SDM 2026:生产规范化与身份核查记录
|
# SDM 2026:生产规范化与身份核查记录
|
||||||
|
|
||||||
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||||
|
|
||||||
## 已核实的生产输出
|
## 已核实的生产输出
|
||||||
|
|
||||||
@@ -52,7 +52,7 @@
|
|||||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||||
|
|
||||||
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||||
|
|
||||||
## 未精确匹配的补充检索
|
## 未精确匹配的补充检索
|
||||||
|
|
||||||
@@ -70,9 +70,28 @@
|
|||||||
|
|
||||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||||
|
|
||||||
|
## 25 条生产导入验收
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||||
|
|
||||||
|
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||||
|
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||||
|
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||||
|
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||||
|
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||||
|
|
||||||
|
| 生产证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||||
|
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||||
|
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||||
|
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||||
|
|
||||||
|
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||||
|
|
||||||
## 下次恢复
|
## 下次恢复
|
||||||
|
|
||||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||||
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||||
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||||
|
|||||||
@@ -74,6 +74,74 @@ def load_inputs(files):
|
|||||||
return records, inputs
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def load_mm_program_inputs(report_path):
|
||||||
|
"""Read every MM observation without turning a program into import records.
|
||||||
|
|
||||||
|
Reconcile the complete output against its frozen raw source before any DB
|
||||||
|
connection. The synthetic unknown status is an audit placeholder only.
|
||||||
|
"""
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
|
||||||
|
report_path = Path(report_path).absolute()
|
||||||
|
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||||
|
report = json.loads(raw_report)
|
||||||
|
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||||
|
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||||
|
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||||
|
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||||
|
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Unreviewed MM observation report')
|
||||||
|
root = report_path.parent
|
||||||
|
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||||
|
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||||
|
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||||
|
raise ValueError('MM raw evidence changed')
|
||||||
|
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||||
|
expected = {r['paper_id']:r for r in expected_rows}
|
||||||
|
records, inputs, seen, tracks = [], [], set(), set()
|
||||||
|
for item in report['files']:
|
||||||
|
track = item.get('track_literal')
|
||||||
|
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||||
|
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||||
|
or item.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Duplicate or unreviewed MM track format')
|
||||||
|
tracks.add(track)
|
||||||
|
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||||
|
path = root/Path(item['file']).name
|
||||||
|
if path.suffix != '.jsonl':
|
||||||
|
raise ValueError('Invalid MM observation filename')
|
||||||
|
raw = mm2026.safe_read(path, 16*1024**2)
|
||||||
|
lines = raw.splitlines(keepends=True)
|
||||||
|
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||||
|
raise ValueError('MM observation file count/hash mismatch')
|
||||||
|
for number,line in enumerate(lines, 1):
|
||||||
|
row = json.loads(line)
|
||||||
|
pid = row.get('paper_id')
|
||||||
|
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||||
|
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||||
|
seen.add(pid)
|
||||||
|
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||||
|
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||||
|
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||||
|
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||||
|
'identifier_hits':[],'title_candidate_ids':[],
|
||||||
|
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'program_id':pid,'program_contexts':row['contexts'],
|
||||||
|
'source_observed_status':row['observed_status'],
|
||||||
|
'missing_source_fields':row['missing_fields'],
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'contribution_type_verified':False})
|
||||||
|
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||||
|
'manifest_sha256':digest(raw_report),'manifest':{
|
||||||
|
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||||
|
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||||
|
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||||
|
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||||
|
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
def _lookup(connection, values, columns, lateral):
|
def _lookup(connection, values, columns, lateral):
|
||||||
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||||
width = len(columns)
|
width = len(columns)
|
||||||
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
|
|||||||
temporary.replace(path)
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
def audit(files, out, *, pg_service, pg_schema):
|
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||||
# Reject input corruption before opening a database connection or reserving output.
|
# Reject input corruption before opening a database connection or reserving output.
|
||||||
records, inputs = load_inputs(files)
|
if bool(files) == bool(mm_program_report):
|
||||||
|
raise ValueError('Choose source files or one MM program report')
|
||||||
|
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||||
out = Path(out)
|
out = Path(out)
|
||||||
out.mkdir(parents=True, exist_ok=False)
|
out.mkdir(parents=True, exist_ok=False)
|
||||||
(out / 'audit.lock').write_text(timestamp() + '\n')
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||||
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
'Name agreement does not prove person or scientific-work identity.',
|
'Name agreement does not prove person or scientific-work identity.',
|
||||||
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||||
'Source/history limits are explicit in compressed entity packets.',
|
'Source/history limits are explicit in compressed entity packets.',
|
||||||
|
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||||
'Not a final whole-database deduplication or coverage audit.']}
|
'Not a final whole-database deduplication or coverage audit.']}
|
||||||
_atomic_json(out / 'postgresql-preflight.json', report)
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||||
return report
|
return report
|
||||||
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
def main(argv=None, *, prog=None):
|
||||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
inputs.add_argument('--files', nargs='+', type=Path)
|
||||||
|
inputs.add_argument('--mm-program-report', type=Path,
|
||||||
|
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||||
parser.add_argument('--pg-service', required=True)
|
parser.add_argument('--pg-service', required=True)
|
||||||
parser.add_argument('--pg-schema', required=True)
|
parser.add_argument('--pg-schema', required=True)
|
||||||
parser.add_argument('--out', required=True, type=Path)
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
mm_program_report=args.mm_program_report)
|
||||||
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||||
|
|||||||
@@ -23,12 +23,16 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icassp_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||||
|
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||||
|
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||||
|
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||||
|
PREFIX = '10.1109/icassp55912.2026.'
|
||||||
|
NONPAPERS = {
|
||||||
|
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||||
|
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||||
|
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||||
|
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||||
|
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||||
|
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||||
|
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||||
|
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||||
|
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||||
|
or len(manifest['pages']) != 10):
|
||||||
|
raise ValueError('Incomplete frozen census receipt')
|
||||||
|
files = {'manifest.json': raw}; rows = []
|
||||||
|
for index, receipt in enumerate(manifest['pages'], 1):
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||||
|
or path.stat().st_size > 4*1024**2):
|
||||||
|
raise ValueError('Unsafe snapshot page')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
data = json.loads(body); message = data['message']
|
||||||
|
expected = 500 if index < 10 else 145
|
||||||
|
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||||
|
raise ValueError('Incomplete registry page')
|
||||||
|
files[name] = body
|
||||||
|
for original in message['items']:
|
||||||
|
rows.append((original, receipt))
|
||||||
|
return manifest, files, rows
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(rows):
|
||||||
|
seen = set(); papers = []; nonpapers = []; other = []
|
||||||
|
for original, receipt in rows:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||||
|
if doi.startswith(PREFIX):
|
||||||
|
raise ValueError('Conference DOI/container conflict')
|
||||||
|
other.append(doi)
|
||||||
|
continue
|
||||||
|
if (original.get('container-title') != [CONTAINER]
|
||||||
|
or original.get('event', {}).get('name') != CONTAINER
|
||||||
|
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||||
|
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||||
|
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||||
|
if doi in NONPAPERS:
|
||||||
|
title, page = NONPAPERS[doi]
|
||||||
|
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||||
|
raise ValueError('Changed reviewed nonpaper evidence')
|
||||||
|
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||||
|
continue
|
||||||
|
titles = original.get('title')
|
||||||
|
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
authors = original.get('author')
|
||||||
|
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||||
|
raise ValueError('Missing authors or changed abstract availability')
|
||||||
|
names = []; flags = []
|
||||||
|
for position, author in enumerate(authors):
|
||||||
|
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||||
|
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||||
|
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||||
|
raise ValueError('Incomplete registry author or sequence')
|
||||||
|
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||||
|
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
if not author.get('given'):
|
||||||
|
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||||
|
names.append(name)
|
||||||
|
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||||
|
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||||
|
raise ValueError('Incomplete reviewed registry partition')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
manifest, files, items = load_snapshot(snapshots)
|
||||||
|
papers, nonpapers, other = registry_papers(items)
|
||||||
|
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||||
|
for name, body in files.items():
|
||||||
|
(evidence_dir/name).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||||
|
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||||
|
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||||
|
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||||
|
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||||
|
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||||
|
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||||
|
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||||
|
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||||
|
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||||
|
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||||
|
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'ismir_program'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||||
|
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||||
|
'research_completed_not_coverage_or_import_approval', {
|
||||||
|
'ismir-accepted.html': ACCEPTED,
|
||||||
|
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||||
|
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||||
|
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||||
|
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||||
|
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||||
|
}
|
||||||
|
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def title_key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshots(root):
|
||||||
|
root = Path(root)
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ValueError('Unsafe snapshot root')
|
||||||
|
result = {}
|
||||||
|
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
files = {'manifest.json': raw}; receipts = {}
|
||||||
|
for filename, url in urls.items():
|
||||||
|
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||||
|
path = directory/filename
|
||||||
|
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body disagree')
|
||||||
|
files[filename] = body; receipts[filename] = receipt
|
||||||
|
result[name] = (files, receipts)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(literal):
|
||||||
|
# Institution names can contain semicolons; split only outside parentheses.
|
||||||
|
parts = []; depth = 0; start = 0
|
||||||
|
for index, char in enumerate(literal):
|
||||||
|
if char == '(': depth += 1
|
||||||
|
elif char == ')':
|
||||||
|
depth -= 1
|
||||||
|
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||||
|
elif char == ';' and depth == 0:
|
||||||
|
parts.append(literal[start:index]); start = index+1
|
||||||
|
if depth:
|
||||||
|
raise ValueError('Unbalanced author affiliation')
|
||||||
|
parts.append(literal[start:])
|
||||||
|
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||||
|
if any(not name for name in names):
|
||||||
|
raise ValueError('Missing accepted-list author')
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_listing(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); tables = []
|
||||||
|
for table in doc.root.walk():
|
||||||
|
if table.tag != 'table': continue
|
||||||
|
rows = []
|
||||||
|
for row in table.walk():
|
||||||
|
if row.tag == 'tr':
|
||||||
|
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||||
|
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||||
|
tables.append(rows)
|
||||||
|
rows = one(tables)[1:]; result = {}
|
||||||
|
if len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete official accepted-list census')
|
||||||
|
for row in rows:
|
||||||
|
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||||
|
raise ValueError('Missing or ambiguous accepted-list title')
|
||||||
|
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(rows, listing):
|
||||||
|
if not isinstance(rows, list) or len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete public program census')
|
||||||
|
observations = []; seen = set(); used = set()
|
||||||
|
for original in rows:
|
||||||
|
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||||
|
raise ValueError('Invalid public program record')
|
||||||
|
pid = original.get('id'); content = original.get('content', {})
|
||||||
|
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||||
|
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||||
|
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||||
|
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||||
|
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||||
|
raise ValueError('Missing program title, author sequence or abstract')
|
||||||
|
key = title_key(title)
|
||||||
|
if key not in listing or key in used:
|
||||||
|
raise ValueError('Program/accepted-list title membership mismatch')
|
||||||
|
used.add(key); accepted = listing[key]
|
||||||
|
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||||
|
session = content.get('session'); reviews = original.get('reviews')
|
||||||
|
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||||
|
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||||
|
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||||
|
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||||
|
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||||
|
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||||
|
or any(not isinstance(v, str) for v in reviews.values())
|
||||||
|
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||||
|
raise ValueError('Unreviewed opt-in review structure')
|
||||||
|
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||||
|
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||||
|
raise ValueError('Review publication preference/body disagree')
|
||||||
|
flags = []
|
||||||
|
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||||
|
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||||
|
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||||
|
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||||
|
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||||
|
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||||
|
raise ValueError('Unreviewed source-version or opt-in census')
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
sources = load_snapshots(snapshots)
|
||||||
|
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||||
|
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||||
|
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||||
|
raise ValueError('Unreviewed public program data path or poster link template')
|
||||||
|
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, (files, _) in sources.items():
|
||||||
|
directory = evidence/name; directory.mkdir()
|
||||||
|
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for observation in observations:
|
||||||
|
original = observation['original']; content = original['content']; pid = original['id']
|
||||||
|
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||||
|
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||||
|
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||||
|
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||||
|
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||||
|
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||||
|
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||||
|
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||||
|
'original_decision_signature_verified':False,'paper_type':None,
|
||||||
|
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||||
|
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||||
|
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||||
|
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||||
|
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||||
|
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||||
|
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||||
|
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||||
|
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||||
|
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||||
|
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||||
|
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
SOURCE = 'mm_program'
|
||||||
|
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||||
|
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||||
|
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||||
|
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||||
|
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||||
|
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||||
|
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||||
|
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||||
|
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||||
|
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||||
|
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||||
|
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def safe_read(path, limit):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||||
|
raise ValueError('Unsafe snapshot path or size')
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||||
|
raise ValueError('Unreviewed manifest scope')
|
||||||
|
evidence = {'manifest.json':raw}
|
||||||
|
for receipt in manifest['entries']:
|
||||||
|
name = receipt['file']
|
||||||
|
if Path(name).name != name or name in evidence:
|
||||||
|
raise ValueError('Unsafe or duplicate receipt name')
|
||||||
|
body = safe_read(directory/name, 12*1024**2)
|
||||||
|
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['content_type'].startswith('application/json')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if name == 'program-data.json':
|
||||||
|
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||||
|
raise ValueError('Wrong program source')
|
||||||
|
elif receipt['http_status'] != 403:
|
||||||
|
raise ValueError('Unreviewed public-access result')
|
||||||
|
evidence[name] = body
|
||||||
|
if 'program-data.json' not in evidence:
|
||||||
|
raise ValueError('Missing program source')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def without_emails(value):
|
||||||
|
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [without_emails(v) for v in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def program_locations(node, pointer=''):
|
||||||
|
"""Independent structural guard against silently skipping a new nesting form."""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
if 'paper_id' in node:
|
||||||
|
yield pointer
|
||||||
|
for key,value in node.items():
|
||||||
|
escaped = key.replace('~','~0').replace('/','~1')
|
||||||
|
yield from program_locations(value, pointer+'/'+escaped)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for i,value in enumerate(node):
|
||||||
|
yield from program_locations(value, pointer+'/'+str(i))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
data = json.loads(raw)
|
||||||
|
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||||
|
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||||
|
or len(data.get('entries',[])) != 157):
|
||||||
|
raise ValueError('Unreviewed program edition or census')
|
||||||
|
records = {}; sessions = []; seen = []
|
||||||
|
|
||||||
|
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||||
|
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||||
|
raise ValueError('Unreviewed program fields')
|
||||||
|
for key in ('paper_id','title','abstract','track'):
|
||||||
|
if not isinstance(paper[key],str):
|
||||||
|
raise ValueError('Wrong program field type')
|
||||||
|
for key in ('authors','affiliations'):
|
||||||
|
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||||
|
raise ValueError('Wrong author or affiliation list')
|
||||||
|
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||||
|
raise ValueError('Missing identity/title or unreviewed source track')
|
||||||
|
core = {k:paper[k] for k in CORE}
|
||||||
|
pid = paper['paper_id']; seen.append(pointer)
|
||||||
|
if pid not in records:
|
||||||
|
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||||
|
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||||
|
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||||
|
'observed_status':'listed_in_official_program',
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'approved_for_import':False,'missing_fields':missing,
|
||||||
|
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||||
|
'contexts':[]}
|
||||||
|
elif {k:records[pid][k] for k in CORE} != core:
|
||||||
|
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||||
|
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||||
|
'subsession_pointer':subsession_pointer,
|
||||||
|
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||||
|
|
||||||
|
for i,entry in enumerate(data['entries']):
|
||||||
|
pointer = '/entries/'+str(i)
|
||||||
|
if entry['track'] != '':
|
||||||
|
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||||
|
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||||
|
session.update(source_pointer=pointer,subsessions=[])
|
||||||
|
for j,paper in enumerate(entry.get('papers') or []):
|
||||||
|
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||||
|
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||||
|
sp = pointer+'/subsessions/'+str(j)
|
||||||
|
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||||
|
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||||
|
for k,item in enumerate(sub.get('items') or []):
|
||||||
|
ip = sp+'/items/'+str(k)
|
||||||
|
if 'paper_id' in item:
|
||||||
|
observe(item,ip,pointer,sp)
|
||||||
|
else:
|
||||||
|
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||||
|
session['subsessions'].append(saved)
|
||||||
|
sessions.append(session)
|
||||||
|
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||||
|
raise ValueError('Unhandled or duplicated program location')
|
||||||
|
items = list(records.values())
|
||||||
|
if (len(seen) != 4326 or len(items) != 2143
|
||||||
|
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||||
|
or sum(not r['abstract'] for r in items) != 216
|
||||||
|
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||||
|
raise ValueError('Program count or missing-field census changed')
|
||||||
|
return items,sessions
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; use a fresh directory')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = load_snapshot(snapshots)
|
||||||
|
rows,sessions = parse_program(evidence['program-data.json'])
|
||||||
|
(out/'evidence').mkdir()
|
||||||
|
for name,body in evidence.items():
|
||||||
|
(out/'evidence'/name).write_bytes(body)
|
||||||
|
outputs = []
|
||||||
|
for track,count in TRACK_COUNTS.items():
|
||||||
|
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||||
|
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||||
|
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||||
|
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||||
|
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||||
|
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||||
|
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||||
|
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||||
|
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||||
|
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||||
|
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||||
|
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||||
|
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||||
|
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||||
|
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||||
|
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||||
|
part = out/'collection-report.json.part'
|
||||||
|
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'recsys_program'
|
||||||
|
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||||
|
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||||
|
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||||
|
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||||
|
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||||
|
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||||
|
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||||
|
('doctoral_symposium', 'Doctoral', 10)]
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot body')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||||
|
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
return receipt, {'manifest.json': raw, name: body}, body
|
||||||
|
|
||||||
|
|
||||||
|
def body_text(node, doi_url):
|
||||||
|
if not isinstance(node, Node):
|
||||||
|
return node
|
||||||
|
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||||
|
return ''
|
||||||
|
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||||
|
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||||
|
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||||
|
raise ValueError('Wrong conference edition')
|
||||||
|
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||||
|
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||||
|
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||||
|
raise ValueError('Unreviewed track menu')
|
||||||
|
observations = []; seen = set()
|
||||||
|
for tab, (track, label, count) in enumerate(TRACKS):
|
||||||
|
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||||
|
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||||
|
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||||
|
if len(rows) != count:
|
||||||
|
raise ValueError('Incomplete reviewed track census')
|
||||||
|
for position, row in enumerate(rows, 1):
|
||||||
|
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||||
|
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||||
|
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||||
|
raise ValueError('Missing or repeated program locator')
|
||||||
|
seen.add(locator)
|
||||||
|
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||||
|
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||||
|
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||||
|
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||||
|
if (type_literal,abbreviation) != expected_type:
|
||||||
|
raise ValueError('Unreviewed source contribution type')
|
||||||
|
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||||
|
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||||
|
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||||
|
if not authors_literal.startswith('by '):
|
||||||
|
raise ValueError('Missing program author prefix')
|
||||||
|
authors_literal = authors_literal[3:]
|
||||||
|
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||||
|
if not title or any(not n or '@' in n for n in authors):
|
||||||
|
raise ValueError('Missing title or program author')
|
||||||
|
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||||
|
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||||
|
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||||
|
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||||
|
raise ValueError('Wrong publication DOI family')
|
||||||
|
abstract = clean(body_text(body, doi_url))
|
||||||
|
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||||
|
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||||
|
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||||
|
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||||
|
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||||
|
'track': track, 'track_label': label})
|
||||||
|
counts = Counter(r['doi'] for r in observations)
|
||||||
|
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||||
|
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||||
|
raise ValueError('Unreviewed DOI observation census')
|
||||||
|
for item in observations:
|
||||||
|
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
(evidence/name).write_bytes(raw)
|
||||||
|
outputs = []
|
||||||
|
for track,label,count in TRACKS:
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['track'] != track:
|
||||||
|
continue
|
||||||
|
doi = item['doi']; url = URL+'#'+item['locator']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||||
|
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||||
|
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||||
|
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||||
|
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||||
|
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified':False,
|
||||||
|
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||||
|
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||||
|
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||||
|
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||||
|
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||||
|
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||||
|
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||||
|
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||||
|
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||||
|
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,95 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icassp2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICASSP2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||||
|
cls.items = []
|
||||||
|
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||||
|
for receipt in cls.manifest['pages']:
|
||||||
|
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name, body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||||
|
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||||
|
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||||
|
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows = [json.loads(l) for l in raw.splitlines()]
|
||||||
|
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||||
|
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
self.assertEqual(actual,gold)
|
||||||
|
original = {r['DOI']:r for r,_ in self.items}
|
||||||
|
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||||
|
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
evidence = r['metadata']['source_evidence']
|
||||||
|
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||||
|
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(root/'out',snapshots=src)
|
||||||
|
|
||||||
|
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||||
|
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||||
|
if kind=='container': r['container-title']=['Foreign']
|
||||||
|
if kind=='event': r['event']['name']='Other'
|
||||||
|
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||||
|
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||||
|
if kind=='author': r['author']=[]
|
||||||
|
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||||
|
if kind=='abstract': r['abstract']='new source version'
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||||
|
for kind in ('title','page','author'):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||||
|
if kind=='title': r['title']=['A research paper']
|
||||||
|
if kind=='page': r['page']='1-5'
|
||||||
|
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||||
|
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_duplicate_and_incomplete_census_rejected(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||||
|
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||||
|
|
||||||
|
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||||
|
if kind=='missing': path.unlink()
|
||||||
|
else: path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_page_and_snapshot_rejected(self):
|
||||||
|
for kind in ('page','directory'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||||
|
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import ismir2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ISMIR2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||||
|
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||||
|
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||||
|
|
||||||
|
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||||
|
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||||
|
self.assertEqual(len(observations),140)
|
||||||
|
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||||
|
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||||
|
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||||
|
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||||
|
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||||
|
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||||
|
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||||
|
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||||
|
|
||||||
|
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||||
|
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||||
|
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||||
|
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||||
|
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||||
|
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||||
|
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||||
|
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||||
|
rows=copy.deepcopy(self.program)
|
||||||
|
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||||
|
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||||
|
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||||
|
if kind=='missing_row':rows.pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||||
|
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||||
|
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_source_optin_no_cannot_publish_review_text(self):
|
||||||
|
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||||
|
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||||
|
|
||||||
|
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||||
|
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||||
|
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||||
|
with self.assertRaises(ValueError):c.author_names(raw)
|
||||||
|
|
||||||
|
def test_ambiguous_official_title_is_not_fused(self):
|
||||||
|
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||||
|
|
||||||
|
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||||
|
for kind in ('manifest','body','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MM2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||||
|
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||||
|
cls.data = json.loads(cls.raw)
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name,body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def parse_changed(self, data):
|
||||||
|
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||||
|
|
||||||
|
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||||
|
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||||
|
actual = {r['paper_id']:r for r in rows}
|
||||||
|
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||||
|
for gold in self.gold['inventory']:
|
||||||
|
r = actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||||
|
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||||
|
|
||||||
|
def test_every_presentation_and_session_context_is_preserved(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
for r in rows:
|
||||||
|
for context in r['contexts']:
|
||||||
|
node = self.data
|
||||||
|
for key in context['source_pointer'].split('/')[1:]:
|
||||||
|
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||||
|
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||||
|
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||||
|
for i,session in enumerate(sessions):
|
||||||
|
original = self.data['entries'][i]
|
||||||
|
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||||
|
self.assertEqual(session['track'],'')
|
||||||
|
for key in original.keys()-{'papers','subsessions'}:
|
||||||
|
self.assertEqual(session[key],original[key])
|
||||||
|
for j,sub in enumerate(original.get('subsessions') or []):
|
||||||
|
saved = session['subsessions'][j]
|
||||||
|
self.assertEqual(saved['title'],sub['title'])
|
||||||
|
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||||
|
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||||
|
|
||||||
|
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||||
|
rows,_ = c.parse_program(self.raw)
|
||||||
|
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||||
|
for r in rows:
|
||||||
|
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['publication_identity_verified'])
|
||||||
|
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||||
|
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||||
|
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||||
|
missing = [r for r in rows if not r['authors']]
|
||||||
|
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||||
|
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||||
|
|
||||||
|
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src')
|
||||||
|
report=c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||||
|
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||||
|
for name,body in self.files.items():
|
||||||
|
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||||
|
n=0
|
||||||
|
for item in report['files']:
|
||||||
|
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||||
|
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||||
|
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||||
|
self.assertEqual(n,2143)
|
||||||
|
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
first=data['entries'][0]['papers'][0]
|
||||||
|
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||||
|
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
moved=data['entries'][0]['papers'].pop()
|
||||||
|
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||||
|
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||||
|
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||||
|
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||||
|
elif kind=='paper_track':paper['track']='New Track'
|
||||||
|
elif kind=='field':paper['doi']='unreviewed'
|
||||||
|
elif kind=='title':paper['title']=''
|
||||||
|
else:paper['authors']='Unparsed author string'
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_edition_or_census_change_requires_review(self):
|
||||||
|
for kind in ('edition','count'):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||||
|
else:data['entries'][0]['papers'].pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||||
|
for kind in ('manifest','program','probe','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||||
|
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||||
|
path=directory/name
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':
|
||||||
|
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||||
|
rows,sessions=c.parse_program(self.raw)
|
||||||
|
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||||
|
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||||
|
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||||
|
|
||||||
|
|
||||||
|
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
src = self.root/'source'
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||||
|
mm2026.collect(self.root/'program', snapshots=src)
|
||||||
|
self.report_path = self.root/'program/collection-report.json'
|
||||||
|
|
||||||
|
def change_report(self, change):
|
||||||
|
report=json.loads(self.report_path.read_text());change(report)
|
||||||
|
self.report_path.write_text(json.dumps(report))
|
||||||
|
|
||||||
|
def assert_rejected_before_database(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||||
|
connection.assert_not_called()
|
||||||
|
self.assertFalse((self.root/'audit').exists())
|
||||||
|
|
||||||
|
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||||
|
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||||
|
actual={r['program_id']:r for r in rows}
|
||||||
|
for gold in self.fixture['independent_inventory']['inventory']:
|
||||||
|
r=actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal']))
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||||
|
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||||
|
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||||
|
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['contribution_type_verified'])
|
||||||
|
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||||
|
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||||
|
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||||
|
|
||||||
|
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||||
|
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143)
|
||||||
|
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||||
|
|
||||||
|
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||||
|
report=json.loads(self.report_path.read_text())
|
||||||
|
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||||
|
|
||||||
|
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||||
|
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||||
|
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||||
|
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||||
|
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||||
|
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||||
|
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||||
|
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||||
|
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_missing_or_repeated_track_rejected(self):
|
||||||
|
original=self.report_path.read_bytes()
|
||||||
|
for kind in ('missing','repeated'):
|
||||||
|
self.report_path.write_bytes(original)
|
||||||
|
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||||
|
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||||
|
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||||
|
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||||
|
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||||
|
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_mixed_input_modes_rejected_before_connection(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||||
|
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||||
|
connection.assert_not_called()
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import base64
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import recsys2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class RecSys2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||||
|
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||||
|
|
||||||
|
def test_all263_observations_against_independent_gold(self):
|
||||||
|
items=c.parse_program(self.body)
|
||||||
|
keys=self.fixture['independent_gold'][0].keys()
|
||||||
|
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||||
|
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||||
|
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||||
|
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||||
|
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||||
|
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||||
|
|
||||||
|
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||||
|
report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||||
|
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||||
|
for manifest in report['files']:
|
||||||
|
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
for r in rows:
|
||||||
|
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||||
|
for key in expected:self.assertEqual(item[key],expected[key])
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||||
|
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_edition_and_track_label_drift_rejected(self):
|
||||||
|
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||||
|
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||||
|
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_and_root_rejected(self):
|
||||||
|
for kind in ('source','root'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||||
|
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||||
Reference in New Issue
Block a user