Compare commits
26 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| afb3b939ca | |||
| 8c147f5e6e | |||
| 6961710e96 | |||
| f4be9de547 | |||
| 33bb378369 | |||
| 5bdf9f0fe5 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 564821d883 | |||
| 2f1b5c959a | |||
| f139c7e49d | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| 06467835c7 | |||
| f2753bc741 | |||
| 1ea17c810d | |||
| f800c6ecb4 | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 |
+117
-1
@@ -12,7 +12,7 @@ jobs:
|
||||
- name: Verify metadata storage and query commands
|
||||
run: |
|
||||
python3 -B - <<'PY'
|
||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
sha = os.environ['ARCHIVE_SHA']
|
||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||
raise SystemExit('Invalid source commit')
|
||||
@@ -46,9 +46,80 @@ jobs:
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside = root / 'outside-checkout'
|
||||
outside.mkdir()
|
||||
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
mm_snapshots = root/'mm2026-snapshots'
|
||||
for name, body in mm_fixture['files'].items():
|
||||
path = mm_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(body.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||
import hashlib
|
||||
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||
row = mm_rows[gold['paper_id']]
|
||||
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||
if service_file.is_file():
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
eccv_official = root/'eccv2026-official'
|
||||
eccv_papers = root/'eccv2026-paper-samples'
|
||||
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||
for name, html in eccv_fixture['official_pages'].items():
|
||||
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||
for pid, html in eccv_fixture['paper_pages'].items():
|
||||
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||
eccv_encoding = eccv_fixture['encoding_sample']
|
||||
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||
eccv_literal = eccv_fixture['literal_title_sample']
|
||||
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
icassp_snapshots = root/'icassp2026-snapshots'
|
||||
icassp_snapshots.mkdir()
|
||||
for name, encoded in icassp_fixture['files'].items():
|
||||
target = icassp_snapshots/name
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
target.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||
assert icassp_report['single_name_author_fields'] == 23
|
||||
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
@@ -66,6 +137,21 @@ jobs:
|
||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
icdm_snapshots = root/'icdm2026-snapshots'
|
||||
for name, text in icdm_fixture['files'].items():
|
||||
target = icdm_snapshots/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(text.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
@@ -92,6 +178,36 @@ jobs:
|
||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
recsys_snapshots = root/'recsys2026-snapshots'
|
||||
recsys_snapshots.mkdir()
|
||||
for name,encoded in recsys_fixture['files'].items():
|
||||
path = recsys_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||
ismir_snapshots = root/'ismir2026-snapshots'
|
||||
ismir_snapshots.mkdir()
|
||||
for name,encoded in ismir_fixture['files'].items():
|
||||
target = ismir_snapshots/name
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
target.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
corl_official = root/'corl2026-official'
|
||||
corl_archive = root/'corl2026-archive'
|
||||
|
||||
@@ -129,4 +129,16 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
||||
|
||||
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||
|
||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||
|
||||
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||
|
||||
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# 会议覆盖与公开决定核查
|
||||
|
||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||
|
||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||
|
||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||
@@ -187,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
||||
|
||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||
|
||||
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||
|
||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||
|
||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||
@@ -196,3 +200,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||
|
||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||
|
||||
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# 2020 起补漏状态:2026-11-17
|
||||
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
|
||||
## 最新数据库盘点
|
||||
|
||||
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||
|
||||
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||
| --- | --- |
|
||||
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||
|
||||
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||
|
||||
## 已独立接受的近期补录
|
||||
|
||||
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||
|
||||
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||
|
||||
## 已采集、尚待关联或规范化的来源
|
||||
|
||||
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||
| --- | --- | --- |
|
||||
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
||||
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||
|
||||
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||
|
||||
## 公开拒稿与后续工作
|
||||
|
||||
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||
|
||||
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||
|
||||
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||
@@ -0,0 +1,35 @@
|
||||
# ECCV 2026:暂定节目与公开元数据
|
||||
|
||||
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||
|
||||
```sh
|
||||
paper-atlas collect eccv2026-program \
|
||||
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||
--max-new-pages 200 \
|
||||
--out /data/paper-index/collected/eccv2026-pass01
|
||||
```
|
||||
|
||||
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||
|
||||
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||
|
||||
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||
|
||||
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||
|
||||
## 来源边界
|
||||
|
||||
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||
|
||||
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||
|
||||
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||
|
||||
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||
|
||||
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||
|
||||
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||
|
||||
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||
@@ -0,0 +1,59 @@
|
||||
# ECCV 2026 来源核查记录
|
||||
|
||||
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||
|
||||
## 贡献类型与 track
|
||||
|
||||
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||
|
||||
| 官方 group 字段值 | 含义摘要 |
|
||||
| --- | --- |
|
||||
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||
| Concept & Feasibility | 新概念及可行性证据 |
|
||||
|
||||
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||
|
||||
## 公开负面决定元数据
|
||||
|
||||
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||
|
||||
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||
|
||||
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||
|
||||
## 可恢复证据与后续入口
|
||||
|
||||
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||
|
||||
| 文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||
|
||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||
|
||||
|
||||
## 2026-11-17 全量采集与只读预检
|
||||
|
||||
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||
|
||||
| 服务器证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||
|
||||
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||
|
||||
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||
|
||||
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||
@@ -0,0 +1,40 @@
|
||||
# ICASSP 2026 注册元数据采集
|
||||
|
||||
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||
|
||||
## 输入与使用
|
||||
|
||||
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||
|
||||
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||
|
||||
```sh
|
||||
paper-atlas collect icassp2026-registry \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||
|
||||
## 数据范围
|
||||
|
||||
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||
|
||||
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||
|
||||
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||
|
||||
## 证据与恢复
|
||||
|
||||
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||
|
||||
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||
|
||||
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||
|
||||
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||
@@ -0,0 +1,46 @@
|
||||
# ICDM2026 冻结日程适配器
|
||||
|
||||
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||
|
||||
```sh
|
||||
paper-atlas collect icdm2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||
```
|
||||
|
||||
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||
|
||||
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||
|
||||
## 两个明确来源轨道
|
||||
|
||||
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||
|
||||
| 输出文件 | 官方明确标签 | 记录数 |
|
||||
| --- | --- | ---: |
|
||||
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||
|
||||
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||
|
||||
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||
|
||||
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||
|
||||
## 始终保留范围限制
|
||||
|
||||
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||
|
||||
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||
|
||||
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||
|
||||
## 验证与恢复
|
||||
|
||||
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||
|
||||
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||
|
||||
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||
|
||||
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||
@@ -0,0 +1,50 @@
|
||||
# ISMIR 2026 官方节目与摘要采集
|
||||
|
||||
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||
|
||||
## 使用
|
||||
|
||||
```sh
|
||||
paper-atlas collect ismir2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输入父目录需要三个冻结目录:
|
||||
|
||||
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||
|
||||
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||
|
||||
## 来源与版本
|
||||
|
||||
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||
|
||||
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||
|
||||
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||
|
||||
## 公开评审与拒稿边界
|
||||
|
||||
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||
|
||||
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||
|
||||
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||
|
||||
## 恢复与验证
|
||||
|
||||
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||
|
||||
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||
|
||||
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||
|
||||
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||
@@ -0,0 +1,110 @@
|
||||
# ACM MM 2026 官方节目来源状态
|
||||
|
||||
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||
|
||||
## 已取得的数据
|
||||
|
||||
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||
|
||||
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||
|
||||
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||
|
||||
| 来源 track 原文 | 不同节目 ID 数 |
|
||||
| --- | ---: |
|
||||
| MRAC | 35 |
|
||||
| SUMAC | 9 |
|
||||
| NeuroMM | 10 |
|
||||
| IMPACT | 8 |
|
||||
| RichMediaGAI | 10 |
|
||||
| MMSports | 16 |
|
||||
| MUWS | 5 |
|
||||
| EReL-MIR | 7 |
|
||||
| GreenMM | 10 |
|
||||
| DFF | 12 |
|
||||
| Main Track | 1703 |
|
||||
| Grand Challenge | 97 |
|
||||
| Doctoral Symposium | 4 |
|
||||
| Datasets | 50 |
|
||||
| Interactive Art | 23 |
|
||||
| Reproducibility | 1 |
|
||||
| Demo and Video | 27 |
|
||||
| Open Source | 14 |
|
||||
| Industry Talk | 7 |
|
||||
| BNI | 25 |
|
||||
| QoEVMA | 4 |
|
||||
| HUMA | 14 |
|
||||
| WIPAI | 8 |
|
||||
| AMI | 11 |
|
||||
| Mulsemedia | 9 |
|
||||
| UAVM | 11 |
|
||||
| MCHM | 7 |
|
||||
| JAV-CG | 6 |
|
||||
|
||||
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||
|
||||
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||
|
||||
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||
|
||||
## 公开负面决定的访问边界
|
||||
|
||||
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||
|
||||
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||
|
||||
## 产物与恢复
|
||||
|
||||
原始目录(均相对 `/data/paper-index/collected/`):
|
||||
|
||||
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||
|
||||
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||
|
||||
| 证据 | SHA256 |
|
||||
| --- | --- |
|
||||
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||
|
||||
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||
|
||||
## 可复用采集命令
|
||||
|
||||
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||
|
||||
```sh
|
||||
paper-atlas collect mm2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/mm2026-native-v1
|
||||
```
|
||||
|
||||
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||
|
||||
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||
|
||||
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||
|
||||
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||
|
||||
## 直接审计节目身份与既往历史
|
||||
|
||||
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||
|
||||
```sh
|
||||
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||
paper-atlas audit source-identities \
|
||||
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||
```
|
||||
|
||||
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||
|
||||
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||
|
||||
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||
@@ -0,0 +1,83 @@
|
||||
# RecSys 2026 公开录用贡献采集
|
||||
|
||||
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||
|
||||
## 使用
|
||||
|
||||
```sh
|
||||
paper-atlas collect recsys2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||
|
||||
## 数据范围与重复项
|
||||
|
||||
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||
|
||||
| track | 观察数 |
|
||||
| --- | ---: |
|
||||
| long | 65 |
|
||||
| short | 29 |
|
||||
| past_present_future | 13 |
|
||||
| reproducibility | 4 |
|
||||
| resource | 7 |
|
||||
| industry | 83 |
|
||||
| research_practice_notes | 34 |
|
||||
| demo | 18 |
|
||||
| doctoral_symposium | 10 |
|
||||
|
||||
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||
|
||||
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||
|
||||
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||
|
||||
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||
|
||||
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||
|
||||
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||
|
||||
## 输出与恢复
|
||||
|
||||
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||
|
||||
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||
|
||||
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||
|
||||
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||
|
||||
## 2026-11-17 PostgreSQL 补录验收
|
||||
|
||||
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||
|
||||
| track | 已导入观察 / 来源应有观察 |
|
||||
| --- | ---: |
|
||||
| demo | 1 / 18 |
|
||||
| doctoral_symposium | 2 / 10 |
|
||||
| industry | 18 / 83 |
|
||||
| long | 13 / 65 |
|
||||
| past_present_future | 2 / 13 |
|
||||
| reproducibility | 1 / 4 |
|
||||
| research_practice_notes | 4 / 34 |
|
||||
| resource | 2 / 7 |
|
||||
| short | 4 / 29 |
|
||||
|
||||
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||
|
||||
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||
|
||||
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||
|
||||
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||
|
||||
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||
|
||||
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||
@@ -0,0 +1,97 @@
|
||||
# SDM 2026:生产规范化与身份核查记录
|
||||
|
||||
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||
|
||||
## 已核实的生产输出
|
||||
|
||||
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||
|
||||
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||
|
||||
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||
|
||||
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||
|
||||
| 证据文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||
|
||||
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||
|
||||
## 当前数据库只读预检
|
||||
|
||||
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||
|
||||
| 候选类别 | 记录数 | 含义 |
|
||||
| --- | ---: | --- |
|
||||
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||
|
||||
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||
|
||||
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||
|
||||
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||
| --- | ---: | --- |
|
||||
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||
|
||||
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||
|
||||
## 需要保护的较早拒稿历史
|
||||
|
||||
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||
| --- | ---: | --- |
|
||||
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||
|
||||
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
|
||||
## 未精确匹配的补充检索
|
||||
|
||||
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||
|
||||
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||
|
||||
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||
|
||||
| 服务器恢复文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||
|
||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||
|
||||
## 25 条生产导入验收
|
||||
|
||||
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||
|
||||
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||
|
||||
| 生产证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||
|
||||
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||
|
||||
## 下次恢复
|
||||
|
||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||
@@ -42,3 +42,5 @@ paper-atlas collect sdm2026-proceedings \
|
||||
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||
|
||||
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||
|
||||
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||
|
||||
@@ -74,6 +74,74 @@ def load_inputs(files):
|
||||
return records, inputs
|
||||
|
||||
|
||||
def load_mm_program_inputs(report_path):
|
||||
"""Read every MM observation without turning a program into import records.
|
||||
|
||||
Reconcile the complete output against its frozen raw source before any DB
|
||||
connection. The synthetic unknown status is an audit placeholder only.
|
||||
"""
|
||||
from paper_atlas.collectors import mm2026
|
||||
|
||||
report_path = Path(report_path).absolute()
|
||||
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||
report = json.loads(raw_report)
|
||||
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||
raise ValueError('Unreviewed MM observation report')
|
||||
root = report_path.parent
|
||||
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||
raise ValueError('MM raw evidence changed')
|
||||
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||
expected = {r['paper_id']:r for r in expected_rows}
|
||||
records, inputs, seen, tracks = [], [], set(), set()
|
||||
for item in report['files']:
|
||||
track = item.get('track_literal')
|
||||
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||
or item.get('approved_for_import') != 0):
|
||||
raise ValueError('Duplicate or unreviewed MM track format')
|
||||
tracks.add(track)
|
||||
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||
path = root/Path(item['file']).name
|
||||
if path.suffix != '.jsonl':
|
||||
raise ValueError('Invalid MM observation filename')
|
||||
raw = mm2026.safe_read(path, 16*1024**2)
|
||||
lines = raw.splitlines(keepends=True)
|
||||
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||
raise ValueError('MM observation file count/hash mismatch')
|
||||
for number,line in enumerate(lines, 1):
|
||||
row = json.loads(line)
|
||||
pid = row.get('paper_id')
|
||||
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||
seen.add(pid)
|
||||
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||
'identifier_hits':[],'title_candidate_ids':[],
|
||||
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'program_id':pid,'program_contexts':row['contexts'],
|
||||
'source_observed_status':row['observed_status'],
|
||||
'missing_source_fields':row['missing_fields'],
|
||||
'publication_identity_verified':False,'final_decision_verified':False,
|
||||
'contribution_type_verified':False})
|
||||
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||
'manifest_sha256':digest(raw_report),'manifest':{
|
||||
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||
return records, inputs
|
||||
|
||||
|
||||
def _lookup(connection, values, columns, lateral):
|
||||
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||
width = len(columns)
|
||||
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def audit(files, out, *, pg_service, pg_schema):
|
||||
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||
# Reject input corruption before opening a database connection or reserving output.
|
||||
records, inputs = load_inputs(files)
|
||||
if bool(files) == bool(mm_program_report):
|
||||
raise ValueError('Choose source files or one MM program report')
|
||||
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||
out = Path(out)
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
|
||||
'Name agreement does not prove person or scientific-work identity.',
|
||||
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||
'Source/history limits are explicit in compressed entity packets.',
|
||||
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||
'Not a final whole-database deduplication or coverage audit.']}
|
||||
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||
return report
|
||||
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
||||
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||
inputs.add_argument('--files', nargs='+', type=Path)
|
||||
inputs.add_argument('--mm-program-report', type=Path,
|
||||
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||
parser.add_argument('--pg-service', required=True)
|
||||
parser.add_argument('--pg-schema', required=True)
|
||||
parser.add_argument('--out', required=True, type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||
mm_program_report=args.mm_program_report)
|
||||
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||
|
||||
@@ -23,10 +23,16 @@ COMMANDS = {
|
||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
from html import unescape
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||
|
||||
SOURCE = 'eccv_program'
|
||||
ORIGIN = 'https://eccv.ecva.net'
|
||||
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||
TRACK = 'program_unclassified'
|
||||
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected one reviewed metadata field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, value):
|
||||
return value in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||
raise ValueError('Unexpected ECCV program edition')
|
||||
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||
raise ValueError('Reviewed provisional-publication notice changed')
|
||||
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||
papers = {}; occurrences = 0
|
||||
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) != 3:
|
||||
raise ValueError('Unexpected program columns')
|
||||
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||
if not row['title'] or not authors:
|
||||
raise ValueError('Incomplete public program fields')
|
||||
occurrences += 1
|
||||
if pid in papers:
|
||||
if papers[pid]['listing'] != row:
|
||||
raise ValueError('Conflicting entries share a program ID')
|
||||
papers[pid]['row_ordinals'].append(occurrences)
|
||||
else:
|
||||
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||
raise ValueError('Incomplete reviewed program census')
|
||||
return papers
|
||||
|
||||
|
||||
def parse_paper(raw, url, program):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||
abstract = clean(abstract_node.text())
|
||||
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||
authors = ld.get('author')
|
||||
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||
or not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||
# Decode only to test agreement with visible text, never to rewrite it.
|
||||
def agrees_with_display(value, expected):
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
for _ in range(2):
|
||||
value = unescape(value)
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
return False
|
||||
|
||||
raw_names = [a['name'] for a in authors]
|
||||
names = [clean(name) for name in raw_names]
|
||||
if not agrees_with_display(ld['name'], title):
|
||||
raise ValueError('Public visible and structured titles disagree')
|
||||
if ' ⋅ '.join(names) != authors_literal:
|
||||
visible_names = authors_literal.split(' ⋅ ')
|
||||
if len(visible_names) != len(raw_names) or any(
|
||||
not agrees_with_display(raw_name, visible_name)
|
||||
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
names = visible_names
|
||||
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||
if raw_names != names or ld['name'] != title:
|
||||
flags.append('jsonld_display_text_normalized_requires_review')
|
||||
if title != program['listing']['title']:
|
||||
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||
if authors_literal != program['listing']['authors_literal']:
|
||||
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||
|
||||
|
||||
def official_snapshot(root):
|
||||
root = Path(root); manifest = root/'manifest.json'
|
||||
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed official snapshot manifest changed')
|
||||
m = json.loads(manifest.read_bytes())
|
||||
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||
raise ValueError('Official snapshot requires a timezone')
|
||||
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||
raise ValueError('Official source set changed')
|
||||
pages = {}; evidence = {}
|
||||
for r in m['sources']:
|
||||
name = r['file']; p = root/name
|
||||
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe official snapshot')
|
||||
raw = p.read_bytes()
|
||||
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||
raise ValueError('Invalid official source receipt')
|
||||
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||
return m, pages, evidence, program
|
||||
|
||||
|
||||
def paper_snapshots(root, program):
|
||||
root = Path(root); path = root/'manifest.json'
|
||||
if root.is_symlink() or path.is_symlink():
|
||||
raise ValueError('Paper snapshot symlinks rejected')
|
||||
if path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Oversized paper manifest')
|
||||
m = json.loads(path.read_bytes()); result = {}
|
||||
for r in m['records']:
|
||||
pid = r['paper_id']
|
||||
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||
raise ValueError('Foreign or repeated paper snapshot')
|
||||
p = root/r['file']
|
||||
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||
raise ValueError('Unsafe paper snapshot')
|
||||
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||
return result
|
||||
|
||||
|
||||
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||
if (paper_snapshots_dir is None) == (cache is None):
|
||||
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
lock = None
|
||||
try:
|
||||
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||
directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in pages.items():
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||
client = None
|
||||
if cache is not None:
|
||||
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||
raise ValueError('Cache lock symlinks rejected')
|
||||
lock = (cache/'.collector.lock').open('a+')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
client = PublicHTMLCache(cache, ORIGIN)
|
||||
rows = []; missing = []; fetched = 0; reused = 0
|
||||
for pid in sorted(program, key=int):
|
||||
p = program[pid]
|
||||
if snapshots is not None:
|
||||
response = snapshots.get(pid)
|
||||
else:
|
||||
response = client.cached(p['url'])
|
||||
if response is not None:
|
||||
reused += 1
|
||||
elif max_new_pages is None or fetched < max_new_pages:
|
||||
response = client.get(p['url']); fetched += 1
|
||||
if response is None:
|
||||
missing.append(pid); continue
|
||||
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||
keyword = p['listing']['keywords_literal']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||
'evidence_url': ACCEPTED,
|
||||
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||
if len(rows) % 10 == 0:
|
||||
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||
files = []
|
||||
if rows:
|
||||
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||
raise
|
||||
finally:
|
||||
if lock is not None:
|
||||
lock.close()
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--official-snapshots', required=True)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--paper-snapshots')
|
||||
group.add_argument('--cache')
|
||||
parser.add_argument('--max-new-pages', type=int)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import clean
|
||||
|
||||
SOURCE = 'crossref_icassp_registry'
|
||||
TRACK = 'proceedings_unclassified'
|
||||
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||
PREFIX = '10.1109/icassp55912.2026.'
|
||||
NONPAPERS = {
|
||||
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||
or len(manifest['pages']) != 10):
|
||||
raise ValueError('Incomplete frozen census receipt')
|
||||
files = {'manifest.json': raw}; rows = []
|
||||
for index, receipt in enumerate(manifest['pages'], 1):
|
||||
name = receipt['file']; path = directory/name
|
||||
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||
or path.stat().st_size > 4*1024**2):
|
||||
raise ValueError('Unsafe snapshot page')
|
||||
body = path.read_bytes()
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
data = json.loads(body); message = data['message']
|
||||
expected = 500 if index < 10 else 145
|
||||
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||
raise ValueError('Incomplete registry page')
|
||||
files[name] = body
|
||||
for original in message['items']:
|
||||
rows.append((original, receipt))
|
||||
return manifest, files, rows
|
||||
|
||||
|
||||
def registry_papers(rows):
|
||||
seen = set(); papers = []; nonpapers = []; other = []
|
||||
for original, receipt in rows:
|
||||
doi = original.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen:
|
||||
raise ValueError('Missing or repeated registry DOI')
|
||||
seen.add(doi)
|
||||
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||
if doi.startswith(PREFIX):
|
||||
raise ValueError('Conference DOI/container conflict')
|
||||
other.append(doi)
|
||||
continue
|
||||
if (original.get('container-title') != [CONTAINER]
|
||||
or original.get('event', {}).get('name') != CONTAINER
|
||||
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||
or original.get('URL') != 'https://doi.org/'+doi
|
||||
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||
if doi in NONPAPERS:
|
||||
title, page = NONPAPERS[doi]
|
||||
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||
raise ValueError('Changed reviewed nonpaper evidence')
|
||||
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||
continue
|
||||
titles = original.get('title')
|
||||
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||
raise ValueError('Missing registry title')
|
||||
authors = original.get('author')
|
||||
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||
raise ValueError('Missing authors or changed abstract availability')
|
||||
names = []; flags = []
|
||||
for position, author in enumerate(authors):
|
||||
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||
raise ValueError('Incomplete registry author or sequence')
|
||||
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||
raise ValueError('Unreviewed author identity')
|
||||
if not author.get('given'):
|
||||
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||
names.append(name)
|
||||
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||
raise ValueError('Incomplete reviewed registry partition')
|
||||
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
manifest, files, items = load_snapshot(snapshots)
|
||||
papers, nonpapers, other = registry_papers(items)
|
||||
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||
for name, body in files.items():
|
||||
(evidence_dir/name).write_bytes(body)
|
||||
rows = []
|
||||
for paper in papers:
|
||||
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||
'original_decision_signature_verified': False,
|
||||
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'icdm_program'
|
||||
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||
PAGES = {
|
||||
'detailed-sessions': PROGRAM,
|
||||
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, name):
|
||||
return name in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def field(node, name):
|
||||
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||
raise ValueError('Unsafe source manifest')
|
||||
raw = manifest.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Frozen manifest hash changed')
|
||||
data = json.loads(raw)
|
||||
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||
raise ValueError('Wrong source edition or incomplete research run')
|
||||
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||
raise ValueError('Incomplete ICDM source set')
|
||||
files = {'manifest.json': raw}; indexed = {}
|
||||
for receipt in receipts:
|
||||
filename = receipt['file']; path = folder/filename
|
||||
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body)
|
||||
or not receipt['content_type'].lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Source receipt/body mismatch')
|
||||
if filename in files:
|
||||
raise ValueError('Repeated source filename')
|
||||
files[filename] = body; indexed[receipt['label']] = receipt
|
||||
return indexed, files
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8'))
|
||||
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||
text = clean(main.text())
|
||||
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||
raise ValueError('Wrong edition or unreviewed program scope')
|
||||
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||
raise ValueError('Incomplete or repeated session census')
|
||||
records = []; seen = set()
|
||||
for session in sessions:
|
||||
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||
form = session.attrs.get('data-format')
|
||||
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||
or form not in ('long', 'short')):
|
||||
raise ValueError('Invalid session ID, track or presentation label')
|
||||
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||
if field(session, 'sd-track') != label:
|
||||
raise ValueError('Visible track and source attribute disagree')
|
||||
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||
if count is None or count[1].lower() != form:
|
||||
raise ValueError('Visible presentation label disagrees')
|
||||
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||
if len(rows) != int(count[2]):
|
||||
raise ValueError('Session paper count changed')
|
||||
note = field(session, 'sd-table-note')
|
||||
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||
raise ValueError('Unreviewed presentation-note semantics')
|
||||
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||
'title': field(session, 'sd-session-title'),
|
||||
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||
for row in rows:
|
||||
pid = row.attrs['data-paper-id']
|
||||
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||
or field(row, 'sd-paper-id') != pid):
|
||||
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||
seen.add(pid)
|
||||
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||
authors = [name.strip() for name in authors_literal.split(';')]
|
||||
if not title or any(not name for name in authors):
|
||||
raise ValueError('Missing program title or author')
|
||||
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||
'session': context.copy(), 'source_position': len(records)+1})
|
||||
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||
return records
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
receipts, files = load_snapshot(snapshots)
|
||||
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in files.items():
|
||||
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||
outputs = []
|
||||
for track in ('research', 'applied'):
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['session']['track_literal'] != track:
|
||||
continue
|
||||
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||
'evidence_url': url,
|
||||
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||
'paper_type': None,
|
||||
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||
temporary.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True, type=Path)
|
||||
parser.add_argument('--out', required=True, type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'ismir_program'
|
||||
TRACK = 'program_unclassified'
|
||||
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||
SNAPSHOTS = {
|
||||
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||
'research_completed_not_coverage_or_import_approval', {
|
||||
'ismir-accepted.html': ACCEPTED,
|
||||
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||
}
|
||||
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def title_key(text):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def load_snapshots(root):
|
||||
root = Path(root)
|
||||
if root.is_symlink():
|
||||
raise ValueError('Unsafe snapshot root')
|
||||
result = {}
|
||||
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||
directory = root/name; path = directory/'manifest.json'
|
||||
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != expected:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||
raise ValueError('Unreviewed snapshot scope')
|
||||
files = {'manifest.json': raw}; receipts = {}
|
||||
for filename, url in urls.items():
|
||||
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||
path = directory/filename
|
||||
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe snapshot source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Source receipt/body disagree')
|
||||
files[filename] = body; receipts[filename] = receipt
|
||||
result[name] = (files, receipts)
|
||||
return result
|
||||
|
||||
|
||||
def author_names(literal):
|
||||
# Institution names can contain semicolons; split only outside parentheses.
|
||||
parts = []; depth = 0; start = 0
|
||||
for index, char in enumerate(literal):
|
||||
if char == '(': depth += 1
|
||||
elif char == ')':
|
||||
depth -= 1
|
||||
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||
elif char == ';' and depth == 0:
|
||||
parts.append(literal[start:index]); start = index+1
|
||||
if depth:
|
||||
raise ValueError('Unbalanced author affiliation')
|
||||
parts.append(literal[start:])
|
||||
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||
if any(not name for name in names):
|
||||
raise ValueError('Missing accepted-list author')
|
||||
return names
|
||||
|
||||
|
||||
def accepted_listing(raw):
|
||||
doc = Document(raw.decode('utf-8')); tables = []
|
||||
for table in doc.root.walk():
|
||||
if table.tag != 'table': continue
|
||||
rows = []
|
||||
for row in table.walk():
|
||||
if row.tag == 'tr':
|
||||
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||
tables.append(rows)
|
||||
rows = one(tables)[1:]; result = {}
|
||||
if len(rows) != 140:
|
||||
raise ValueError('Incomplete official accepted-list census')
|
||||
for row in rows:
|
||||
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||
raise ValueError('Missing or ambiguous accepted-list title')
|
||||
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||
return result
|
||||
|
||||
|
||||
def program_observations(rows, listing):
|
||||
if not isinstance(rows, list) or len(rows) != 140:
|
||||
raise ValueError('Incomplete public program census')
|
||||
observations = []; seen = set(); used = set()
|
||||
for original in rows:
|
||||
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||
raise ValueError('Invalid public program record')
|
||||
pid = original.get('id'); content = original.get('content', {})
|
||||
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||
raise ValueError('Missing program title, author sequence or abstract')
|
||||
key = title_key(title)
|
||||
if key not in listing or key in used:
|
||||
raise ValueError('Program/accepted-list title membership mismatch')
|
||||
used.add(key); accepted = listing[key]
|
||||
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||
session = content.get('session'); reviews = original.get('reviews')
|
||||
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||
or any(not isinstance(v, str) for v in reviews.values())
|
||||
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||
raise ValueError('Unreviewed opt-in review structure')
|
||||
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||
raise ValueError('Review publication preference/body disagree')
|
||||
flags = []
|
||||
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||
raise ValueError('Unreviewed source-version or opt-in census')
|
||||
return observations
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
sources = load_snapshots(snapshots)
|
||||
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||
raise ValueError('Unreviewed public program data path or poster link template')
|
||||
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
for name, (files, _) in sources.items():
|
||||
directory = evidence/name; directory.mkdir()
|
||||
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||
rows = []
|
||||
for observation in observations:
|
||||
original = observation['original']; content = original['content']; pid = original['id']
|
||||
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||
'original_decision_signature_verified':False,'paper_type':None,
|
||||
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
SOURCE = 'mm_program'
|
||||
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def safe_read(path, limit):
|
||||
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||
raise ValueError('Unsafe snapshot path or size')
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
directory = Path(root)/SNAPSHOT
|
||||
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||
raise ValueError('Unreviewed manifest scope')
|
||||
evidence = {'manifest.json':raw}
|
||||
for receipt in manifest['entries']:
|
||||
name = receipt['file']
|
||||
if Path(name).name != name or name in evidence:
|
||||
raise ValueError('Unsafe or duplicate receipt name')
|
||||
body = safe_read(directory/name, 12*1024**2)
|
||||
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||
or receipt['url'] != receipt['final_url']
|
||||
or not receipt['content_type'].startswith('application/json')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
if name == 'program-data.json':
|
||||
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||
raise ValueError('Wrong program source')
|
||||
elif receipt['http_status'] != 403:
|
||||
raise ValueError('Unreviewed public-access result')
|
||||
evidence[name] = body
|
||||
if 'program-data.json' not in evidence:
|
||||
raise ValueError('Missing program source')
|
||||
return evidence
|
||||
|
||||
|
||||
def without_emails(value):
|
||||
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||
if isinstance(value, dict):
|
||||
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||
if isinstance(value, list):
|
||||
return [without_emails(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def program_locations(node, pointer=''):
|
||||
"""Independent structural guard against silently skipping a new nesting form."""
|
||||
if isinstance(node, dict):
|
||||
if 'paper_id' in node:
|
||||
yield pointer
|
||||
for key,value in node.items():
|
||||
escaped = key.replace('~','~0').replace('/','~1')
|
||||
yield from program_locations(value, pointer+'/'+escaped)
|
||||
elif isinstance(node, list):
|
||||
for i,value in enumerate(node):
|
||||
yield from program_locations(value, pointer+'/'+str(i))
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
data = json.loads(raw)
|
||||
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||
or len(data.get('entries',[])) != 157):
|
||||
raise ValueError('Unreviewed program edition or census')
|
||||
records = {}; sessions = []; seen = []
|
||||
|
||||
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||
raise ValueError('Unreviewed program fields')
|
||||
for key in ('paper_id','title','abstract','track'):
|
||||
if not isinstance(paper[key],str):
|
||||
raise ValueError('Wrong program field type')
|
||||
for key in ('authors','affiliations'):
|
||||
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||
raise ValueError('Wrong author or affiliation list')
|
||||
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||
raise ValueError('Missing identity/title or unreviewed source track')
|
||||
core = {k:paper[k] for k in CORE}
|
||||
pid = paper['paper_id']; seen.append(pointer)
|
||||
if pid not in records:
|
||||
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||
'observed_status':'listed_in_official_program',
|
||||
'publication_identity_verified':False,'final_decision_verified':False,
|
||||
'approved_for_import':False,'missing_fields':missing,
|
||||
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||
'contexts':[]}
|
||||
elif {k:records[pid][k] for k in CORE} != core:
|
||||
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||
'subsession_pointer':subsession_pointer,
|
||||
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||
|
||||
for i,entry in enumerate(data['entries']):
|
||||
pointer = '/entries/'+str(i)
|
||||
if entry['track'] != '':
|
||||
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||
session.update(source_pointer=pointer,subsessions=[])
|
||||
for j,paper in enumerate(entry.get('papers') or []):
|
||||
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||
sp = pointer+'/subsessions/'+str(j)
|
||||
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||
for k,item in enumerate(sub.get('items') or []):
|
||||
ip = sp+'/items/'+str(k)
|
||||
if 'paper_id' in item:
|
||||
observe(item,ip,pointer,sp)
|
||||
else:
|
||||
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||
session['subsessions'].append(saved)
|
||||
sessions.append(session)
|
||||
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||
raise ValueError('Unhandled or duplicated program location')
|
||||
items = list(records.values())
|
||||
if (len(seen) != 4326 or len(items) != 2143
|
||||
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||
or sum(not r['abstract'] for r in items) != 216
|
||||
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||
raise ValueError('Program count or missing-field census changed')
|
||||
return items,sessions
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists() or out.is_symlink():
|
||||
raise ValueError('Output exists; use a fresh directory')
|
||||
if any(p.is_symlink() for p in out.parents):
|
||||
raise ValueError('Unsafe output parent')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
evidence = load_snapshot(snapshots)
|
||||
rows,sessions = parse_program(evidence['program-data.json'])
|
||||
(out/'evidence').mkdir()
|
||||
for name,body in evidence.items():
|
||||
(out/'evidence'/name).write_bytes(body)
|
||||
outputs = []
|
||||
for track,count in TRACK_COUNTS.items():
|
||||
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||
path.write_bytes(raw)
|
||||
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||
part = out/'collection-report.json.part'
|
||||
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--snapshots',required=True)
|
||||
parser.add_argument('--out',required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'recsys_program'
|
||||
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||
('doctoral_symposium', 'Doctoral', 10)]
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||
raise ValueError('Unreviewed snapshot scope')
|
||||
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||
name = receipt['file']; path = directory/name
|
||||
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Unsafe snapshot body')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
return receipt, {'manifest.json': raw, name: body}, body
|
||||
|
||||
|
||||
def body_text(node, doi_url):
|
||||
if not isinstance(node, Node):
|
||||
return node
|
||||
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||
return ''
|
||||
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||
raise ValueError('Wrong conference edition')
|
||||
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||
raise ValueError('Unreviewed track menu')
|
||||
observations = []; seen = set()
|
||||
for tab, (track, label, count) in enumerate(TRACKS):
|
||||
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||
if len(rows) != count:
|
||||
raise ValueError('Incomplete reviewed track census')
|
||||
for position, row in enumerate(rows, 1):
|
||||
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||
raise ValueError('Missing or repeated program locator')
|
||||
seen.add(locator)
|
||||
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||
if (type_literal,abbreviation) != expected_type:
|
||||
raise ValueError('Unreviewed source contribution type')
|
||||
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||
if not authors_literal.startswith('by '):
|
||||
raise ValueError('Missing program author prefix')
|
||||
authors_literal = authors_literal[3:]
|
||||
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||
if not title or any(not n or '@' in n for n in authors):
|
||||
raise ValueError('Missing title or program author')
|
||||
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||
raise ValueError('Wrong publication DOI family')
|
||||
abstract = clean(body_text(body, doi_url))
|
||||
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||
'track': track, 'track_label': label})
|
||||
counts = Counter(r['doi'] for r in observations)
|
||||
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||
raise ValueError('Unreviewed DOI observation census')
|
||||
for item in observations:
|
||||
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||
return observations
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
for name, raw in files.items():
|
||||
(evidence/name).write_bytes(raw)
|
||||
outputs = []
|
||||
for track,label,count in TRACKS:
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['track'] != track:
|
||||
continue
|
||||
doi = item['doi']; url = URL+'#'+item['locator']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||
'original_decision_signature_verified':False,
|
||||
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def checked_html(raw, receipt, url, limit):
|
||||
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||
or receipt.get('sha256') != digest(raw)
|
||||
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||
raise ValueError('Access challenge; stop without bypass')
|
||||
return raw, receipt
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
class PublicHTMLCache:
|
||||
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||
self.root = Path(directory)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
if self.root.is_symlink():
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
self.origin = origin
|
||||
self.limit = limit
|
||||
self.delay = delay
|
||||
self.opener = urllib.request.build_opener(NoRedirect())
|
||||
|
||||
def paths(self, url):
|
||||
u = urlsplit(url)
|
||||
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||
raise ValueError('Unapproved public metadata origin')
|
||||
key = digest(url.encode())
|
||||
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||
|
||||
def cached(self, url):
|
||||
body, receipt, failure = self.paths(url)
|
||||
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
if any(p.exists() for p in parts):
|
||||
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||
if failure.exists():
|
||||
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||
if body.exists() != receipt.exists():
|
||||
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||
if not body.exists():
|
||||
return None
|
||||
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||
raise ValueError('Oversized cache entry')
|
||||
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||
|
||||
def get(self, url):
|
||||
cached = self.cached(url)
|
||||
if cached is not None:
|
||||
return cached
|
||||
body, receipt, failure = self.paths(url)
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||
with self.opener.open(request, timeout=30) as response:
|
||||
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||
'content_type': response.headers.get('Content-Type', '')}
|
||||
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||
raw = response.read(self.limit+1)
|
||||
if len(raw) > self.limit:
|
||||
raise ValueError('Metadata response exceeds size limit')
|
||||
expected = response.headers.get('Content-Length')
|
||||
if expected is not None and int(expected) != len(raw):
|
||||
raise ConnectionError('Incomplete metadata response')
|
||||
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||
checked_html(raw, rec, url, self.limit)
|
||||
except (urllib.error.HTTPError, ValueError) as error:
|
||||
if isinstance(error, urllib.error.HTTPError):
|
||||
status = error.code
|
||||
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||
error.close()
|
||||
else:
|
||||
status = None
|
||||
retry_after = None
|
||||
# Access errors remain sticky until the operator verifies availability.
|
||||
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||
raise
|
||||
part = body.with_suffix('.html.part')
|
||||
part.write_bytes(raw); part.replace(body)
|
||||
part = receipt.with_suffix('.json.part')
|
||||
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||
time.sleep(self.delay)
|
||||
return raw, rec
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,139 @@
|
||||
import copy
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import eccv2026 as c
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||
|
||||
|
||||
class ECCV2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||
|
||||
def snapshots(self, root):
|
||||
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||
for name, raw in self.bundle['official_pages'].items():
|
||||
(official/name).write_bytes(raw.encode('utf-8'))
|
||||
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||
for pid, raw in self.bundle['paper_pages'].items():
|
||||
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||
return official, paper
|
||||
|
||||
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||
self.assertEqual(len(self.program), 2834)
|
||||
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||
item = self.program[gold['paper_id']]
|
||||
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||
self.assertIn(ordinal, item['row_ordinals'])
|
||||
|
||||
def test_conflicting_duplicate_not_silently_folded(self):
|
||||
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||
|
||||
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||
html = self.bundle['official_pages']['accepted.html']
|
||||
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_public_page_fields_match_independent_sample_review(self):
|
||||
for gold in self.bundle['paper_gold']['records']:
|
||||
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||
|
||||
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_paper(changed, url, self.program['4048'])
|
||||
|
||||
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||
|
||||
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root)
|
||||
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
|
||||
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||
for r in self.bundle['paper_manifest']['records']:
|
||||
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||
|
||||
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_structured_math_markup_requires_explicit_review(self):
|
||||
raw = self.bundle['paper_pages']['4048']
|
||||
self.assertIn('<p>Proactive tamper', raw)
|
||||
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||
|
||||
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||
import hashlib
|
||||
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||
self.assertEqual(len(raw), receipt['bytes'])
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
for receipt in self.bundle['paper_manifest']['records']:
|
||||
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
|
||||
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||
import hashlib
|
||||
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
# Real changes to the author or an extra encoding layer remain errors.
|
||||
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||
|
||||
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||
import hashlib
|
||||
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertIn('&', fields['title'])
|
||||
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
@@ -0,0 +1,90 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
import urllib.error
|
||||
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||
|
||||
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||
|
||||
|
||||
class PublicHTMLCacheTests(unittest.TestCase):
|
||||
def receipt(self, raw):
|
||||
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||
|
||||
def test_old_cache_reused_without_network_or_expiry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||
|
||||
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||
for kind in ('hash','orphan','part'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||
c.opener = MagicMock()
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
c.opener.open.assert_not_called()
|
||||
|
||||
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||
self.assertEqual(c.opener.open.call_count,1)
|
||||
|
||||
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||
for kind in ('redirect','type','challenge','size'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||
if kind=='size':r.read.return_value=b'x'*101
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
self.assertFalse(c.paths(URL)[0].exists())
|
||||
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||
|
||||
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||
|
||||
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
|
||||
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = URL
|
||||
raw = b'<html>public metadata</html>'
|
||||
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||
self.assertEqual(c.opener.open.call_count, 1)
|
||||
body, receipt, _ = c.paths(URL)
|
||||
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||
|
||||
def test_changed_url_is_rejected_before_reading_body(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||
response.headers = {'Content-Type':'text/html'}
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
response.read.assert_not_called()
|
||||
@@ -0,0 +1,95 @@
|
||||
import base64
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icassp2026 as c
|
||||
|
||||
|
||||
class ICASSP2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||
cls.items = []
|
||||
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||
for receipt in cls.manifest['pages']:
|
||||
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||
|
||||
def materialize(self, root):
|
||||
for name, body in self.files.items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||
|
||||
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||
report = c.collect(root/'out', snapshots=src)
|
||||
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||
rows = [json.loads(l) for l in raw.splitlines()]
|
||||
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
self.assertEqual(actual,gold)
|
||||
original = {r['DOI']:r for r,_ in self.items}
|
||||
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||
for r in rows:
|
||||
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
evidence = r['metadata']['source_evidence']
|
||||
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||
c.collect(root/'out',snapshots=src)
|
||||
|
||||
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||
with self.subTest(kind=kind):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||
if kind=='container': r['container-title']=['Foreign']
|
||||
if kind=='event': r['event']['name']='Other'
|
||||
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||
if kind=='author': r['author']=[]
|
||||
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||
if kind=='abstract': r['abstract']='new source version'
|
||||
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||
|
||||
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||
for kind in ('title','page','author'):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||
if kind=='title': r['title']=['A research paper']
|
||||
if kind=='page': r['page']='1-5'
|
||||
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||
|
||||
def test_duplicate_and_incomplete_census_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||
|
||||
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||
if kind=='missing': path.unlink()
|
||||
else: path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_page_and_snapshot_rejected(self):
|
||||
for kind in ('page','directory'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||
@@ -0,0 +1,113 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icdm2026 as c
|
||||
|
||||
|
||||
class ICDM2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||
cls.gold = cls.fixture['independent_inventory']
|
||||
|
||||
def snapshots(self, root):
|
||||
for name, text in self.fixture['files'].items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||
|
||||
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||
result = c.parse_program(self.raw)
|
||||
self.assertEqual(len(result), len(self.gold['entries']))
|
||||
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||
self.assertEqual(row[field], gold[field])
|
||||
self.assertEqual(row['source_position'], position)
|
||||
for field, value in gold['session'].items():
|
||||
self.assertEqual(row['session'][field], value)
|
||||
|
||||
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
report = c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||
all_rows = []
|
||||
for manifest in report['files']:
|
||||
path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||
self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(line) for line in raw.splitlines()]
|
||||
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||
for row in rows:
|
||||
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||
all_rows.extend(rows)
|
||||
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||
for item in report['evidence']:
|
||||
raw = (root/'out'/item['output_file']).read_bytes()
|
||||
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
|
||||
def test_stale_or_different_conference_rejected(self):
|
||||
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_track_attribute_must_agree_with_visible_label(self):
|
||||
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'track'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new, 1))
|
||||
|
||||
def test_empty_author_is_not_silently_dropped(self):
|
||||
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'author'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertFalse((root/'out').exists())
|
||||
|
||||
def test_snapshot_source_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
@@ -0,0 +1,92 @@
|
||||
import base64
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import ismir2026 as c
|
||||
|
||||
|
||||
class ISMIR2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||
|
||||
def materialize(self,root):
|
||||
for name,raw in self.files.items():
|
||||
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||
|
||||
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||
self.assertEqual(len(observations),140)
|
||||
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||
|
||||
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||
|
||||
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||
listing=c.accepted_listing(self.accepted)
|
||||
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||
rows=copy.deepcopy(self.program)
|
||||
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||
if kind=='missing_row':rows.pop()
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||
|
||||
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||
listing=c.accepted_listing(self.accepted)
|
||||
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||
|
||||
def test_source_optin_no_cannot_publish_review_text(self):
|
||||
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||
|
||||
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||
with self.assertRaises(ValueError):c.author_names(raw)
|
||||
|
||||
def test_ambiguous_official_title_is_not_fused(self):
|
||||
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||
|
||||
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||
for kind in ('manifest','body','missing','symlink'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||
if kind=='missing':path.unlink()
|
||||
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
@@ -0,0 +1,146 @@
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import mm2026 as c
|
||||
|
||||
|
||||
class MM2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||
cls.data = json.loads(cls.raw)
|
||||
cls.gold = cls.fixture['independent_inventory']
|
||||
|
||||
def materialize(self, root):
|
||||
for name,body in self.files.items():
|
||||
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||
|
||||
def parse_changed(self, data):
|
||||
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||
|
||||
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||
rows,sessions = c.parse_program(self.raw)
|
||||
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||
actual = {r['paper_id']:r for r in rows}
|
||||
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||
for gold in self.gold['inventory']:
|
||||
r = actual[gold['paper_id']]
|
||||
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||
|
||||
def test_every_presentation_and_session_context_is_preserved(self):
|
||||
rows,sessions = c.parse_program(self.raw)
|
||||
for r in rows:
|
||||
for context in r['contexts']:
|
||||
node = self.data
|
||||
for key in context['source_pointer'].split('/')[1:]:
|
||||
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||
for i,session in enumerate(sessions):
|
||||
original = self.data['entries'][i]
|
||||
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||
self.assertEqual(session['track'],'')
|
||||
for key in original.keys()-{'papers','subsessions'}:
|
||||
self.assertEqual(session[key],original[key])
|
||||
for j,sub in enumerate(original.get('subsessions') or []):
|
||||
saved = session['subsessions'][j]
|
||||
self.assertEqual(saved['title'],sub['title'])
|
||||
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||
|
||||
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||
rows,_ = c.parse_program(self.raw)
|
||||
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||
for r in rows:
|
||||
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||
self.assertFalse(r['publication_identity_verified'])
|
||||
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||
missing = [r for r in rows if not r['authors']]
|
||||
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||
|
||||
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'src')
|
||||
report=c.collect(root/'out',snapshots=root/'src')
|
||||
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||
for name,body in self.files.items():
|
||||
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||
n=0
|
||||
for item in report['files']:
|
||||
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||
self.assertEqual(n,2143)
|
||||
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||
|
||||
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||
data=copy.deepcopy(self.data)
|
||||
first=data['entries'][0]['papers'][0]
|
||||
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||
|
||||
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||
data=copy.deepcopy(self.data)
|
||||
moved=data['entries'][0]['papers'].pop()
|
||||
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||
|
||||
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||
elif kind=='paper_track':paper['track']='New Track'
|
||||
elif kind=='field':paper['doi']='unreviewed'
|
||||
elif kind=='title':paper['title']=''
|
||||
else:paper['authors']='Unparsed author string'
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||
|
||||
def test_edition_or_census_change_requires_review(self):
|
||||
for kind in ('edition','count'):
|
||||
data=copy.deepcopy(self.data)
|
||||
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||
else:data['entries'][0]['papers'].pop()
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||
|
||||
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||
for kind in ('manifest','program','probe','missing','symlink'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||
path=directory/name
|
||||
if kind=='missing':path.unlink()
|
||||
elif kind=='symlink':
|
||||
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||
rows,sessions=c.parse_program(self.raw)
|
||||
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||
@@ -0,0 +1,101 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import mm2026
|
||||
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||
|
||||
|
||||
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
src = self.root/'source'
|
||||
for name,body in self.fixture['files'].items():
|
||||
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||
mm2026.collect(self.root/'program', snapshots=src)
|
||||
self.report_path = self.root/'program/collection-report.json'
|
||||
|
||||
def change_report(self, change):
|
||||
report=json.loads(self.report_path.read_text());change(report)
|
||||
self.report_path.write_text(json.dumps(report))
|
||||
|
||||
def assert_rejected_before_database(self):
|
||||
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||
with self.assertRaises(ValueError):
|
||||
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||
connection.assert_not_called()
|
||||
self.assertFalse((self.root/'audit').exists())
|
||||
|
||||
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||
actual={r['program_id']:r for r in rows}
|
||||
for gold in self.fixture['independent_inventory']['inventory']:
|
||||
r=actual[gold['paper_id']]
|
||||
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||
(gold['title'],gold['authors'],gold['track_literal']))
|
||||
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||
self.assertFalse(r['contribution_type_verified'])
|
||||
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||
|
||||
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||
self.assertEqual(len(rows),2143)
|
||||
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||
|
||||
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||
report=json.loads(self.report_path.read_text())
|
||||
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||
|
||||
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
self.assert_rejected_before_database()
|
||||
|
||||
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||
|
||||
def test_missing_or_repeated_track_rejected(self):
|
||||
original=self.report_path.read_bytes()
|
||||
for kind in ('missing','repeated'):
|
||||
self.report_path.write_bytes(original)
|
||||
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||
|
||||
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||
self.assert_rejected_before_database()
|
||||
|
||||
def test_mixed_input_modes_rejected_before_connection(self):
|
||||
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||
connection.assert_not_called()
|
||||
@@ -0,0 +1,79 @@
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import recsys2026 as c
|
||||
|
||||
|
||||
class RecSys2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||
|
||||
def materialize(self,root):
|
||||
for name,raw in self.files.items():
|
||||
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||
|
||||
def test_all263_observations_against_independent_gold(self):
|
||||
items=c.parse_program(self.body)
|
||||
keys=self.fixture['independent_gold'][0].keys()
|
||||
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||
|
||||
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||
report=c.collect(out,snapshots=src)
|
||||
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||
for manifest in report['files']:
|
||||
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||
for r in rows:
|
||||
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||
for key in expected:self.assertEqual(item[key],expected[key])
|
||||
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||
|
||||
def test_edition_and_track_label_drift_rejected(self):
|
||||
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||
|
||||
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||
if kind=='missing':path.unlink()
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_source_and_root_rejected(self):
|
||||
for kind in ('source','root'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||
Reference in New Issue
Block a user