Compare commits
23 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| afb3b939ca | |||
| 6961710e96 | |||
| f4be9de547 | |||
| 33bb378369 | |||
| 5bdf9f0fe5 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 564821d883 | |||
| 2f1b5c959a | |||
| f139c7e49d | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| 06467835c7 | |||
| f2753bc741 | |||
| 1ea17c810d | |||
| f800c6ecb4 | |||
| 28255743ef | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 | |||
| 59053fd4fb |
@@ -12,7 +12,7 @@ jobs:
|
||||
- name: Verify metadata storage and query commands
|
||||
run: |
|
||||
python3 -B - <<'PY'
|
||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
sha = os.environ['ARCHIVE_SHA']
|
||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||
raise SystemExit('Invalid source commit')
|
||||
@@ -49,6 +49,82 @@ jobs:
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
eccv_official = root/'eccv2026-official'
|
||||
eccv_papers = root/'eccv2026-paper-samples'
|
||||
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||
for name, html in eccv_fixture['official_pages'].items():
|
||||
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||
for pid, html in eccv_fixture['paper_pages'].items():
|
||||
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||
eccv_encoding = eccv_fixture['encoding_sample']
|
||||
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||
eccv_literal = eccv_fixture['literal_title_sample']
|
||||
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
icassp_snapshots = root/'icassp2026-snapshots'
|
||||
icassp_snapshots.mkdir()
|
||||
for name, encoded in icassp_fixture['files'].items():
|
||||
target = icassp_snapshots/name
|
||||
target.parent.mkdir(parents=True,exist_ok=True)
|
||||
target.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||
assert icassp_report['single_name_author_fields'] == 23
|
||||
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
target = sdm_snapshots/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(text.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
||||
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
||||
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
||||
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
||||
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
||||
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
||||
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
||||
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
icdm_snapshots = root/'icdm2026-snapshots'
|
||||
for name, text in icdm_fixture['files'].items():
|
||||
target = icdm_snapshots/name
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(text.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
@@ -75,6 +151,21 @@ jobs:
|
||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
recsys_snapshots = root/'recsys2026-snapshots'
|
||||
recsys_snapshots.mkdir()
|
||||
for name,encoded in recsys_fixture['files'].items():
|
||||
path = recsys_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(base64.b64decode(encoded))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
corl_official = root/'corl2026-official'
|
||||
corl_archive = root/'corl2026-archive'
|
||||
|
||||
@@ -128,3 +128,13 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
||||
### CoRL 2026 官方程序快照
|
||||
|
||||
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||
|
||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||
|
||||
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# 会议覆盖与公开决定核查
|
||||
|
||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||
|
||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||
|
||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||
@@ -187,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
||||
|
||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||
|
||||
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||
|
||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||
|
||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||
@@ -196,3 +200,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||
|
||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||
|
||||
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# 2020 起补漏状态:2026-11-17
|
||||
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
|
||||
## 最新数据库盘点
|
||||
|
||||
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||
|
||||
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||
| --- | --- |
|
||||
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||
|
||||
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||
|
||||
## 已独立接受的近期补录
|
||||
|
||||
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||
|
||||
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||
|
||||
## 已采集、尚待关联或规范化的来源
|
||||
|
||||
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||
| --- | --- | --- |
|
||||
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
||||
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||
|
||||
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||
|
||||
## 公开拒稿与后续工作
|
||||
|
||||
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||
|
||||
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||
|
||||
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||
@@ -0,0 +1,35 @@
|
||||
# ECCV 2026:暂定节目与公开元数据
|
||||
|
||||
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||
|
||||
```sh
|
||||
paper-atlas collect eccv2026-program \
|
||||
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||
--max-new-pages 200 \
|
||||
--out /data/paper-index/collected/eccv2026-pass01
|
||||
```
|
||||
|
||||
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||
|
||||
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||
|
||||
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||
|
||||
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||
|
||||
## 来源边界
|
||||
|
||||
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||
|
||||
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||
|
||||
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||
|
||||
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||
|
||||
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||
|
||||
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||
|
||||
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||
@@ -0,0 +1,59 @@
|
||||
# ECCV 2026 来源核查记录
|
||||
|
||||
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||
|
||||
## 贡献类型与 track
|
||||
|
||||
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||
|
||||
| 官方 group 字段值 | 含义摘要 |
|
||||
| --- | --- |
|
||||
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||
| Concept & Feasibility | 新概念及可行性证据 |
|
||||
|
||||
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||
|
||||
## 公开负面决定元数据
|
||||
|
||||
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||
|
||||
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||
|
||||
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||
|
||||
## 可恢复证据与后续入口
|
||||
|
||||
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||
|
||||
| 文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||
|
||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||
|
||||
|
||||
## 2026-11-17 全量采集与只读预检
|
||||
|
||||
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||
|
||||
| 服务器证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||
|
||||
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||
|
||||
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||
|
||||
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||
@@ -0,0 +1,40 @@
|
||||
# ICASSP 2026 注册元数据采集
|
||||
|
||||
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||
|
||||
## 输入与使用
|
||||
|
||||
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||
|
||||
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||
|
||||
```sh
|
||||
paper-atlas collect icassp2026-registry \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||
|
||||
## 数据范围
|
||||
|
||||
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||
|
||||
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||
|
||||
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||
|
||||
## 证据与恢复
|
||||
|
||||
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||
|
||||
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||
|
||||
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||
|
||||
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||
@@ -0,0 +1,46 @@
|
||||
# ICDM2026 冻结日程适配器
|
||||
|
||||
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||
|
||||
```sh
|
||||
paper-atlas collect icdm2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||
```
|
||||
|
||||
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||
|
||||
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||
|
||||
## 两个明确来源轨道
|
||||
|
||||
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||
|
||||
| 输出文件 | 官方明确标签 | 记录数 |
|
||||
| --- | --- | ---: |
|
||||
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||
|
||||
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||
|
||||
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||
|
||||
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||
|
||||
## 始终保留范围限制
|
||||
|
||||
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||
|
||||
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||
|
||||
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||
|
||||
## 验证与恢复
|
||||
|
||||
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||
|
||||
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||
|
||||
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||
|
||||
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||
@@ -0,0 +1,76 @@
|
||||
# ACM MM 2026 官方节目来源状态
|
||||
|
||||
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||
|
||||
## 已取得的数据
|
||||
|
||||
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||
|
||||
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||
|
||||
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||
|
||||
| 来源 track 原文 | 不同节目 ID 数 |
|
||||
| --- | ---: |
|
||||
| MRAC | 35 |
|
||||
| SUMAC | 9 |
|
||||
| NeuroMM | 10 |
|
||||
| IMPACT | 8 |
|
||||
| RichMediaGAI | 10 |
|
||||
| MMSports | 16 |
|
||||
| MUWS | 5 |
|
||||
| EReL-MIR | 7 |
|
||||
| GreenMM | 10 |
|
||||
| DFF | 12 |
|
||||
| Main Track | 1703 |
|
||||
| Grand Challenge | 97 |
|
||||
| Doctoral Symposium | 4 |
|
||||
| Datasets | 50 |
|
||||
| Interactive Art | 23 |
|
||||
| Reproducibility | 1 |
|
||||
| Demo and Video | 27 |
|
||||
| Open Source | 14 |
|
||||
| Industry Talk | 7 |
|
||||
| BNI | 25 |
|
||||
| QoEVMA | 4 |
|
||||
| HUMA | 14 |
|
||||
| WIPAI | 8 |
|
||||
| AMI | 11 |
|
||||
| Mulsemedia | 9 |
|
||||
| UAVM | 11 |
|
||||
| MCHM | 7 |
|
||||
| JAV-CG | 6 |
|
||||
|
||||
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||
|
||||
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||
|
||||
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||
|
||||
## 公开负面决定的访问边界
|
||||
|
||||
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||
|
||||
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||
|
||||
## 产物与恢复
|
||||
|
||||
原始目录(均相对 `/data/paper-index/collected/`):
|
||||
|
||||
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||
|
||||
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||
|
||||
| 证据 | SHA256 |
|
||||
| --- | --- |
|
||||
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||
|
||||
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||
|
||||
本批尚未形成生产导入队列。下一步需要可复用采集器保留每个节目 ID 的全部展示上下文、明确贡献类型与来源边界,单列缺失字段,再做作品/出版身份及既往投稿审查。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||
@@ -0,0 +1,83 @@
|
||||
# RecSys 2026 公开录用贡献采集
|
||||
|
||||
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||
|
||||
## 使用
|
||||
|
||||
```sh
|
||||
paper-atlas collect recsys2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||
```
|
||||
|
||||
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||
|
||||
## 数据范围与重复项
|
||||
|
||||
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||
|
||||
| track | 观察数 |
|
||||
| --- | ---: |
|
||||
| long | 65 |
|
||||
| short | 29 |
|
||||
| past_present_future | 13 |
|
||||
| reproducibility | 4 |
|
||||
| resource | 7 |
|
||||
| industry | 83 |
|
||||
| research_practice_notes | 34 |
|
||||
| demo | 18 |
|
||||
| doctoral_symposium | 10 |
|
||||
|
||||
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||
|
||||
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||
|
||||
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||
|
||||
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||
|
||||
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||
|
||||
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||
|
||||
## 输出与恢复
|
||||
|
||||
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||
|
||||
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||
|
||||
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||
|
||||
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||
|
||||
## 2026-11-17 PostgreSQL 补录验收
|
||||
|
||||
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||
|
||||
| track | 已导入观察 / 来源应有观察 |
|
||||
| --- | ---: |
|
||||
| demo | 1 / 18 |
|
||||
| doctoral_symposium | 2 / 10 |
|
||||
| industry | 18 / 83 |
|
||||
| long | 13 / 65 |
|
||||
| past_present_future | 2 / 13 |
|
||||
| reproducibility | 1 / 4 |
|
||||
| research_practice_notes | 4 / 34 |
|
||||
| resource | 2 / 7 |
|
||||
| short | 4 / 29 |
|
||||
|
||||
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||
|
||||
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||
|
||||
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||
|
||||
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||
|
||||
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||
|
||||
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||
@@ -0,0 +1,97 @@
|
||||
# SDM 2026:生产规范化与身份核查记录
|
||||
|
||||
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||
|
||||
## 已核实的生产输出
|
||||
|
||||
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||
|
||||
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||
|
||||
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||
|
||||
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||
|
||||
| 证据文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||
|
||||
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||
|
||||
## 当前数据库只读预检
|
||||
|
||||
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||
|
||||
| 候选类别 | 记录数 | 含义 |
|
||||
| --- | ---: | --- |
|
||||
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||
|
||||
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||
|
||||
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||
|
||||
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||
| --- | ---: | --- |
|
||||
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||
|
||||
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||
|
||||
## 需要保护的较早拒稿历史
|
||||
|
||||
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||
| --- | ---: | --- |
|
||||
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||
|
||||
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
|
||||
## 未精确匹配的补充检索
|
||||
|
||||
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||
|
||||
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||
|
||||
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||
|
||||
| 服务器恢复文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||
|
||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||
|
||||
## 25 条生产导入验收
|
||||
|
||||
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||
|
||||
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||
|
||||
| 生产证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||
|
||||
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||
|
||||
## 下次恢复
|
||||
|
||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||
@@ -0,0 +1,46 @@
|
||||
# SDM 2026:论文集与日程版本
|
||||
|
||||
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
|
||||
|
||||
```sh
|
||||
paper-atlas collect sdm2026-proceedings \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/sdm2026-normalized-pass01
|
||||
```
|
||||
|
||||
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
|
||||
|
||||
## 两种输出
|
||||
|
||||
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
|
||||
|
||||
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
|
||||
|
||||
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
|
||||
|
||||
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
|
||||
|
||||
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
|
||||
|
||||
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
|
||||
|
||||
## 公开拒稿与访问限制
|
||||
|
||||
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
|
||||
|
||||
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
|
||||
|
||||
## 来源恢复与后续核查
|
||||
|
||||
| 冻结目录 | manifest SHA-256 |
|
||||
| --- | --- |
|
||||
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
|
||||
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
|
||||
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
|
||||
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
|
||||
|
||||
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||
|
||||
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||
|
||||
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||
@@ -24,8 +24,13 @@ COMMANDS = {
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
from html import unescape
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||
|
||||
SOURCE = 'eccv_program'
|
||||
ORIGIN = 'https://eccv.ecva.net'
|
||||
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||
TRACK = 'program_unclassified'
|
||||
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected one reviewed metadata field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, value):
|
||||
return value in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||
raise ValueError('Unexpected ECCV program edition')
|
||||
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||
raise ValueError('Reviewed provisional-publication notice changed')
|
||||
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||
papers = {}; occurrences = 0
|
||||
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) != 3:
|
||||
raise ValueError('Unexpected program columns')
|
||||
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||
if not row['title'] or not authors:
|
||||
raise ValueError('Incomplete public program fields')
|
||||
occurrences += 1
|
||||
if pid in papers:
|
||||
if papers[pid]['listing'] != row:
|
||||
raise ValueError('Conflicting entries share a program ID')
|
||||
papers[pid]['row_ordinals'].append(occurrences)
|
||||
else:
|
||||
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||
raise ValueError('Incomplete reviewed program census')
|
||||
return papers
|
||||
|
||||
|
||||
def parse_paper(raw, url, program):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||
abstract = clean(abstract_node.text())
|
||||
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||
authors = ld.get('author')
|
||||
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||
or not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||
# Decode only to test agreement with visible text, never to rewrite it.
|
||||
def agrees_with_display(value, expected):
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
for _ in range(2):
|
||||
value = unescape(value)
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
return False
|
||||
|
||||
raw_names = [a['name'] for a in authors]
|
||||
names = [clean(name) for name in raw_names]
|
||||
if not agrees_with_display(ld['name'], title):
|
||||
raise ValueError('Public visible and structured titles disagree')
|
||||
if ' ⋅ '.join(names) != authors_literal:
|
||||
visible_names = authors_literal.split(' ⋅ ')
|
||||
if len(visible_names) != len(raw_names) or any(
|
||||
not agrees_with_display(raw_name, visible_name)
|
||||
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
names = visible_names
|
||||
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||
if raw_names != names or ld['name'] != title:
|
||||
flags.append('jsonld_display_text_normalized_requires_review')
|
||||
if title != program['listing']['title']:
|
||||
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||
if authors_literal != program['listing']['authors_literal']:
|
||||
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||
|
||||
|
||||
def official_snapshot(root):
|
||||
root = Path(root); manifest = root/'manifest.json'
|
||||
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed official snapshot manifest changed')
|
||||
m = json.loads(manifest.read_bytes())
|
||||
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||
raise ValueError('Official snapshot requires a timezone')
|
||||
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||
raise ValueError('Official source set changed')
|
||||
pages = {}; evidence = {}
|
||||
for r in m['sources']:
|
||||
name = r['file']; p = root/name
|
||||
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe official snapshot')
|
||||
raw = p.read_bytes()
|
||||
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||
raise ValueError('Invalid official source receipt')
|
||||
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||
return m, pages, evidence, program
|
||||
|
||||
|
||||
def paper_snapshots(root, program):
|
||||
root = Path(root); path = root/'manifest.json'
|
||||
if root.is_symlink() or path.is_symlink():
|
||||
raise ValueError('Paper snapshot symlinks rejected')
|
||||
if path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Oversized paper manifest')
|
||||
m = json.loads(path.read_bytes()); result = {}
|
||||
for r in m['records']:
|
||||
pid = r['paper_id']
|
||||
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||
raise ValueError('Foreign or repeated paper snapshot')
|
||||
p = root/r['file']
|
||||
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||
raise ValueError('Unsafe paper snapshot')
|
||||
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||
return result
|
||||
|
||||
|
||||
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||
if (paper_snapshots_dir is None) == (cache is None):
|
||||
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
lock = None
|
||||
try:
|
||||
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||
directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in pages.items():
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||
client = None
|
||||
if cache is not None:
|
||||
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||
raise ValueError('Cache lock symlinks rejected')
|
||||
lock = (cache/'.collector.lock').open('a+')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
client = PublicHTMLCache(cache, ORIGIN)
|
||||
rows = []; missing = []; fetched = 0; reused = 0
|
||||
for pid in sorted(program, key=int):
|
||||
p = program[pid]
|
||||
if snapshots is not None:
|
||||
response = snapshots.get(pid)
|
||||
else:
|
||||
response = client.cached(p['url'])
|
||||
if response is not None:
|
||||
reused += 1
|
||||
elif max_new_pages is None or fetched < max_new_pages:
|
||||
response = client.get(p['url']); fetched += 1
|
||||
if response is None:
|
||||
missing.append(pid); continue
|
||||
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||
keyword = p['listing']['keywords_literal']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||
'evidence_url': ACCEPTED,
|
||||
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||
if len(rows) % 10 == 0:
|
||||
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||
files = []
|
||||
if rows:
|
||||
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||
raise
|
||||
finally:
|
||||
if lock is not None:
|
||||
lock.close()
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--official-snapshots', required=True)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--paper-snapshots')
|
||||
group.add_argument('--cache')
|
||||
parser.add_argument('--max-new-pages', type=int)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import clean
|
||||
|
||||
SOURCE = 'crossref_icassp_registry'
|
||||
TRACK = 'proceedings_unclassified'
|
||||
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||
PREFIX = '10.1109/icassp55912.2026.'
|
||||
NONPAPERS = {
|
||||
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||
or len(manifest['pages']) != 10):
|
||||
raise ValueError('Incomplete frozen census receipt')
|
||||
files = {'manifest.json': raw}; rows = []
|
||||
for index, receipt in enumerate(manifest['pages'], 1):
|
||||
name = receipt['file']; path = directory/name
|
||||
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||
or path.stat().st_size > 4*1024**2):
|
||||
raise ValueError('Unsafe snapshot page')
|
||||
body = path.read_bytes()
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
data = json.loads(body); message = data['message']
|
||||
expected = 500 if index < 10 else 145
|
||||
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||
raise ValueError('Incomplete registry page')
|
||||
files[name] = body
|
||||
for original in message['items']:
|
||||
rows.append((original, receipt))
|
||||
return manifest, files, rows
|
||||
|
||||
|
||||
def registry_papers(rows):
|
||||
seen = set(); papers = []; nonpapers = []; other = []
|
||||
for original, receipt in rows:
|
||||
doi = original.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen:
|
||||
raise ValueError('Missing or repeated registry DOI')
|
||||
seen.add(doi)
|
||||
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||
if doi.startswith(PREFIX):
|
||||
raise ValueError('Conference DOI/container conflict')
|
||||
other.append(doi)
|
||||
continue
|
||||
if (original.get('container-title') != [CONTAINER]
|
||||
or original.get('event', {}).get('name') != CONTAINER
|
||||
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||
or original.get('URL') != 'https://doi.org/'+doi
|
||||
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||
if doi in NONPAPERS:
|
||||
title, page = NONPAPERS[doi]
|
||||
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||
raise ValueError('Changed reviewed nonpaper evidence')
|
||||
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||
continue
|
||||
titles = original.get('title')
|
||||
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||
raise ValueError('Missing registry title')
|
||||
authors = original.get('author')
|
||||
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||
raise ValueError('Missing authors or changed abstract availability')
|
||||
names = []; flags = []
|
||||
for position, author in enumerate(authors):
|
||||
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||
raise ValueError('Incomplete registry author or sequence')
|
||||
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||
raise ValueError('Unreviewed author identity')
|
||||
if not author.get('given'):
|
||||
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||
names.append(name)
|
||||
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||
raise ValueError('Incomplete reviewed registry partition')
|
||||
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
manifest, files, items = load_snapshot(snapshots)
|
||||
papers, nonpapers, other = registry_papers(items)
|
||||
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||
for name, body in files.items():
|
||||
(evidence_dir/name).write_bytes(body)
|
||||
rows = []
|
||||
for paper in papers:
|
||||
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||
'original_decision_signature_verified': False,
|
||||
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,191 @@
|
||||
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'icdm_program'
|
||||
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||
PAGES = {
|
||||
'detailed-sessions': PROGRAM,
|
||||
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, name):
|
||||
return name in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def field(node, name):
|
||||
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||
raise ValueError('Unsafe source manifest')
|
||||
raw = manifest.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Frozen manifest hash changed')
|
||||
data = json.loads(raw)
|
||||
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||
raise ValueError('Wrong source edition or incomplete research run')
|
||||
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||
raise ValueError('Incomplete ICDM source set')
|
||||
files = {'manifest.json': raw}; indexed = {}
|
||||
for receipt in receipts:
|
||||
filename = receipt['file']; path = folder/filename
|
||||
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body)
|
||||
or not receipt['content_type'].lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Source receipt/body mismatch')
|
||||
if filename in files:
|
||||
raise ValueError('Repeated source filename')
|
||||
files[filename] = body; indexed[receipt['label']] = receipt
|
||||
return indexed, files
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8'))
|
||||
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||
text = clean(main.text())
|
||||
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||
raise ValueError('Wrong edition or unreviewed program scope')
|
||||
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||
raise ValueError('Incomplete or repeated session census')
|
||||
records = []; seen = set()
|
||||
for session in sessions:
|
||||
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||
form = session.attrs.get('data-format')
|
||||
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||
or form not in ('long', 'short')):
|
||||
raise ValueError('Invalid session ID, track or presentation label')
|
||||
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||
if field(session, 'sd-track') != label:
|
||||
raise ValueError('Visible track and source attribute disagree')
|
||||
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||
if count is None or count[1].lower() != form:
|
||||
raise ValueError('Visible presentation label disagrees')
|
||||
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||
if len(rows) != int(count[2]):
|
||||
raise ValueError('Session paper count changed')
|
||||
note = field(session, 'sd-table-note')
|
||||
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||
raise ValueError('Unreviewed presentation-note semantics')
|
||||
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||
'title': field(session, 'sd-session-title'),
|
||||
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||
for row in rows:
|
||||
pid = row.attrs['data-paper-id']
|
||||
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||
or field(row, 'sd-paper-id') != pid):
|
||||
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||
seen.add(pid)
|
||||
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||
authors = [name.strip() for name in authors_literal.split(';')]
|
||||
if not title or any(not name for name in authors):
|
||||
raise ValueError('Missing program title or author')
|
||||
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||
'session': context.copy(), 'source_position': len(records)+1})
|
||||
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||
return records
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
receipts, files = load_snapshot(snapshots)
|
||||
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in files.items():
|
||||
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||
outputs = []
|
||||
for track in ('research', 'applied'):
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['session']['track_literal'] != track:
|
||||
continue
|
||||
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||
'evidence_url': url,
|
||||
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||
'paper_type': None,
|
||||
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||
temporary.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True, type=Path)
|
||||
parser.add_argument('--out', required=True, type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,172 @@
|
||||
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'recsys_program'
|
||||
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||
('doctoral_symposium', 'Doctoral', 10)]
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||
raise ValueError('Unreviewed snapshot scope')
|
||||
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||
name = receipt['file']; path = directory/name
|
||||
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Unsafe snapshot body')
|
||||
body = path.read_bytes()
|
||||
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
return receipt, {'manifest.json': raw, name: body}, body
|
||||
|
||||
|
||||
def body_text(node, doi_url):
|
||||
if not isinstance(node, Node):
|
||||
return node
|
||||
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||
return ''
|
||||
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||
raise ValueError('Wrong conference edition')
|
||||
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||
raise ValueError('Unreviewed track menu')
|
||||
observations = []; seen = set()
|
||||
for tab, (track, label, count) in enumerate(TRACKS):
|
||||
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||
if len(rows) != count:
|
||||
raise ValueError('Incomplete reviewed track census')
|
||||
for position, row in enumerate(rows, 1):
|
||||
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||
raise ValueError('Missing or repeated program locator')
|
||||
seen.add(locator)
|
||||
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||
if (type_literal,abbreviation) != expected_type:
|
||||
raise ValueError('Unreviewed source contribution type')
|
||||
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||
if not authors_literal.startswith('by '):
|
||||
raise ValueError('Missing program author prefix')
|
||||
authors_literal = authors_literal[3:]
|
||||
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||
if not title or any(not n or '@' in n for n in authors):
|
||||
raise ValueError('Missing title or program author')
|
||||
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||
raise ValueError('Wrong publication DOI family')
|
||||
abstract = clean(body_text(body, doi_url))
|
||||
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||
'track': track, 'track_label': label})
|
||||
counts = Counter(r['doi'] for r in observations)
|
||||
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||
raise ValueError('Unreviewed DOI observation census')
|
||||
for item in observations:
|
||||
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||
return observations
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
for name, raw in files.items():
|
||||
(evidence/name).write_bytes(raw)
|
||||
outputs = []
|
||||
for track,label,count in TRACKS:
|
||||
rows = []
|
||||
for item in observations:
|
||||
if item['track'] != track:
|
||||
continue
|
||||
doi = item['doi']; url = URL+'#'+item['locator']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||
'original_decision_signature_verified':False,
|
||||
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
@@ -0,0 +1,313 @@
|
||||
"""Frozen SDM 2026 publication records and separate public program observations."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import date, datetime
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
SOURCE = 'crossref_sdm_registry'
|
||||
TRACK = 'proceedings_unclassified'
|
||||
BOOK = '10.1137/1.9781611979329'
|
||||
ISBN = '9781611979329'
|
||||
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
|
||||
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
|
||||
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
|
||||
SNAPSHOTS = {
|
||||
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
|
||||
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
|
||||
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
|
||||
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
|
||||
}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected exactly one reviewed source field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def load_snapshot(root, name):
|
||||
expected, key = SNAPSHOTS[name]
|
||||
directory = root/name; path = directory/'manifest.json'
|
||||
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
|
||||
raise ValueError('Unsafe snapshot manifest')
|
||||
raw = path.read_bytes()
|
||||
if digest(raw) != expected:
|
||||
raise ValueError('Reviewed snapshot manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
|
||||
raise ValueError('Snapshot timestamp requires timezone')
|
||||
files = {'manifest.json': raw}
|
||||
for receipt in manifest[key]:
|
||||
if 'file' not in receipt:
|
||||
# A recorded 403 is an access observation, not an empty data set.
|
||||
continue
|
||||
filename = receipt['file']; path = directory/filename
|
||||
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
|
||||
raise ValueError('Unsafe snapshot source file')
|
||||
body = path.read_bytes()
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
if filename in files:
|
||||
raise ValueError('Repeated snapshot source file')
|
||||
files[filename] = body
|
||||
return manifest, files
|
||||
|
||||
|
||||
def registry_papers(message):
|
||||
items = message.get('items', [])
|
||||
if message.get('total-results') != 55 or len(items) != 55:
|
||||
raise ValueError('Incomplete exact-ISBN registry census')
|
||||
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
|
||||
raise ValueError('Unreviewed publication item types')
|
||||
papers = []; excluded = []; seen = set()
|
||||
for original in items:
|
||||
doi = original.get('DOI')
|
||||
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
|
||||
raise ValueError('Missing/repeated DOI or wrong ISBN')
|
||||
seen.add(doi)
|
||||
if doi in (BOOK, BOOK+'.fm'):
|
||||
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
|
||||
if (original['type'], original.get('title')) != expected:
|
||||
raise ValueError('Unreviewed excluded volume/front matter')
|
||||
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
|
||||
continue
|
||||
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
|
||||
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
|
||||
or original.get('URL') != 'https://doi.org/'+doi
|
||||
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
|
||||
or original.get('relation', {}).get('is-part-of') != [
|
||||
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
|
||||
raise ValueError('Wrong publication edition, publisher or parent DOI')
|
||||
title = one(original.get('title', []))
|
||||
if not isinstance(title, str) or not clean(title):
|
||||
raise ValueError('Missing registry title')
|
||||
flags = []
|
||||
if '<' in title or '>' in title:
|
||||
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
|
||||
if not match:
|
||||
raise ValueError('Unreviewed title markup')
|
||||
comparison_title = clean(match[1])
|
||||
flags.append('title_sup_asterisk_footnote_requires_review')
|
||||
else:
|
||||
comparison_title = clean(Document(title).root.text())
|
||||
displayed_title = clean(Document(title).root.text())
|
||||
authors = original.get('author')
|
||||
if (not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
|
||||
or not clean(a['given']) or not clean(a['family']) for a in authors)
|
||||
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||
raise ValueError('Incomplete registry authors or order evidence')
|
||||
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||
if any('@' in n or 'anonymous' in n.casefold() for n in names):
|
||||
raise ValueError('Unreviewed registry author identity')
|
||||
dates = {}
|
||||
for field in ('published', 'published-print', 'published-online', 'issued'):
|
||||
if field in original:
|
||||
parts = one(original[field]['date-parts'])
|
||||
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
|
||||
raise ValueError('Unreviewed registry publication date')
|
||||
dates[field] = date(*parts).isoformat()
|
||||
if not dates or original.get('abstract'):
|
||||
raise ValueError('Reviewed registry date/abstract availability changed')
|
||||
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
|
||||
'authors': names, 'review_flags': flags, 'publication_dates': dates})
|
||||
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
|
||||
raise ValueError('Incomplete reviewed 53-chapter DOI set')
|
||||
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
|
||||
|
||||
|
||||
class SessionParser(HTMLParser):
|
||||
"""The source omits closing dt/dd tags; each next dt starts a new event."""
|
||||
def __init__(self):
|
||||
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
|
||||
|
||||
def finish(self):
|
||||
if self.current is not None:
|
||||
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
|
||||
if not match or not clean(self.current['authors_literal']):
|
||||
raise ValueError('Incomplete contributed program event')
|
||||
self.current['time_literal'], self.current['title'] = match.groups()
|
||||
self.current['authors_literal'] = clean(self.current['authors_literal'])
|
||||
self.rows.append(self.current)
|
||||
self.current = None; self.strong = False; self.author = False
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
attrs = dict(attrs)
|
||||
if tag == 'dt':
|
||||
self.finish()
|
||||
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
|
||||
if self.current is not None:
|
||||
if tag == 'strong': self.strong = True
|
||||
elif tag == 'dd': self.author = True
|
||||
elif tag == 's': self.current['cancelled_presentation'] = True
|
||||
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
|
||||
self.current['abstract_links'].append(attrs['href'])
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'strong': self.strong = False
|
||||
elif tag == 'dl': self.finish()
|
||||
|
||||
def handle_data(self, text):
|
||||
if self.current is not None:
|
||||
if self.strong: self.current['timed_title'] += text
|
||||
elif self.author: self.current['authors_literal'] += text
|
||||
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
|
||||
|
||||
|
||||
def talk_fields(raw, session_url):
|
||||
nodes = list(Document(raw.decode('utf-8')).root.walk())
|
||||
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
|
||||
raise ValueError('Wrong program page edition')
|
||||
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
|
||||
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
|
||||
abstract = clean(abstract_node.text())
|
||||
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
|
||||
raise ValueError('Missing public program title/abstract')
|
||||
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
|
||||
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
|
||||
def session_key(url):
|
||||
u = urlsplit(url)
|
||||
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
|
||||
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
|
||||
raise ValueError('Foreign program session link')
|
||||
return query['sessioncode']
|
||||
if session_key(link) != session_key(session_url):
|
||||
raise ValueError('Program page belongs to a different session')
|
||||
authors = []
|
||||
for n in nodes:
|
||||
if n.tag == 'li':
|
||||
# Source rows contain name, affiliations, country and contact email.
|
||||
# Only the independently checked leading name is a metadata field.
|
||||
literal = clean(n.text()); name, separator, _ = literal.partition(',')
|
||||
if not separator or not name or '@' in name:
|
||||
raise ValueError('Unreviewed program author row')
|
||||
authors.append(name)
|
||||
if not authors:
|
||||
raise ValueError('Missing source-reported program authors')
|
||||
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
|
||||
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
|
||||
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
|
||||
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
|
||||
|
||||
|
||||
def program_observations(snapshots, papers):
|
||||
official, official_files = snapshots['sdm2026-official-research-v1']
|
||||
sessions, session_files = snapshots['sdm2026-program-research-v1']
|
||||
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
|
||||
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
|
||||
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
|
||||
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
|
||||
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
|
||||
raise ValueError('Contributed session membership disagrees with main program')
|
||||
receipts = {r['url']: r for r in pages['records']}
|
||||
if len(receipts) != len(pages['records']) or len(receipts) != 52:
|
||||
raise ValueError('Incomplete or repeated public talk receipts')
|
||||
rows = []; used = set()
|
||||
for session in sessions['sessions']:
|
||||
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
|
||||
for position, event in enumerate(parser.rows, 1):
|
||||
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
|
||||
event.update(record_kind='source_observation_not_import_record', session=session['title'],
|
||||
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
|
||||
if not urls and event['cancelled_presentation']:
|
||||
event.update(program_page=None, public_page_fields=None)
|
||||
else:
|
||||
url = one(urls)
|
||||
if url not in receipts or url in used:
|
||||
raise ValueError('Missing or repeated public program page')
|
||||
used.add(url); receipt = receipts[url]
|
||||
if receipt['session_url'] != session['url']:
|
||||
raise ValueError('Public page receipt session disagrees')
|
||||
fields = talk_fields(page_files[receipt['file']], session['url'])
|
||||
if fields['title'] != event['title']:
|
||||
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
|
||||
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
|
||||
candidates = []
|
||||
for paper in papers:
|
||||
if paper['comparison_title'].casefold() == event['title'].casefold():
|
||||
fields = event['public_page_fields']
|
||||
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
|
||||
'comparison': 'title_candidate_only_not_identity_approval',
|
||||
'registry_title_review_flags': paper['review_flags'],
|
||||
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
|
||||
event['registry_candidates'] = candidates
|
||||
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
|
||||
rows.append(event)
|
||||
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
|
||||
raise ValueError('Incomplete reviewed program event census')
|
||||
return rows
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
|
||||
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
|
||||
message = json.loads(registry_files['isbn-works.json'])['message']
|
||||
papers, excluded = registry_papers(message)
|
||||
observations = program_observations(loaded, papers)
|
||||
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||
for name, (manifest, files) in loaded.items():
|
||||
for filename, raw in files.items():
|
||||
rel = 'evidence/'+digest(raw)+Path(filename).suffix
|
||||
(out/rel).write_bytes(raw)
|
||||
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
|
||||
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
|
||||
rows = []
|
||||
for p in papers:
|
||||
original = p['original']; doi = original['DOI']
|
||||
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
|
||||
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
|
||||
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
|
||||
'observed_at': receipt['at'], 'evidence_url': QUERY,
|
||||
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
|
||||
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
|
||||
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
|
||||
'original_decision_signature_verified': False,
|
||||
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
|
||||
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
|
||||
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
|
||||
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
|
||||
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||
sidecar = out/'program-observations.jsonl'
|
||||
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
|
||||
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
|
||||
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
|
||||
'cancelled_presentations': 1, 'not_an_import_manifest': True},
|
||||
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
|
||||
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
|
||||
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def checked_html(raw, receipt, url, limit):
|
||||
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||
or receipt.get('sha256') != digest(raw)
|
||||
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||
raise ValueError('Access challenge; stop without bypass')
|
||||
return raw, receipt
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
class PublicHTMLCache:
|
||||
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||
self.root = Path(directory)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
if self.root.is_symlink():
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
self.origin = origin
|
||||
self.limit = limit
|
||||
self.delay = delay
|
||||
self.opener = urllib.request.build_opener(NoRedirect())
|
||||
|
||||
def paths(self, url):
|
||||
u = urlsplit(url)
|
||||
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||
raise ValueError('Unapproved public metadata origin')
|
||||
key = digest(url.encode())
|
||||
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||
|
||||
def cached(self, url):
|
||||
body, receipt, failure = self.paths(url)
|
||||
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
if any(p.exists() for p in parts):
|
||||
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||
if failure.exists():
|
||||
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||
if body.exists() != receipt.exists():
|
||||
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||
if not body.exists():
|
||||
return None
|
||||
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||
raise ValueError('Oversized cache entry')
|
||||
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||
|
||||
def get(self, url):
|
||||
cached = self.cached(url)
|
||||
if cached is not None:
|
||||
return cached
|
||||
body, receipt, failure = self.paths(url)
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||
with self.opener.open(request, timeout=30) as response:
|
||||
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||
'content_type': response.headers.get('Content-Type', '')}
|
||||
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||
raw = response.read(self.limit+1)
|
||||
if len(raw) > self.limit:
|
||||
raise ValueError('Metadata response exceeds size limit')
|
||||
expected = response.headers.get('Content-Length')
|
||||
if expected is not None and int(expected) != len(raw):
|
||||
raise ConnectionError('Incomplete metadata response')
|
||||
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||
checked_html(raw, rec, url, self.limit)
|
||||
except (urllib.error.HTTPError, ValueError) as error:
|
||||
if isinstance(error, urllib.error.HTTPError):
|
||||
status = error.code
|
||||
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||
error.close()
|
||||
else:
|
||||
status = None
|
||||
retry_after = None
|
||||
# Access errors remain sticky until the operator verifies availability.
|
||||
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||
raise
|
||||
part = body.with_suffix('.html.part')
|
||||
part.write_bytes(raw); part.replace(body)
|
||||
part = receipt.with_suffix('.json.part')
|
||||
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||
time.sleep(self.delay)
|
||||
return raw, rec
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,139 @@
|
||||
import copy
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import eccv2026 as c
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||
|
||||
|
||||
class ECCV2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||
|
||||
def snapshots(self, root):
|
||||
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||
for name, raw in self.bundle['official_pages'].items():
|
||||
(official/name).write_bytes(raw.encode('utf-8'))
|
||||
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||
for pid, raw in self.bundle['paper_pages'].items():
|
||||
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||
return official, paper
|
||||
|
||||
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||
self.assertEqual(len(self.program), 2834)
|
||||
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||
item = self.program[gold['paper_id']]
|
||||
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||
self.assertIn(ordinal, item['row_ordinals'])
|
||||
|
||||
def test_conflicting_duplicate_not_silently_folded(self):
|
||||
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||
|
||||
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||
html = self.bundle['official_pages']['accepted.html']
|
||||
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_public_page_fields_match_independent_sample_review(self):
|
||||
for gold in self.bundle['paper_gold']['records']:
|
||||
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||
|
||||
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_paper(changed, url, self.program['4048'])
|
||||
|
||||
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||
|
||||
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root)
|
||||
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
|
||||
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||
for r in self.bundle['paper_manifest']['records']:
|
||||
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||
|
||||
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_structured_math_markup_requires_explicit_review(self):
|
||||
raw = self.bundle['paper_pages']['4048']
|
||||
self.assertIn('<p>Proactive tamper', raw)
|
||||
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||
|
||||
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||
import hashlib
|
||||
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||
self.assertEqual(len(raw), receipt['bytes'])
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
for receipt in self.bundle['paper_manifest']['records']:
|
||||
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
|
||||
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||
import hashlib
|
||||
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
# Real changes to the author or an extra encoding layer remain errors.
|
||||
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||
|
||||
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||
import hashlib
|
||||
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertIn('&', fields['title'])
|
||||
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
@@ -0,0 +1,90 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
import urllib.error
|
||||
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||
|
||||
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||
|
||||
|
||||
class PublicHTMLCacheTests(unittest.TestCase):
|
||||
def receipt(self, raw):
|
||||
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||
|
||||
def test_old_cache_reused_without_network_or_expiry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||
|
||||
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||
for kind in ('hash','orphan','part'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||
c.opener = MagicMock()
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
c.opener.open.assert_not_called()
|
||||
|
||||
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||
self.assertEqual(c.opener.open.call_count,1)
|
||||
|
||||
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||
for kind in ('redirect','type','challenge','size'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||
if kind=='size':r.read.return_value=b'x'*101
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
self.assertFalse(c.paths(URL)[0].exists())
|
||||
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||
|
||||
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||
|
||||
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
|
||||
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = URL
|
||||
raw = b'<html>public metadata</html>'
|
||||
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||
self.assertEqual(c.opener.open.call_count, 1)
|
||||
body, receipt, _ = c.paths(URL)
|
||||
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||
|
||||
def test_changed_url_is_rejected_before_reading_body(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||
response.headers = {'Content-Type':'text/html'}
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
response.read.assert_not_called()
|
||||
@@ -0,0 +1,95 @@
|
||||
import base64
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icassp2026 as c
|
||||
|
||||
|
||||
class ICASSP2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||
cls.items = []
|
||||
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||
for receipt in cls.manifest['pages']:
|
||||
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||
|
||||
def materialize(self, root):
|
||||
for name, body in self.files.items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||
|
||||
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||
report = c.collect(root/'out', snapshots=src)
|
||||
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||
rows = [json.loads(l) for l in raw.splitlines()]
|
||||
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||
self.assertEqual(actual,gold)
|
||||
original = {r['DOI']:r for r,_ in self.items}
|
||||
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||
for r in rows:
|
||||
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
evidence = r['metadata']['source_evidence']
|
||||
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||
c.collect(root/'out',snapshots=src)
|
||||
|
||||
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||
with self.subTest(kind=kind):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||
if kind=='container': r['container-title']=['Foreign']
|
||||
if kind=='event': r['event']['name']='Other'
|
||||
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||
if kind=='author': r['author']=[]
|
||||
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||
if kind=='abstract': r['abstract']='new source version'
|
||||
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||
|
||||
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||
for kind in ('title','page','author'):
|
||||
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||
if kind=='title': r['title']=['A research paper']
|
||||
if kind=='page': r['page']='1-5'
|
||||
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||
|
||||
def test_duplicate_and_incomplete_census_rejected(self):
|
||||
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||
|
||||
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||
if kind=='missing': path.unlink()
|
||||
else: path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_page_and_snapshot_rejected(self):
|
||||
for kind in ('page','directory'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||
@@ -0,0 +1,113 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icdm2026 as c
|
||||
|
||||
|
||||
class ICDM2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||
cls.gold = cls.fixture['independent_inventory']
|
||||
|
||||
def snapshots(self, root):
|
||||
for name, text in self.fixture['files'].items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||
|
||||
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||
result = c.parse_program(self.raw)
|
||||
self.assertEqual(len(result), len(self.gold['entries']))
|
||||
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||
self.assertEqual(row[field], gold[field])
|
||||
self.assertEqual(row['source_position'], position)
|
||||
for field, value in gold['session'].items():
|
||||
self.assertEqual(row['session'][field], value)
|
||||
|
||||
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
report = c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||
all_rows = []
|
||||
for manifest in report['files']:
|
||||
path = Path(manifest['file']); raw = path.read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||
self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(line) for line in raw.splitlines()]
|
||||
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||
for row in rows:
|
||||
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||
all_rows.extend(rows)
|
||||
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||
for item in report['evidence']:
|
||||
raw = (root/'out'/item['output_file']).read_bytes()
|
||||
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
|
||||
def test_stale_or_different_conference_rejected(self):
|
||||
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_track_attribute_must_agree_with_visible_label(self):
|
||||
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'track'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new))
|
||||
|
||||
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||
c.parse_program(self.raw.replace(old, new, 1))
|
||||
|
||||
def test_empty_author_is_not_silently_dropped(self):
|
||||
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||
with self.assertRaisesRegex(ValueError, 'author'):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertFalse((root/'out').exists())
|
||||
|
||||
def test_snapshot_source_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
@@ -0,0 +1,79 @@
|
||||
import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import recsys2026 as c
|
||||
|
||||
|
||||
class RecSys2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||
|
||||
def materialize(self,root):
|
||||
for name,raw in self.files.items():
|
||||
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||
|
||||
def test_all263_observations_against_independent_gold(self):
|
||||
items=c.parse_program(self.body)
|
||||
keys=self.fixture['independent_gold'][0].keys()
|
||||
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||
|
||||
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||
report=c.collect(out,snapshots=src)
|
||||
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||
for manifest in report['files']:
|
||||
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||
for r in rows:
|
||||
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||
for key in expected:self.assertEqual(item[key],expected[key])
|
||||
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||
|
||||
def test_edition_and_track_label_drift_rejected(self):
|
||||
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||
|
||||
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||
|
||||
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||
for kind in ('manifest','body','missing'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||
if kind=='missing':path.unlink()
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_source_and_root_rejected(self):
|
||||
for kind in ('source','root'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||
@@ -0,0 +1,142 @@
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import sdm2026 as c
|
||||
|
||||
|
||||
class SDM2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
|
||||
|
||||
def snapshots(self, root):
|
||||
for name, text in self.fixture['files'].items():
|
||||
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
|
||||
|
||||
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
|
||||
papers, excluded = c.registry_papers(self.message)
|
||||
self.assertEqual(len(papers), 53)
|
||||
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
|
||||
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
|
||||
marked = [p for p in papers if p['review_flags']]
|
||||
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
|
||||
for p in marked:
|
||||
self.assertTrue(p['title'].endswith('*'))
|
||||
self.assertIn('<sup>*</sup>', p['original']['title'][0])
|
||||
|
||||
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
report = c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
|
||||
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
|
||||
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
||||
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
|
||||
for row in rows:
|
||||
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
|
||||
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
|
||||
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
|
||||
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
|
||||
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
|
||||
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
|
||||
sidecar = report['program_observations']
|
||||
self.assertTrue(sidecar['not_an_import_manifest'])
|
||||
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
|
||||
self.assertEqual(len(observations), 53)
|
||||
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
|
||||
for observation in observations:
|
||||
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
|
||||
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
|
||||
cancelled, = [x for x in observations if x['cancelled_presentation']]
|
||||
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
|
||||
self.assertIsNone(cancelled['public_page_fields'])
|
||||
# The real source has conflicting author sets under the same title.
|
||||
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
|
||||
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
|
||||
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
|
||||
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
|
||||
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
|
||||
# Every copied source and manifest remains byte-identical, including CRLF.
|
||||
for evidence in report['evidence']:
|
||||
body = (root/'out'/evidence['output_file']).read_bytes()
|
||||
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
|
||||
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
|
||||
def test_all_program_events_match_independent_inventory(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root)
|
||||
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
|
||||
papers, _ = c.registry_papers(self.message)
|
||||
observations = c.program_observations(loaded, papers)
|
||||
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
|
||||
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
|
||||
for item in observations:
|
||||
original = gold[(item['session_url'], item['title'])]
|
||||
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
|
||||
self.assertEqual(item[key], original[key])
|
||||
|
||||
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
|
||||
files = self.fixture['files']
|
||||
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
|
||||
receipts = {r['url']: r for r in manifest['records']}
|
||||
self.assertEqual(len(self.fixture['page_gold']), 52)
|
||||
for gold in self.fixture['page_gold']:
|
||||
receipt = receipts[gold['program_url']]
|
||||
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
|
||||
fields = c.talk_fields(raw, receipt['session_url'])
|
||||
self.assertEqual(fields['title'], gold['title'])
|
||||
self.assertEqual(fields['source_reported_authors'], gold['authors'])
|
||||
self.assertEqual(fields['abstract'], gold['abstract'])
|
||||
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
|
||||
|
||||
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
|
||||
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
|
||||
with self.subTest(kind=kind):
|
||||
message = copy.deepcopy(self.message)
|
||||
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
|
||||
if kind == 'isbn': row['ISBN'] = ['9780000000000']
|
||||
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
|
||||
if kind == 'repeat': row['DOI'] = c.BOOK
|
||||
if kind == 'type': row['type'] = 'other'
|
||||
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
|
||||
if kind == 'author': row['author'][0]['sequence'] = 'additional'
|
||||
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
|
||||
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
|
||||
if kind == 'missing': message['items'].pop()
|
||||
with self.assertRaises(ValueError): c.registry_papers(message)
|
||||
|
||||
def test_foreign_session_and_changed_edition_rejected(self):
|
||||
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
|
||||
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
|
||||
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
|
||||
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
|
||||
with self.assertRaises(ValueError): c.talk_fields(changed, url)
|
||||
|
||||
def test_corruption_preserves_failure_without_success_report(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
|
||||
with self.assertRaisesRegex(ValueError, 'receipt/body'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
self.assertTrue((root/'out/failed.json').exists())
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_source_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); self.snapshots(root/'src')
|
||||
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
|
||||
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
c.collect(root/'out', snapshots=root/'src')
|
||||
Reference in New Issue
Block a user