Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 13536d6268 | |||
| 44f4146f3e | |||
| b0bd9b69dc | |||
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| afb3b939ca |
@@ -19,8 +19,9 @@ jobs:
|
||||
# Public repository archive: no owner credentials or runner SSH keys.
|
||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
archive = response.read(16*1024*1024+1)
|
||||
if len(archive)>16*1024*1024:
|
||||
# Full publication-source fixtures retain original bibliographic evidence.
|
||||
archive = response.read(24*1024*1024+1)
|
||||
if len(archive)>24*1024*1024:
|
||||
raise SystemExit('Unexpectedly large source archive')
|
||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
@@ -46,6 +47,53 @@ jobs:
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside = root / 'outside-checkout'
|
||||
outside.mkdir()
|
||||
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
mm_snapshots = root/'mm2026-snapshots'
|
||||
for name, body in mm_fixture['files'].items():
|
||||
path = mm_snapshots/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(body.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||
import hashlib
|
||||
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||
row = mm_rows[gold['paper_id']]
|
||||
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
||||
mm_pub_registry = root/'mm2026-publication-registry'
|
||||
for name,body in mm_pub_fixture['files'].items():
|
||||
path=mm_pub_registry/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(body.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
|
||||
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||
assert len(mm_pub_rows)==1918
|
||||
for gold in mm_pub_fixture['independent_gold']:
|
||||
row=mm_pub_rows[gold['doi']]
|
||||
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
||||
if service_file.is_file():
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
||||
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
||||
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -140,3 +140,7 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
@@ -189,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
||||
|
||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||
|
||||
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||
|
||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||
|
||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||
@@ -198,3 +200,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||
|
||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||
|
||||
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||
|
||||
@@ -1,47 +1,48 @@
|
||||
# 2020 起补漏状态:2026-11-17
|
||||
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页补充新的生产结果,不回写 [2026-11-10 恢复清单](backfill-handoff-2026-11-10.md) 或更早快照。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
|
||||
## 最新数据库盘点
|
||||
|
||||
只读快照 `2026-11-17 09:38:31.677743+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 270 个至少有一条投稿记录,8 个仍为空,均为 2026 年:**CIKM、ECCV、ICASSP、ICDM、IROS、ISMIR、MM、RecSys**。本次 SDM2026 新增记录使空槽从 9 减到 8。
|
||||
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||
|
||||
这只说明有无记录,**不表示 270 个槽位已经逐篇、逐 track、摘要或公开拒稿覆盖完整**;不能据此宣布第 14 步完成。2020–2025 没有整年空槽,仍有逐来源完整性和公开负面决定缺口。SDM2026 仅导入 25/53 条,状态为 partial,同样不能称为整届补齐。
|
||||
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA-256 |
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||
| --- | --- |
|
||||
| collected/active-scope-census-20261117-v17/summary.json | bfb1db8b50619c3756abb91ad0f73fa58ff998c7db58a216661342f12047749f |
|
||||
| collected/active-scope-census-20261117-v17/snapshot.json | b5389205f8b34f7ed57946416f49c253e0aed4a05e45eb818801f18fca6a8cb4 |
|
||||
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||
|
||||
## 已推进的两批来源
|
||||
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||
|
||||
- **SDM2026**:53 个正式注册章节、53 个独立日程观察和 52 个公开摘要页已冻结并规范化。25 条发表记录已在 PostgreSQL 独立验收,新增论文实体为 0;28 条待身份/版本核查。完整旧历史和 5,653 篇受保护论文的资料、来源、标签和既有标识保留。见 [生产验收与恢复](../sdm2026-identity-review.md)。这批队列已经消费,禁止重放。
|
||||
- **ECCV2026**:2,834 个唯一初步日程条目及公开摘要已全量采集并独立验收;新的只读身份预检和 1,997 个候选历史证据包已保存,尚未入库。初步日程不等于已核验出版社最终论文集;负面决定元数据仍受匿名访问限制。见 [来源状态](../eccv2026-source-status.md)。
|
||||
## 已独立接受的近期补录
|
||||
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时具有较早 ICLR 拒稿与后来的 SDM 发表事实。旧的两次 ICLR 投稿及四条决定来源观察仍保留,没有被较晚 accepted 覆盖;多个来源观察不直接累加成多次拒稿。
|
||||
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||
|
||||
## 其余七个会议的最新入口核查
|
||||
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||
|
||||
已对下表七个会议的 14 个明确官方 URL 保存原始响应和回执:13 个 HTTP 200,IROS 官方论文索引 HTTP 403。所有请求均匿名,不绕过访问限制,没有下载论文 PDF;保存入口不是完成逐篇采集或负面决定普查。
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||
|
||||
| 会议 | 已保存的明确官方入口 | 尚未完成 |
|
||||
## 已采集、尚待关联或规范化的来源
|
||||
|
||||
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||
| --- | --- | --- |
|
||||
| ICDM | [详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)、[研究轨道政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)、[应用轨道政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm) | 官方页面称为线下展示列表;需要核对逐篇范围、研究/应用轨道及演讲长短标签,不能把展示长短自动等同于论文类型;摘要和正式出版身份仍待核查 |
|
||||
| ICASSP | [录用标题清单](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[作者指南](https://2026.ieeeicassp.org/author-guidelines/) | 取得逐篇完整作者/摘要/出版记录,区别会议投稿与期刊展示,不能只导入标题或从缺席推断拒稿 |
|
||||
| IROS | [官方论文索引](https://2026.ieee-iros.org/program/paper_index/) | 当前服务器返回 403,保留缺口并停止该入口;第三方浏览器页面不能替代独立核验的官方原文 |
|
||||
| ISMIR | [官网](https://ismir2026.ismir.net/)、[官方虚拟平台](https://ismir2026program.ismir.net/)、[作者指南](https://ismir2026.ismir.net/authors/author-guidelines) | 核查平台公开静态数据与正式开放论文集的逐篇映射;首页可读取不代表论文目录已经取得 |
|
||||
| CIKM | [录用页](https://cikm2026.diag.uniroma1.it/accepted-papers/)、[日程页](https://cikm2026.diag.uniroma1.it/program/) | 网页提取的录用页仍显示 TBD;需要检查原文引用的节目数据与独立正式出版目录,不能当成没有论文 |
|
||||
| RecSys | [官网](https://recsys.acm.org/recsys26/)、[投稿说明](https://recsys.acm.org/recsys26/call/) | 继续取得可核验的逐篇正式记录,并分开主轨、工业、复现、demo、challenge/workshop |
|
||||
| MM | [官网](https://2026.acmmm.org/) | 继续核对官方完整日程和出版目录,分开研究论文与 grand challenge、workshop 等;首页响应不证明覆盖完整 |
|
||||
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
||||
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||
|
||||
冻结目录:`/data/paper-index/collected/remaining2026-official-leads-v1`。manifest SHA-256:`0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。恢复包:`/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256:`0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。HTTP 成功仅是入口可读性,不能作为可导入记录数量。
|
||||
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||
|
||||
## 第 14 步仍需做什么
|
||||
## 公开拒稿与后续工作
|
||||
|
||||
1. 取得并规范化其余可公开取得的完整逐篇来源,记录应有范围、排除项和版本;对缺失、受限或尚未公开的入口保存明确证据,不能伪造“已补齐”。优先推进当前可读的 ICDM 详细日程、ICASSP 清单及其他会议的正式目录。
|
||||
2. 完成来源与年度、track、摘要完整性的对应核查;已经有若干记录的年份也要看目录差集,不能只消除空槽。
|
||||
3. 继续近期和历史年度实际公开的 rejected、desk_rejected、withdrawn 来源核查,特别是 UAI2024–2026 的 opt-in 拒稿、EMNLP2023 的实际公开范围以及其他会议的逐年限定政策。公开评审和公开决定不同;没有找到不等于零条或未曾拒稿。
|
||||
4. 通过身份核查的记录再执行新快照守护、恢复备份和增量导入,保留较早决定。深层作品版本、跨会历史及类型/标签协调继续列为第 15 步;不能为减少空槽创建重复论文,也不能只盯住少量身份疑难项而停止其他来源采集。
|
||||
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||
|
||||
第 16 步仍是所有导入结束后的独立全库去重与完整审计,第 17 步是最终长期运行/恢复和交付整理。上述任务均未因单批验收或 CI 成功而完成;总体 goal 继续保持 active。
|
||||
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||
|
||||
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
# MM 2026 出版条目采集与恢复
|
||||
|
||||
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
|
||||
|
||||
## 使用
|
||||
|
||||
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
|
||||
|
||||
```sh
|
||||
paper-atlas collect mm2026-publications \
|
||||
--registry-snapshots /data/paper-index/collected \
|
||||
--program-snapshots /data/paper-index/collected \
|
||||
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||
--out /data/paper-index/collected/mm2026-publications-native-v1
|
||||
```
|
||||
|
||||
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||
|
||||
## 注册目录与节目分别保留
|
||||
|
||||
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
|
||||
|
||||
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
|
||||
|
||||
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
|
||||
|
||||
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
|
||||
|
||||
## 分轨、摘要与版本差异
|
||||
|
||||
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
|
||||
|
||||
| 输出分区 | 注册条目 |
|
||||
| --- | ---: |
|
||||
| main | 1533 |
|
||||
| grand_challenge | 77 |
|
||||
| interactive_art | 17 |
|
||||
| datasets | 40 |
|
||||
| reproducibility | 1 |
|
||||
| bni | 22 |
|
||||
| open_source | 11 |
|
||||
| doctoral_symposium | 2 |
|
||||
| demo_and_video | 20 |
|
||||
| industry_talk | 3 |
|
||||
| proceedings_unclassified | 192 |
|
||||
|
||||
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
|
||||
|
||||
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
|
||||
|
||||
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||
|
||||
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`crossref:type` 与 `mm_program:track` 分别保留来源命名空间,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||
|
||||
## 只读数据库预检与导入边界
|
||||
|
||||
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
|
||||
|
||||
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
|
||||
|
||||
## 恢复
|
||||
|
||||
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
|
||||
|
||||
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||
@@ -0,0 +1,110 @@
|
||||
# ACM MM 2026 官方节目来源状态
|
||||
|
||||
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||
|
||||
## 已取得的数据
|
||||
|
||||
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||
|
||||
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||
|
||||
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||
|
||||
| 来源 track 原文 | 不同节目 ID 数 |
|
||||
| --- | ---: |
|
||||
| MRAC | 35 |
|
||||
| SUMAC | 9 |
|
||||
| NeuroMM | 10 |
|
||||
| IMPACT | 8 |
|
||||
| RichMediaGAI | 10 |
|
||||
| MMSports | 16 |
|
||||
| MUWS | 5 |
|
||||
| EReL-MIR | 7 |
|
||||
| GreenMM | 10 |
|
||||
| DFF | 12 |
|
||||
| Main Track | 1703 |
|
||||
| Grand Challenge | 97 |
|
||||
| Doctoral Symposium | 4 |
|
||||
| Datasets | 50 |
|
||||
| Interactive Art | 23 |
|
||||
| Reproducibility | 1 |
|
||||
| Demo and Video | 27 |
|
||||
| Open Source | 14 |
|
||||
| Industry Talk | 7 |
|
||||
| BNI | 25 |
|
||||
| QoEVMA | 4 |
|
||||
| HUMA | 14 |
|
||||
| WIPAI | 8 |
|
||||
| AMI | 11 |
|
||||
| Mulsemedia | 9 |
|
||||
| UAVM | 11 |
|
||||
| MCHM | 7 |
|
||||
| JAV-CG | 6 |
|
||||
|
||||
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||
|
||||
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||
|
||||
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||
|
||||
## 公开负面决定的访问边界
|
||||
|
||||
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||
|
||||
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||
|
||||
## 产物与恢复
|
||||
|
||||
原始目录(均相对 `/data/paper-index/collected/`):
|
||||
|
||||
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||
|
||||
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||
|
||||
| 证据 | SHA256 |
|
||||
| --- | --- |
|
||||
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||
|
||||
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||
|
||||
## 可复用采集命令
|
||||
|
||||
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||
|
||||
```sh
|
||||
paper-atlas collect mm2026-program \
|
||||
--snapshots /data/paper-index/collected \
|
||||
--out /data/paper-index/collected/mm2026-native-v1
|
||||
```
|
||||
|
||||
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||
|
||||
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||
|
||||
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||
|
||||
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||
|
||||
## 直接审计节目身份与既往历史
|
||||
|
||||
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||
|
||||
```sh
|
||||
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||
paper-atlas audit source-identities \
|
||||
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||
```
|
||||
|
||||
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||
|
||||
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||
|
||||
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||
@@ -51,3 +51,33 @@ paper-atlas collect recsys2026-program \
|
||||
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||
|
||||
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||
|
||||
## 2026-11-17 PostgreSQL 补录验收
|
||||
|
||||
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||
|
||||
| track | 已导入观察 / 来源应有观察 |
|
||||
| --- | ---: |
|
||||
| demo | 1 / 18 |
|
||||
| doctoral_symposium | 2 / 10 |
|
||||
| industry | 18 / 83 |
|
||||
| long | 13 / 65 |
|
||||
| past_present_future | 2 / 13 |
|
||||
| reproducibility | 1 / 4 |
|
||||
| research_practice_notes | 4 / 34 |
|
||||
| resource | 2 / 7 |
|
||||
| short | 4 / 29 |
|
||||
|
||||
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||
|
||||
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||
|
||||
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||
|
||||
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||
|
||||
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||
|
||||
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||
|
||||
@@ -74,6 +74,74 @@ def load_inputs(files):
|
||||
return records, inputs
|
||||
|
||||
|
||||
def load_mm_program_inputs(report_path):
|
||||
"""Read every MM observation without turning a program into import records.
|
||||
|
||||
Reconcile the complete output against its frozen raw source before any DB
|
||||
connection. The synthetic unknown status is an audit placeholder only.
|
||||
"""
|
||||
from paper_atlas.collectors import mm2026
|
||||
|
||||
report_path = Path(report_path).absolute()
|
||||
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||
report = json.loads(raw_report)
|
||||
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||
raise ValueError('Unreviewed MM observation report')
|
||||
root = report_path.parent
|
||||
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||
raise ValueError('MM raw evidence changed')
|
||||
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||
expected = {r['paper_id']:r for r in expected_rows}
|
||||
records, inputs, seen, tracks = [], [], set(), set()
|
||||
for item in report['files']:
|
||||
track = item.get('track_literal')
|
||||
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||
or item.get('approved_for_import') != 0):
|
||||
raise ValueError('Duplicate or unreviewed MM track format')
|
||||
tracks.add(track)
|
||||
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||
path = root/Path(item['file']).name
|
||||
if path.suffix != '.jsonl':
|
||||
raise ValueError('Invalid MM observation filename')
|
||||
raw = mm2026.safe_read(path, 16*1024**2)
|
||||
lines = raw.splitlines(keepends=True)
|
||||
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||
raise ValueError('MM observation file count/hash mismatch')
|
||||
for number,line in enumerate(lines, 1):
|
||||
row = json.loads(line)
|
||||
pid = row.get('paper_id')
|
||||
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||
seen.add(pid)
|
||||
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||
'identifier_hits':[],'title_candidate_ids':[],
|
||||
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'program_id':pid,'program_contexts':row['contexts'],
|
||||
'source_observed_status':row['observed_status'],
|
||||
'missing_source_fields':row['missing_fields'],
|
||||
'publication_identity_verified':False,'final_decision_verified':False,
|
||||
'contribution_type_verified':False})
|
||||
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||
'manifest_sha256':digest(raw_report),'manifest':{
|
||||
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||
return records, inputs
|
||||
|
||||
|
||||
def _lookup(connection, values, columns, lateral):
|
||||
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||
width = len(columns)
|
||||
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def audit(files, out, *, pg_service, pg_schema):
|
||||
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||
# Reject input corruption before opening a database connection or reserving output.
|
||||
records, inputs = load_inputs(files)
|
||||
if bool(files) == bool(mm_program_report):
|
||||
raise ValueError('Choose source files or one MM program report')
|
||||
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||
out = Path(out)
|
||||
out.mkdir(parents=True, exist_ok=False)
|
||||
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
|
||||
'Name agreement does not prove person or scientific-work identity.',
|
||||
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||
'Source/history limits are explicit in compressed entity packets.',
|
||||
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||
'Not a final whole-database deduplication or coverage audit.']}
|
||||
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||
return report
|
||||
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
||||
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||
inputs.add_argument('--files', nargs='+', type=Path)
|
||||
inputs.add_argument('--mm-program-report', type=Path,
|
||||
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||
parser.add_argument('--pg-service', required=True)
|
||||
parser.add_argument('--pg-schema', required=True)
|
||||
parser.add_argument('--out', required=True, type=Path)
|
||||
args = parser.parse_args(argv)
|
||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||
mm_program_report=args.mm_program_report)
|
||||
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||
|
||||
@@ -32,6 +32,8 @@ COMMANDS = {
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
SOURCE = 'mm_program'
|
||||
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def safe_read(path, limit):
|
||||
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||
raise ValueError('Unsafe snapshot path or size')
|
||||
return path.read_bytes()
|
||||
|
||||
|
||||
def load_snapshot(root):
|
||||
directory = Path(root)/SNAPSHOT
|
||||
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||
if digest(raw) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed manifest hash changed')
|
||||
manifest = json.loads(raw)
|
||||
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||
raise ValueError('Unreviewed manifest scope')
|
||||
evidence = {'manifest.json':raw}
|
||||
for receipt in manifest['entries']:
|
||||
name = receipt['file']
|
||||
if Path(name).name != name or name in evidence:
|
||||
raise ValueError('Unsafe or duplicate receipt name')
|
||||
body = safe_read(directory/name, 12*1024**2)
|
||||
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||
or receipt['url'] != receipt['final_url']
|
||||
or not receipt['content_type'].startswith('application/json')
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||
raise ValueError('Snapshot receipt/body disagree')
|
||||
if name == 'program-data.json':
|
||||
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||
raise ValueError('Wrong program source')
|
||||
elif receipt['http_status'] != 403:
|
||||
raise ValueError('Unreviewed public-access result')
|
||||
evidence[name] = body
|
||||
if 'program-data.json' not in evidence:
|
||||
raise ValueError('Missing program source')
|
||||
return evidence
|
||||
|
||||
|
||||
def without_emails(value):
|
||||
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||
if isinstance(value, dict):
|
||||
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||
if isinstance(value, list):
|
||||
return [without_emails(v) for v in value]
|
||||
return value
|
||||
|
||||
|
||||
def program_locations(node, pointer=''):
|
||||
"""Independent structural guard against silently skipping a new nesting form."""
|
||||
if isinstance(node, dict):
|
||||
if 'paper_id' in node:
|
||||
yield pointer
|
||||
for key,value in node.items():
|
||||
escaped = key.replace('~','~0').replace('/','~1')
|
||||
yield from program_locations(value, pointer+'/'+escaped)
|
||||
elif isinstance(node, list):
|
||||
for i,value in enumerate(node):
|
||||
yield from program_locations(value, pointer+'/'+str(i))
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
data = json.loads(raw)
|
||||
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||
or len(data.get('entries',[])) != 157):
|
||||
raise ValueError('Unreviewed program edition or census')
|
||||
records = {}; sessions = []; seen = []
|
||||
|
||||
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||
raise ValueError('Unreviewed program fields')
|
||||
for key in ('paper_id','title','abstract','track'):
|
||||
if not isinstance(paper[key],str):
|
||||
raise ValueError('Wrong program field type')
|
||||
for key in ('authors','affiliations'):
|
||||
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||
raise ValueError('Wrong author or affiliation list')
|
||||
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||
raise ValueError('Missing identity/title or unreviewed source track')
|
||||
core = {k:paper[k] for k in CORE}
|
||||
pid = paper['paper_id']; seen.append(pointer)
|
||||
if pid not in records:
|
||||
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||
'observed_status':'listed_in_official_program',
|
||||
'publication_identity_verified':False,'final_decision_verified':False,
|
||||
'approved_for_import':False,'missing_fields':missing,
|
||||
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||
'contexts':[]}
|
||||
elif {k:records[pid][k] for k in CORE} != core:
|
||||
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||
'subsession_pointer':subsession_pointer,
|
||||
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||
|
||||
for i,entry in enumerate(data['entries']):
|
||||
pointer = '/entries/'+str(i)
|
||||
if entry['track'] != '':
|
||||
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||
session.update(source_pointer=pointer,subsessions=[])
|
||||
for j,paper in enumerate(entry.get('papers') or []):
|
||||
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||
sp = pointer+'/subsessions/'+str(j)
|
||||
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||
for k,item in enumerate(sub.get('items') or []):
|
||||
ip = sp+'/items/'+str(k)
|
||||
if 'paper_id' in item:
|
||||
observe(item,ip,pointer,sp)
|
||||
else:
|
||||
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||
session['subsessions'].append(saved)
|
||||
sessions.append(session)
|
||||
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||
raise ValueError('Unhandled or duplicated program location')
|
||||
items = list(records.values())
|
||||
if (len(seen) != 4326 or len(items) != 2143
|
||||
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||
or sum(not r['abstract'] for r in items) != 216
|
||||
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||
raise ValueError('Program count or missing-field census changed')
|
||||
return items,sessions
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists() or out.is_symlink():
|
||||
raise ValueError('Output exists; use a fresh directory')
|
||||
if any(p.is_symlink() for p in out.parents):
|
||||
raise ValueError('Unsafe output parent')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
evidence = load_snapshot(snapshots)
|
||||
rows,sessions = parse_program(evidence['program-data.json'])
|
||||
(out/'evidence').mkdir()
|
||||
for name,body in evidence.items():
|
||||
(out/'evidence'/name).write_bytes(body)
|
||||
outputs = []
|
||||
for track,count in TRACK_COUNTS.items():
|
||||
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||
path.write_bytes(raw)
|
||||
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||
part = out/'collection-report.json.part'
|
||||
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--snapshots',required=True)
|
||||
parser.add_argument('--out',required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||
@@ -0,0 +1,227 @@
|
||||
"""Frozen MM 2026 registered publications with separately attributed program evidence.
|
||||
|
||||
All publications survive normalization. A program metadata match is not a
|
||||
scientific-work identity decision; publisher article type is not research type.
|
||||
"""
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.collectors import mm2026 as program
|
||||
|
||||
SOURCE = 'mm_registry_program'
|
||||
SNAPSHOT = 'mm2026-container-registry-v2'
|
||||
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
|
||||
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
|
||||
PARENT_DOI = '10.1145/3767308'
|
||||
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
|
||||
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
|
||||
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
|
||||
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
|
||||
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
|
||||
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
|
||||
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
|
||||
'publisher','member','type','container-title','short-container-title','event','abstract',
|
||||
'page','ISBN','isbn-type','published','published-print','published-online','issued',
|
||||
'license','resource','relation','alternative-id'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def key(text):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def author_names(values):
|
||||
return Counter(key(v) for v in values)
|
||||
|
||||
|
||||
def load_registry(root, parent_path):
|
||||
directory = Path(root)/SNAPSHOT
|
||||
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
|
||||
raw_parent = program.safe_read(Path(parent_path), 128*1024)
|
||||
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
|
||||
raise ValueError('Reviewed registry or parent hash changed')
|
||||
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
|
||||
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
|
||||
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
|
||||
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
|
||||
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
|
||||
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
|
||||
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
|
||||
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
|
||||
raise ValueError('Incomplete reviewed registry or wrong proceedings')
|
||||
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
|
||||
for number,receipt in enumerate(manifest['pages'],1):
|
||||
name=receipt['file']
|
||||
if name != f'page-{number:03}.json':
|
||||
raise ValueError('Unexpected registry page filename')
|
||||
body=program.safe_read(directory/name,8*1024**2)
|
||||
url=urlsplit(receipt['url']);query=parse_qs(url.query)
|
||||
expected_filter='container-title:'+CONTAINER+',member:320'
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
|
||||
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
|
||||
or query['rows'] != ['250'] or not query['cursor'][0]
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Registry receipt and source body disagree')
|
||||
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
|
||||
if (obj['status'] != 'ok' or message['total-results'] != 1918
|
||||
or len(message['items']) != expected or receipt['items'] != expected):
|
||||
raise ValueError('Incomplete registry pagination')
|
||||
evidence[name]=body
|
||||
for position,original in enumerate(message['items']):
|
||||
doi=original.get('DOI')
|
||||
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
|
||||
raise ValueError('Missing, foreign or repeated publication DOI')
|
||||
seen.add(doi)
|
||||
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
|
||||
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
|
||||
or original.get('member') != '320' or original.get('publisher') != 'ACM'
|
||||
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
|
||||
raise ValueError('Wrong registered publication context or changed abstract availability')
|
||||
titles=original.get('title');authors=original.get('author')
|
||||
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
|
||||
or not isinstance(authors,list) or not authors):
|
||||
raise ValueError('Missing publication title or authors')
|
||||
names=[];flags=[]
|
||||
for i,author in enumerate(authors,1):
|
||||
family=author.get('family');given=author.get('given','')
|
||||
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
|
||||
raise ValueError('Invalid registered author fields')
|
||||
name=' '.join(x.strip() for x in (given,family) if x.strip())
|
||||
if '@' in name or name.casefold()=='anonymous':
|
||||
raise ValueError('Unreviewed author identity')
|
||||
names.append(name)
|
||||
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
|
||||
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
|
||||
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
|
||||
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
|
||||
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
|
||||
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
|
||||
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
|
||||
'receipt':receipt,'source_position':position})
|
||||
if len(rows)!=1918:
|
||||
raise ValueError('Incomplete registered publication census')
|
||||
return evidence,rows
|
||||
|
||||
|
||||
def correlate(rows, observations):
|
||||
titles=defaultdict(list)
|
||||
for row in observations:titles[key(row['title'])].append(row)
|
||||
matched=set();kinds=Counter()
|
||||
for row in rows:
|
||||
candidates=titles[key(row['title'])]
|
||||
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||
if len(exact)==1:
|
||||
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
|
||||
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||
if exact[0]['track'] not in TRACKS:
|
||||
raise ValueError('New registered program track requires review')
|
||||
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||
elif candidates:row['kind']='program_author_version_requires_review'
|
||||
else:row['kind']='no_exact_program_title_requires_review'
|
||||
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||
kinds[row['kind']]+=1
|
||||
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
|
||||
if kinds != expected:
|
||||
raise ValueError('Reviewed program/publication correspondence changed')
|
||||
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||
|
||||
|
||||
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
out=Path(out)
|
||||
if out.exists() or out.is_symlink():
|
||||
raise ValueError('Output exists; inspect prior operation, never overwrite')
|
||||
if any(p.is_symlink() for p in out.parents):
|
||||
raise ValueError('Unsafe output parent')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
|
||||
program_evidence=program.load_snapshot(program_snapshots)
|
||||
observations,_=program.parse_program(program_evidence['program-data.json'])
|
||||
kinds,unmatched=correlate(publications,observations)
|
||||
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
|
||||
directory=out/'evidence'/group;directory.mkdir(parents=True)
|
||||
for name,body in files.items():(directory/name).write_bytes(body)
|
||||
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
|
||||
for item in sorted(publications,key=lambda r:r['doi']):
|
||||
p=item.get('program');doi=item['doi'];receipt=item['receipt']
|
||||
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
|
||||
abstract=p['abstract'] if p and p['abstract'] else None
|
||||
missing+=abstract is None;copied+=abstract is not None
|
||||
if p:known_tracks[p['track']]+=1
|
||||
tags=[{'namespace':'crossref:type','value':'proceedings-article'}]
|
||||
if p:tags.append({'namespace':'mm_program:track','value':p['track']})
|
||||
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
|
||||
'title':item['title'],'authors':item['authors'],'abstract':abstract,
|
||||
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
|
||||
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
|
||||
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
|
||||
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
|
||||
'program_fields':p,'program_match_kind':item['kind'],
|
||||
'program_title_candidates':item['program_title_candidates'],
|
||||
'program_exact_candidates':item['program_exact_candidates'],
|
||||
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
|
||||
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||
'review_flags':item['flags'],
|
||||
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
|
||||
outputs=[]
|
||||
for track,rows in sorted(buckets.items()):
|
||||
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
|
||||
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
|
||||
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
|
||||
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
|
||||
raise ValueError('Reviewed output partition or abstract census changed')
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--registry-snapshots',required=True)
|
||||
parser.add_argument('--program-snapshots',required=True)
|
||||
parser.add_argument('--parent-snapshot',required=True)
|
||||
parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv)
|
||||
report=collect(args.out,registry_snapshots=args.registry_snapshots,
|
||||
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
|
||||
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
|
||||
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
|
||||
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
|
||||
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,146 @@
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import mm2026 as c
|
||||
|
||||
|
||||
class MM2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||
cls.data = json.loads(cls.raw)
|
||||
cls.gold = cls.fixture['independent_inventory']
|
||||
|
||||
def materialize(self, root):
|
||||
for name,body in self.files.items():
|
||||
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||
|
||||
def parse_changed(self, data):
|
||||
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||
|
||||
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||
rows,sessions = c.parse_program(self.raw)
|
||||
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||
actual = {r['paper_id']:r for r in rows}
|
||||
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||
for gold in self.gold['inventory']:
|
||||
r = actual[gold['paper_id']]
|
||||
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||
|
||||
def test_every_presentation_and_session_context_is_preserved(self):
|
||||
rows,sessions = c.parse_program(self.raw)
|
||||
for r in rows:
|
||||
for context in r['contexts']:
|
||||
node = self.data
|
||||
for key in context['source_pointer'].split('/')[1:]:
|
||||
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||
for i,session in enumerate(sessions):
|
||||
original = self.data['entries'][i]
|
||||
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||
self.assertEqual(session['track'],'')
|
||||
for key in original.keys()-{'papers','subsessions'}:
|
||||
self.assertEqual(session[key],original[key])
|
||||
for j,sub in enumerate(original.get('subsessions') or []):
|
||||
saved = session['subsessions'][j]
|
||||
self.assertEqual(saved['title'],sub['title'])
|
||||
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||
|
||||
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||
rows,_ = c.parse_program(self.raw)
|
||||
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||
for r in rows:
|
||||
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||
self.assertFalse(r['publication_identity_verified'])
|
||||
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||
missing = [r for r in rows if not r['authors']]
|
||||
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||
|
||||
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'src')
|
||||
report=c.collect(root/'out',snapshots=root/'src')
|
||||
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||
for name,body in self.files.items():
|
||||
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||
n=0
|
||||
for item in report['files']:
|
||||
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||
self.assertEqual(n,2143)
|
||||
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||
|
||||
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||
data=copy.deepcopy(self.data)
|
||||
first=data['entries'][0]['papers'][0]
|
||||
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||
|
||||
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||
data=copy.deepcopy(self.data)
|
||||
moved=data['entries'][0]['papers'].pop()
|
||||
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||
|
||||
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||
elif kind=='paper_track':paper['track']='New Track'
|
||||
elif kind=='field':paper['doi']='unreviewed'
|
||||
elif kind=='title':paper['title']=''
|
||||
else:paper['authors']='Unparsed author string'
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||
|
||||
def test_edition_or_census_change_requires_review(self):
|
||||
for kind in ('edition','count'):
|
||||
data=copy.deepcopy(self.data)
|
||||
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||
else:data['entries'][0]['papers'].pop()
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||
|
||||
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||
for kind in ('manifest','program','probe','missing','symlink'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||
path=directory/name
|
||||
if kind=='missing':path.unlink()
|
||||
elif kind=='symlink':
|
||||
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||
else:path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||
rows,sessions=c.parse_program(self.raw)
|
||||
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||
@@ -0,0 +1,155 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import mm_publications2026 as c
|
||||
|
||||
|
||||
class MMPublications2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
fixtures=Path(__file__).parent/'fixtures'
|
||||
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||
cls.out=cls.root/'out'
|
||||
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
||||
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
||||
cls.by_doi={r['external_id']:r for r in cls.rows}
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.temp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def materialize(root,files):
|
||||
for name,body in files.items():
|
||||
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
||||
|
||||
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
||||
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
||||
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
||||
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
||||
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
||||
|
||||
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];m=r['metadata']
|
||||
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||
if g['abstract_sha256'] is not None:
|
||||
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
|
||||
else:
|
||||
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
|
||||
|
||||
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||
for p in self.report['unmatched_program']:
|
||||
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
|
||||
gold=self.program_fixture['independent_inventory']['inventory']
|
||||
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||
for g in gold:
|
||||
p=programs[g['paper_id']]
|
||||
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
||||
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
||||
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
||||
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
||||
|
||||
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
||||
for r in self.rows:
|
||||
doi=r['external_id'];m=r['metadata']
|
||||
self.assertEqual(r['identifiers'],{'doi':doi})
|
||||
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
||||
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
||||
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
||||
self.assertFalse(m['scientific_work_identity_approved'])
|
||||
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
||||
self.assertIn({'namespace':'crossref:type','value':'proceedings-article'},r['tags'])
|
||||
self.assertTrue({x['namespace'] for x in r['tags']} <= {'crossref:type','mm_program:track'})
|
||||
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||
self.assertEqual(len(variants),192)
|
||||
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||
|
||||
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||
authors_by_page={}
|
||||
for name,body in self.fixture['files'].items():
|
||||
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
||||
self.assertEqual(actual.read_bytes(),body.encode())
|
||||
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
||||
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
||||
for name,body in self.program_fixture['files'].items():
|
||||
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
||||
for r in self.rows:
|
||||
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
||||
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
||||
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
||||
for f in self.report['files']:
|
||||
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
||||
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
||||
|
||||
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||
n=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||
for i in actual:
|
||||
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
||||
self.assertEqual(n,8)
|
||||
|
||||
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||
count+=1;r=self.by_doi[g['doi']]
|
||||
self.assertEqual(r['title'],g['title'])
|
||||
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||
|
||||
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
||||
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
||||
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
||||
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
||||
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||
|
||||
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
|
||||
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
|
||||
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||
self.assertIn('unknown',self.report['public_negative_availability'])
|
||||
@@ -0,0 +1,101 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import mm2026
|
||||
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||
|
||||
|
||||
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.tmp.cleanup)
|
||||
self.root = Path(self.tmp.name)
|
||||
src = self.root/'source'
|
||||
for name,body in self.fixture['files'].items():
|
||||
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||
mm2026.collect(self.root/'program', snapshots=src)
|
||||
self.report_path = self.root/'program/collection-report.json'
|
||||
|
||||
def change_report(self, change):
|
||||
report=json.loads(self.report_path.read_text());change(report)
|
||||
self.report_path.write_text(json.dumps(report))
|
||||
|
||||
def assert_rejected_before_database(self):
|
||||
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||
with self.assertRaises(ValueError):
|
||||
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||
connection.assert_not_called()
|
||||
self.assertFalse((self.root/'audit').exists())
|
||||
|
||||
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||
actual={r['program_id']:r for r in rows}
|
||||
for gold in self.fixture['independent_inventory']['inventory']:
|
||||
r=actual[gold['paper_id']]
|
||||
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||
(gold['title'],gold['authors'],gold['track_literal']))
|
||||
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||
self.assertFalse(r['contribution_type_verified'])
|
||||
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||
|
||||
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||
self.assertEqual(len(rows),2143)
|
||||
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||
|
||||
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||
report=json.loads(self.report_path.read_text())
|
||||
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||
|
||||
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
self.assert_rejected_before_database()
|
||||
|
||||
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||
|
||||
def test_missing_or_repeated_track_rejected(self):
|
||||
original=self.report_path.read_bytes()
|
||||
for kind in ('missing','repeated'):
|
||||
self.report_path.write_bytes(original)
|
||||
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||
|
||||
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||
self.assert_rejected_before_database()
|
||||
|
||||
def test_mixed_input_modes_rejected_before_connection(self):
|
||||
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||
connection.assert_not_called()
|
||||
Reference in New Issue
Block a user