Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7e63035570 |
@@ -19,9 +19,8 @@ jobs:
|
||||
# Public repository archive: no owner credentials or runner SSH keys.
|
||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||
with urllib.request.urlopen(url, timeout=60) as response:
|
||||
# Full publication-source fixtures retain original bibliographic evidence.
|
||||
archive = response.read(24*1024*1024+1)
|
||||
if len(archive)>24*1024*1024:
|
||||
archive = response.read(16*1024*1024+1)
|
||||
if len(archive)>16*1024*1024:
|
||||
raise SystemExit('Unexpectedly large source archive')
|
||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
@@ -67,27 +66,7 @@ jobs:
|
||||
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
||||
mm_pub_registry = root/'mm2026-publication-registry'
|
||||
for name,body in mm_pub_fixture['files'].items():
|
||||
path=mm_pub_registry/name
|
||||
path.parent.mkdir(parents=True,exist_ok=True)
|
||||
path.write_bytes(body.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
|
||||
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||
assert len(mm_pub_rows)==1918
|
||||
for gold in mm_pub_fixture['independent_gold']:
|
||||
row=mm_pub_rows[gold['doi']]
|
||||
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
||||
if service_file.is_file():
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
||||
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
||||
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||
|
||||
@@ -142,5 +142,3 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# 会议覆盖与公开决定核查
|
||||
|
||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||
最新生产验收与剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):当日 `14:06:28 UTC` 固定范围快照中,278 个会议年份槽位有 272 个含记录、6 个仍为空;这不代表完整覆盖,第 14 步仍未完成。后续 MM 规范化、只读候选审计与出版核查不等于生产导入。
|
||||
|
||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||
|
||||
@@ -72,7 +72,7 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
|
||||
|
||||
## 优先缺口
|
||||
|
||||
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,当前主库拒稿仍为零。原 notes 接口要求 Challenge 验证,公开镜像尚无已核验来源,必须保留这个真实缺口;不能根据 PMLR 录用名单推断拒稿。参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
||||
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,首份生产快照拒稿为零。原 notes 接口要求 Challenge 验证;新核验镜像的 2022–2025 年 902 个论坛来自录用子集,不能填补公开拒稿缺口。2024 年的元评审也不能直接转成最终决定,见 [2026-11-17 原始来源与回复结构核查](uai-public-negative-source-status-2026-11-17.md)。不能根据 PMLR 或镜像录用名单反推拒稿;政策参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
||||
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
||||
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
||||
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
||||
|
||||
@@ -33,7 +33,7 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
|
||||
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
||||
| MM | 全28轨2143个节目ID/4326展示位置已规范化;814个只读候选历史证据包已核验,24篇候选涉及24次负面投稿/37条来源观察;67项原始arXiv DOI中66项注册元数据一致、1项404暂缓 | 不是2143篇已核验科研论文。继续逐轨贡献/出版依据与作品审查,0身份批准、0入库。ISBN查询与父XML均只返回父记录,不作逐篇清单。见 [来源、审计与恢复证据](../mm2026-source-status.md) |
|
||||
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||
|
||||
@@ -43,6 +43,8 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
|
||||
|
||||
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||
|
||||
UAI新增固定镜像已核对2022–2025的902个原始论坛及3053个回复的元数据/字段结构。整理版均标录用,2022保存230条明确Accept决定;2024仅有评审、作者回复、元评审,2023/2025未保存回复,不能据此补拒稿。来源限制和恢复见 [逐年原始核查](uai-public-negative-source-status-2026-11-17.md)。这不是UAI没有公开拒稿的结论,缺口保持未补齐。
|
||||
|
||||
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||
|
||||
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
# UAI 公开负面决定来源:2026-11-17 核查
|
||||
|
||||
**公开拒稿缺口仍未补齐,总体仍为 13/17。** 本页保存新来源的实际范围与恢复线索,避免下次把录用子集误当作全部投稿。主动补漏范围仍从2020年开始;本次镜像只含2022–2025,不能替代其他年份或其他会议的核查。
|
||||
|
||||
## 政策与访问边界
|
||||
|
||||
已有逐年核查记录保存在 [覆盖清单](inventory-2026-11-08.json) 的 UAI `policy_checks`:2023–2026 官方说明允许决定后公开录用及作者选择公开的拒稿,默认私密拒稿不属于公开可采集范围。政策并不证明任何镜像完整。
|
||||
|
||||
- [2023 投稿说明](https://www.auai.org/uai2023/submission_instructions)
|
||||
- [2024 投稿说明](https://auai.org/uai2024/submission_instructions)
|
||||
- [2025 投稿说明](https://www.auai.org/uai2025/submission_instructions)
|
||||
- [2026 投稿说明](https://www.auai.org/uai2026/submission_instructions)
|
||||
|
||||
先前匿名 notes 请求遇到 ChallengeRequiredError/403,仍作为访问限制保留。本轮没有认证、重试或绕过这些入口;读取的是整理者公开发布的固定版本镜像。复制的 `readers`、`signatures` 和邀请字段仍只是镜像内的来源断言,不是对当前原始接口可见性或签名真实性的验证。
|
||||
|
||||
## 实际镜像范围
|
||||
|
||||
来源:[latkes/ai-research-index-iclr-openreview](https://huggingface.co/datasets/latkes/ai-research-index-iclr-openreview/tree/4c9e09931a7814d7500630ffd064b74f1c01bcf3),固定提交 `4c9e09931a7814d7500630ffd064b74f1c01bcf3`。目录中存在 `papers/uai-YYYY/papers.jsonl` 与四个 `raw/uaiYYYY.jsonl`,不能仅凭数据集说明宣称拒稿覆盖。
|
||||
|
||||
| 年份 | 整理版及原始论坛数 | 原始回复对象 | 实际回复角色 | 标准结构化决定字段 |
|
||||
| --- | ---: | ---: | --- | --- |
|
||||
| 2022 | 230 | 1059 | Official_Review 829,Decision 230 | Accept (Poster) 194,Accept (Oral) 36 |
|
||||
| 2023 | 243 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||
| 2024 | 200 | 1994 | Official_Review 905,Rebuttal 889,Meta_Review 200 | 没有单独 Decision 记录或结构化 decision 字段 |
|
||||
| 2025 | 229 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||
|
||||
整理版902条均被整理者标为 `accepted=true`;原始根记录的 `venue` 为相应会议名称或 oral/poster/spotlight 文本,不能替代逐次决定证据,更不能算作902条已认证的原始录用决定。这不是四届全部投稿或全部录用的完整性证明。2024年还有 `Edit` 邀请,属于同一批回复上的另一个邀请,不能再累加成1994个新回复。
|
||||
|
||||
为排除字段漏读,对含回复的2022、2024快照另做了全部3053个回复的字段名、包装结构、邀请角色和短决定类字段核查;重新读取的正文哈希与首次捕获完全一致。2022的230条明确字段是录用标签。2024仅保存评审、作者回复和元评审,没有可据此新增的结构化拒稿/撤稿决定。没有从元评审文本、评分、推荐或缺少录用推断负面决定,也没有保存评审正文、作者回复正文或论文全文。
|
||||
|
||||
**这只描述上述固定镜像子集,不证明UAI没有公开拒稿。** 本批没有生成负面决定导入队列,也未写数据库。2020、2021、2026及其他可用来源仍需分别核查;某文件未出现在这个镜像目录中不代表全网不存在。
|
||||
|
||||
## 产物与恢复
|
||||
|
||||
服务器目录均相对 `/data/paper-index/collected/`:
|
||||
|
||||
- `uai2022-2025-mirror-metadata-v1`:902条整理版元数据投影、字段盘点与标签计数。
|
||||
- `uai2022-2025-raw-decisions-v1`:902个原始论坛的元数据、明确决定字段及来源正文/逐行哈希。
|
||||
- `uai2022-2024-reply-schema-v2`:3053个回复的元数据、字段名与短标签;评审正文不保存。角色核查包含直接记录和 `note` 包装,未发生静默遗漏。
|
||||
|
||||
源响应正文仅在内存中读取、校验,保存元数据投影与固定来源URL/字节哈希;不是保留了含评审文本的完整原始快照。复核原始源字节时应使用固定提交,并区分投影文件哈希与源正文哈希。恢复包已在本机独立核验全部902个论坛、3053个回复的元数据结构、逐文件哈希及来源关联。
|
||||
|
||||
| 恢复证据 | SHA256 |
|
||||
| --- | --- |
|
||||
| 整理版元数据恢复包 | 55889fb2b9005260cef1001b2cf36e203f3b7e1deeec0291f058b52eeae880e7 |
|
||||
| 原始元数据/回复结构恢复包 | caade7df899dd3e65572d06e2bf2326fad692cc5e1c1e4ab002fdc6d7823743f |
|
||||
| 原始结构独立核验报告 | 5f0ef2948d0db68fe0cd89eb4c8b57a1088b94b53bed98f180d2b96629746e72 |
|
||||
|
||||
服务器恢复包分别为 `/data/paper-index/validation/uai2022-2025-mirror-metadata-recovery-v1.tar.gz` 与 `uai2022-2025-raw-reply-recovery-v1.tar.gz`。本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。先检查实际单元、完整manifest和哈希,再恢复;不要因进度文件旧状态或日志安静重跑。
|
||||
|
||||
后续继续寻找能明确覆盖作者公开拒稿选择的来源,保留每次投稿及决定原文标签、原始ID与镜像关系,核对跨来源重叠和后续录用。UAI缺口不关闭,全部43个目标的逐年公开负面决定核查也不缩减;最终全库去重仍为独立必做阶段。
|
||||
@@ -1,65 +0,0 @@
|
||||
# MM 2026 出版条目采集与恢复
|
||||
|
||||
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
|
||||
|
||||
## 使用
|
||||
|
||||
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
|
||||
|
||||
```sh
|
||||
paper-atlas collect mm2026-publications \
|
||||
--registry-snapshots /data/paper-index/collected \
|
||||
--program-snapshots /data/paper-index/collected \
|
||||
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||
--out /data/paper-index/collected/mm2026-publications-native-v1
|
||||
```
|
||||
|
||||
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||
|
||||
## 注册目录与节目分别保留
|
||||
|
||||
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
|
||||
|
||||
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
|
||||
|
||||
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
|
||||
|
||||
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
|
||||
|
||||
## 分轨、摘要与版本差异
|
||||
|
||||
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
|
||||
|
||||
| 输出分区 | 注册条目 |
|
||||
| --- | ---: |
|
||||
| main | 1533 |
|
||||
| grand_challenge | 77 |
|
||||
| interactive_art | 17 |
|
||||
| datasets | 40 |
|
||||
| reproducibility | 1 |
|
||||
| bni | 22 |
|
||||
| open_source | 11 |
|
||||
| doctoral_symposium | 2 |
|
||||
| demo_and_video | 20 |
|
||||
| industry_talk | 3 |
|
||||
| proceedings_unclassified | 192 |
|
||||
|
||||
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
|
||||
|
||||
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
|
||||
|
||||
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||
|
||||
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`crossref:type` 与 `mm_program:track` 分别保留来源命名空间,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||
|
||||
## 只读数据库预检与导入边界
|
||||
|
||||
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
|
||||
|
||||
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
|
||||
|
||||
## 恢复
|
||||
|
||||
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
|
||||
|
||||
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||
@@ -108,3 +108,32 @@ paper-atlas audit source-identities \
|
||||
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||
|
||||
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||
|
||||
## 已完成的规范化、只读审计与出版核对
|
||||
|
||||
采集器提交 `c8db611` 和直接节目审计提交 `09a2d4e` 已通过PR #65、#66由用户合并。授权远程runner的535/543项测试、wheel构建安装和安装后全量命令检查通过;没有本机测试或构建。服务器规范化和只读候选报告也已独立验收,并在本机备份、复核全部2143项与814个候选证据包。
|
||||
|
||||
| 只读候选分类 | 来源节目项数 |
|
||||
| --- | ---: |
|
||||
| 标题与完整作者多重集合一致,仍需作品核查 | 660 |
|
||||
| 仅部分作者重叠 | 153 |
|
||||
| 无精确标题候选,不证明是新作品 | 1329 |
|
||||
| 有标题候选但无作者一致性 | 1 |
|
||||
|
||||
全部4326个展示上下文一致,814个证据包均无截断。24篇候选作品涉及24次已有负面投稿、37条负面来源观察;37不能直接算作37次拒稿。所有候选均尚未批准身份或入库,旧负面决定必须在之后关联时保留。
|
||||
|
||||
对已有作品的原始arXiv记录、标题/完整作者多重集合和MM2026语境进一步筛查,得到67个 Main Track 出版DOI核对候选。独立查询其明确给出的DOI及父论文集,共68次单次请求,67次HTTP200、1次404,没有认证或重试。父注册记录为[第34届ACM Multimedia论文集](https://doi.org/10.1145/3767308),ISBN `9798400722134`,出版者ACM,出版年2026。
|
||||
|
||||
其中66项的原始arXiv引用DOI与注册DOI一致,按既有审计规则规范化的标题、完整作者多重集合、出版年、出版者、明确会议名和论文集标题也一致。注册文章未提供ISBN,不能伪称逐篇ISBN已核验;归属依据是文章明确给出的会议和论文集字段,以及独立的原始arXiv语境和DOI。仍需逐项作品/版本审查及新鲜数据库保护检查,**不是66项已入库或最终身份批准**。
|
||||
|
||||
`mfp7067` 对应既有候选作品2749947,原始arXiv提供的 `10.1145/3767308.3836169` 返回404,暂缓。404不是撤稿/拒稿证据,也不据此删除或修正既有标识;需独立核对后再处理。
|
||||
|
||||
完整会议出版名单仍需继续核对。实际ISBN查询只返回父论文集一个条目,父DOI的公开Unixref XML也只含该父DOI、0个 `conference_paper` 节点;都不能代表逐篇名单。按会议名检索得到的 `total-results` 是出版者全年范围内的宽泛检索量,不能当作MM篇数;必须逐项核对明确会议/论文集字段、节目与分轨归属,并保留未匹配项。全部28个来源track继续保留,不能把本批Main Track候选当成会议完整覆盖。
|
||||
|
||||
| 已独立验证的恢复证据 | SHA256 |
|
||||
| --- | --- |
|
||||
| 全轨规范化恢复包 | b119081618b8ce98037dc139be159512f9fdb86a0ccfb127c8b8a1c5722f3d87 |
|
||||
| 只读候选/814证据包恢复包 | 42d0d0404bba04e64e638344b050ad4e7287c3e4c75a777b7069ecc7d7a6ed60 |
|
||||
| 67项DOI及父论文集原始注册回执恢复包 | dd7a0ff76f12070f403cc7352ac54269e454758d807896a086c58c1c1ac7b798 |
|
||||
|
||||
服务器恢复包位于 `/data/paper-index/validation/`,分别名为 `mm2026-native-recovery-v1.tar.gz`、`mm2026-preflight-recovery-v1.tar.gz`、`mm2026-publication-metadata-recovery-v1.tar.gz`;本地同名备份在 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。来源目录为 `collected/mm2026-native-v1`、`mm2026-preflight-v1`、`mm2026-publication-metadata-v1`。先核对完整manifest及实际单元结果,不能用旧进度文件的running字样重启。其他会议已消费的导入队列仍禁止重放;本轮均为来源核查、只读候选审计,没有生产写入。
|
||||
|
||||
@@ -33,7 +33,6 @@ COMMANDS = {
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
|
||||
@@ -1,227 +0,0 @@
|
||||
"""Frozen MM 2026 registered publications with separately attributed program evidence.
|
||||
|
||||
All publications survive normalization. A program metadata match is not a
|
||||
scientific-work identity decision; publisher article type is not research type.
|
||||
"""
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.collectors import mm2026 as program
|
||||
|
||||
SOURCE = 'mm_registry_program'
|
||||
SNAPSHOT = 'mm2026-container-registry-v2'
|
||||
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
|
||||
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
|
||||
PARENT_DOI = '10.1145/3767308'
|
||||
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
|
||||
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
|
||||
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
|
||||
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
|
||||
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
|
||||
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
|
||||
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
|
||||
'publisher','member','type','container-title','short-container-title','event','abstract',
|
||||
'page','ISBN','isbn-type','published','published-print','published-online','issued',
|
||||
'license','resource','relation','alternative-id'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def key(text):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def author_names(values):
|
||||
return Counter(key(v) for v in values)
|
||||
|
||||
|
||||
def load_registry(root, parent_path):
|
||||
directory = Path(root)/SNAPSHOT
|
||||
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
|
||||
raw_parent = program.safe_read(Path(parent_path), 128*1024)
|
||||
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
|
||||
raise ValueError('Reviewed registry or parent hash changed')
|
||||
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
|
||||
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
|
||||
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
|
||||
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
|
||||
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
|
||||
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
|
||||
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
|
||||
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
|
||||
raise ValueError('Incomplete reviewed registry or wrong proceedings')
|
||||
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
|
||||
for number,receipt in enumerate(manifest['pages'],1):
|
||||
name=receipt['file']
|
||||
if name != f'page-{number:03}.json':
|
||||
raise ValueError('Unexpected registry page filename')
|
||||
body=program.safe_read(directory/name,8*1024**2)
|
||||
url=urlsplit(receipt['url']);query=parse_qs(url.query)
|
||||
expected_filter='container-title:'+CONTAINER+',member:320'
|
||||
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
|
||||
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
|
||||
or query['rows'] != ['250'] or not query['cursor'][0]
|
||||
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||
raise ValueError('Registry receipt and source body disagree')
|
||||
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
|
||||
if (obj['status'] != 'ok' or message['total-results'] != 1918
|
||||
or len(message['items']) != expected or receipt['items'] != expected):
|
||||
raise ValueError('Incomplete registry pagination')
|
||||
evidence[name]=body
|
||||
for position,original in enumerate(message['items']):
|
||||
doi=original.get('DOI')
|
||||
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
|
||||
raise ValueError('Missing, foreign or repeated publication DOI')
|
||||
seen.add(doi)
|
||||
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
|
||||
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
|
||||
or original.get('member') != '320' or original.get('publisher') != 'ACM'
|
||||
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
|
||||
raise ValueError('Wrong registered publication context or changed abstract availability')
|
||||
titles=original.get('title');authors=original.get('author')
|
||||
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
|
||||
or not isinstance(authors,list) or not authors):
|
||||
raise ValueError('Missing publication title or authors')
|
||||
names=[];flags=[]
|
||||
for i,author in enumerate(authors,1):
|
||||
family=author.get('family');given=author.get('given','')
|
||||
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
|
||||
raise ValueError('Invalid registered author fields')
|
||||
name=' '.join(x.strip() for x in (given,family) if x.strip())
|
||||
if '@' in name or name.casefold()=='anonymous':
|
||||
raise ValueError('Unreviewed author identity')
|
||||
names.append(name)
|
||||
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
|
||||
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
|
||||
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
|
||||
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
|
||||
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
|
||||
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
|
||||
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
|
||||
'receipt':receipt,'source_position':position})
|
||||
if len(rows)!=1918:
|
||||
raise ValueError('Incomplete registered publication census')
|
||||
return evidence,rows
|
||||
|
||||
|
||||
def correlate(rows, observations):
|
||||
titles=defaultdict(list)
|
||||
for row in observations:titles[key(row['title'])].append(row)
|
||||
matched=set();kinds=Counter()
|
||||
for row in rows:
|
||||
candidates=titles[key(row['title'])]
|
||||
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||
if len(exact)==1:
|
||||
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
|
||||
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||
if exact[0]['track'] not in TRACKS:
|
||||
raise ValueError('New registered program track requires review')
|
||||
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||
elif candidates:row['kind']='program_author_version_requires_review'
|
||||
else:row['kind']='no_exact_program_title_requires_review'
|
||||
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||
kinds[row['kind']]+=1
|
||||
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
|
||||
if kinds != expected:
|
||||
raise ValueError('Reviewed program/publication correspondence changed')
|
||||
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||
|
||||
|
||||
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
out=Path(out)
|
||||
if out.exists() or out.is_symlink():
|
||||
raise ValueError('Output exists; inspect prior operation, never overwrite')
|
||||
if any(p.is_symlink() for p in out.parents):
|
||||
raise ValueError('Unsafe output parent')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
|
||||
program_evidence=program.load_snapshot(program_snapshots)
|
||||
observations,_=program.parse_program(program_evidence['program-data.json'])
|
||||
kinds,unmatched=correlate(publications,observations)
|
||||
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
|
||||
directory=out/'evidence'/group;directory.mkdir(parents=True)
|
||||
for name,body in files.items():(directory/name).write_bytes(body)
|
||||
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
|
||||
for item in sorted(publications,key=lambda r:r['doi']):
|
||||
p=item.get('program');doi=item['doi'];receipt=item['receipt']
|
||||
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
|
||||
abstract=p['abstract'] if p and p['abstract'] else None
|
||||
missing+=abstract is None;copied+=abstract is not None
|
||||
if p:known_tracks[p['track']]+=1
|
||||
tags=[{'namespace':'crossref:type','value':'proceedings-article'}]
|
||||
if p:tags.append({'namespace':'mm_program:track','value':p['track']})
|
||||
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
|
||||
'title':item['title'],'authors':item['authors'],'abstract':abstract,
|
||||
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
|
||||
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
|
||||
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
|
||||
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
|
||||
'program_fields':p,'program_match_kind':item['kind'],
|
||||
'program_title_candidates':item['program_title_candidates'],
|
||||
'program_exact_candidates':item['program_exact_candidates'],
|
||||
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
|
||||
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||
'review_flags':item['flags'],
|
||||
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
|
||||
outputs=[]
|
||||
for track,rows in sorted(buckets.items()):
|
||||
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
|
||||
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
|
||||
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
|
||||
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
|
||||
raise ValueError('Reviewed output partition or abstract census changed')
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||
parser.add_argument('--registry-snapshots',required=True)
|
||||
parser.add_argument('--program-snapshots',required=True)
|
||||
parser.add_argument('--parent-snapshot',required=True)
|
||||
parser.add_argument('--out',required=True)
|
||||
args=parser.parse_args(argv)
|
||||
report=collect(args.out,registry_snapshots=args.registry_snapshots,
|
||||
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
|
||||
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
|
||||
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
|
||||
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
|
||||
BIN
Binary file not shown.
@@ -1,155 +0,0 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import mm_publications2026 as c
|
||||
|
||||
|
||||
class MMPublications2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
fixtures=Path(__file__).parent/'fixtures'
|
||||
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||
cls.out=cls.root/'out'
|
||||
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
||||
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
||||
cls.by_doi={r['external_id']:r for r in cls.rows}
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.temp.cleanup()
|
||||
|
||||
@staticmethod
|
||||
def materialize(root,files):
|
||||
for name,body in files.items():
|
||||
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
||||
|
||||
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
||||
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
||||
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
||||
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
||||
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
||||
|
||||
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];m=r['metadata']
|
||||
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||
if g['abstract_sha256'] is not None:
|
||||
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
|
||||
else:
|
||||
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
|
||||
|
||||
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||
for p in self.report['unmatched_program']:
|
||||
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
|
||||
gold=self.program_fixture['independent_inventory']['inventory']
|
||||
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||
for g in gold:
|
||||
p=programs[g['paper_id']]
|
||||
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
||||
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
||||
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
||||
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
||||
|
||||
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
||||
for r in self.rows:
|
||||
doi=r['external_id'];m=r['metadata']
|
||||
self.assertEqual(r['identifiers'],{'doi':doi})
|
||||
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
||||
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
||||
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
||||
self.assertFalse(m['scientific_work_identity_approved'])
|
||||
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
||||
self.assertIn({'namespace':'crossref:type','value':'proceedings-article'},r['tags'])
|
||||
self.assertTrue({x['namespace'] for x in r['tags']} <= {'crossref:type','mm_program:track'})
|
||||
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||
self.assertEqual(len(variants),192)
|
||||
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||
|
||||
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||
authors_by_page={}
|
||||
for name,body in self.fixture['files'].items():
|
||||
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
||||
self.assertEqual(actual.read_bytes(),body.encode())
|
||||
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
||||
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
||||
for name,body in self.program_fixture['files'].items():
|
||||
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
||||
for r in self.rows:
|
||||
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
||||
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
||||
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
||||
for f in self.report['files']:
|
||||
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
||||
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
||||
|
||||
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||
n=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||
for i in actual:
|
||||
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
||||
self.assertEqual(n,8)
|
||||
|
||||
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||
count+=1;r=self.by_doi[g['doi']]
|
||||
self.assertEqual(r['title'],g['title'])
|
||||
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||
|
||||
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
||||
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
||||
path.write_bytes(path.read_bytes()+b' ')
|
||||
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
||||
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
||||
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
||||
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
||||
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
||||
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||
|
||||
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
|
||||
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
|
||||
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||
self.assertIn('unknown',self.report['public_negative_availability'])
|
||||
Reference in New Issue
Block a user