Compare commits
13 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 2000d340fa | |||
| f982851560 | |||
| 88aeb4d2eb | |||
| 83f3ebd26b | |||
| 13536d6268 | |||
| 7e63035570 | |||
| 44f4146f3e | |||
| b0bd9b69dc | |||
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| 8c147f5e6e |
@@ -19,8 +19,9 @@ jobs:
|
|||||||
# Public repository archive: no owner credentials or runner SSH keys.
|
# Public repository archive: no owner credentials or runner SSH keys.
|
||||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||||
with urllib.request.urlopen(url, timeout=60) as response:
|
with urllib.request.urlopen(url, timeout=60) as response:
|
||||||
archive = response.read(16*1024*1024+1)
|
# Full publication-source fixtures retain original bibliographic evidence.
|
||||||
if len(archive)>16*1024*1024:
|
archive = response.read(24*1024*1024+1)
|
||||||
|
if len(archive)>24*1024*1024:
|
||||||
raise SystemExit('Unexpectedly large source archive')
|
raise SystemExit('Unexpectedly large source archive')
|
||||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||||
root = pathlib.Path(tmp)
|
root = pathlib.Path(tmp)
|
||||||
@@ -46,6 +47,53 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
mm_snapshots = root/'mm2026-snapshots'
|
||||||
|
for name, body in mm_fixture['files'].items():
|
||||||
|
path = mm_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||||
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||||
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||||
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||||
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||||
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
import hashlib
|
||||||
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||||
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||||
|
row = mm_rows[gold['paper_id']]
|
||||||
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||||
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||||
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||||
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||||
|
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
||||||
|
mm_pub_registry = root/'mm2026-publication-registry'
|
||||||
|
for name,body in mm_pub_fixture['files'].items():
|
||||||
|
path=mm_pub_registry/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||||
|
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||||
|
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
|
||||||
|
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||||
|
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
assert len(mm_pub_rows)==1918
|
||||||
|
for gold in mm_pub_fixture['independent_gold']:
|
||||||
|
row=mm_pub_rows[gold['doi']]
|
||||||
|
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||||
|
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||||
|
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
||||||
|
if service_file.is_file():
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||||
|
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||||
|
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||||
|
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
@@ -166,6 +214,21 @@ jobs:
|
|||||||
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
ismir_snapshots = root/'ismir2026-snapshots'
|
||||||
|
ismir_snapshots.mkdir()
|
||||||
|
for name,encoded in ismir_fixture['files'].items():
|
||||||
|
target = ismir_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||||
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||||
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||||
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||||
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||||
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||||
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
|
|||||||
@@ -117,6 +117,8 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
|||||||
|
|
||||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||||
|
|
||||||
|
完整出版来源的新旧版本导致分轨或摘要变化时,使用 [source-revisions 流程](docs/source-revisions.md):核对同一 DOI 全集,保存完整修正前数据与版本链,支持已有记录的目标 track,按新版本重算分区覆盖数,并阻止旧版元数据刷新。普通增量导入仍禁止缩小预期数量。
|
||||||
|
|
||||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||||
|
|
||||||
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||||
@@ -131,6 +133,8 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
|||||||
|
|
||||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||||
|
|
||||||
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||||
|
|
||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
@@ -138,3 +142,7 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
|||||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|
||||||
|
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# 会议覆盖与公开决定核查
|
# 会议覆盖与公开决定核查
|
||||||
|
|
||||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
最新生产验收与剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):当日 `14:06:28 UTC` 固定范围快照中,278 个会议年份槽位有 272 个含记录、6 个仍为空;这不代表完整覆盖,第 14 步仍未完成。后续 MM 规范化、只读候选审计与出版核查不等于生产导入。
|
||||||
|
|
||||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||||
|
|
||||||
@@ -72,7 +72,7 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
|
|||||||
|
|
||||||
## 优先缺口
|
## 优先缺口
|
||||||
|
|
||||||
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,当前主库拒稿仍为零。原 notes 接口要求 Challenge 验证,公开镜像尚无已核验来源,必须保留这个真实缺口;不能根据 PMLR 录用名单推断拒稿。参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,首份生产快照拒稿为零。原 notes 接口要求 Challenge 验证;新核验镜像的 2022–2025 年 902 个论坛来自录用子集,不能填补公开拒稿缺口。2024 年的元评审也不能直接转成最终决定,见 [2026-11-17 原始来源与回复结构核查](uai-public-negative-source-status-2026-11-17.md)。不能根据 PMLR 或镜像录用名单反推拒稿;政策参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
||||||
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
||||||
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
||||||
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
||||||
|
|||||||
@@ -33,7 +33,7 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
|
|||||||
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||||
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||||
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||||
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) |
|
| MM | 全28轨2143个节目ID/4326展示位置已规范化;814个只读候选历史证据包已核验,24篇候选涉及24次负面投稿/37条来源观察;67项原始arXiv DOI中66项注册元数据一致、1项404暂缓 | 不是2143篇已核验科研论文。继续逐轨贡献/出版依据与作品审查,0身份批准、0入库。ISBN查询与父XML均只返回父记录,不作逐篇清单。见 [来源、审计与恢复证据](../mm2026-source-status.md) |
|
||||||
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||||
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||||
|
|
||||||
@@ -43,6 +43,8 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
|
|||||||
|
|
||||||
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||||
|
|
||||||
|
UAI新增固定镜像已核对2022–2025的902个原始论坛及3053个回复的元数据/字段结构。整理版均标录用,2022保存230条明确Accept决定;2024仅有评审、作者回复、元评审,2023/2025未保存回复,不能据此补拒稿。来源限制和恢复见 [逐年原始核查](uai-public-negative-source-status-2026-11-17.md)。这不是UAI没有公开拒稿的结论,缺口保持未补齐。
|
||||||
|
|
||||||
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||||
|
|
||||||
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||||
|
|||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# UAI 公开负面决定来源:2026-11-17 核查
|
||||||
|
|
||||||
|
**公开拒稿缺口仍未补齐,总体仍为 13/17。** 本页保存新来源的实际范围与恢复线索,避免下次把录用子集误当作全部投稿。主动补漏范围仍从2020年开始;本次镜像只含2022–2025,不能替代其他年份或其他会议的核查。
|
||||||
|
|
||||||
|
## 政策与访问边界
|
||||||
|
|
||||||
|
已有逐年核查记录保存在 [覆盖清单](inventory-2026-11-08.json) 的 UAI `policy_checks`:2023–2026 官方说明允许决定后公开录用及作者选择公开的拒稿,默认私密拒稿不属于公开可采集范围。政策并不证明任何镜像完整。
|
||||||
|
|
||||||
|
- [2023 投稿说明](https://www.auai.org/uai2023/submission_instructions)
|
||||||
|
- [2024 投稿说明](https://auai.org/uai2024/submission_instructions)
|
||||||
|
- [2025 投稿说明](https://www.auai.org/uai2025/submission_instructions)
|
||||||
|
- [2026 投稿说明](https://www.auai.org/uai2026/submission_instructions)
|
||||||
|
|
||||||
|
先前匿名 notes 请求遇到 ChallengeRequiredError/403,仍作为访问限制保留。本轮没有认证、重试或绕过这些入口;读取的是整理者公开发布的固定版本镜像。复制的 `readers`、`signatures` 和邀请字段仍只是镜像内的来源断言,不是对当前原始接口可见性或签名真实性的验证。
|
||||||
|
|
||||||
|
## 实际镜像范围
|
||||||
|
|
||||||
|
来源:[latkes/ai-research-index-iclr-openreview](https://huggingface.co/datasets/latkes/ai-research-index-iclr-openreview/tree/4c9e09931a7814d7500630ffd064b74f1c01bcf3),固定提交 `4c9e09931a7814d7500630ffd064b74f1c01bcf3`。目录中存在 `papers/uai-YYYY/papers.jsonl` 与四个 `raw/uaiYYYY.jsonl`,不能仅凭数据集说明宣称拒稿覆盖。
|
||||||
|
|
||||||
|
| 年份 | 整理版及原始论坛数 | 原始回复对象 | 实际回复角色 | 标准结构化决定字段 |
|
||||||
|
| --- | ---: | ---: | --- | --- |
|
||||||
|
| 2022 | 230 | 1059 | Official_Review 829,Decision 230 | Accept (Poster) 194,Accept (Oral) 36 |
|
||||||
|
| 2023 | 243 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
| 2024 | 200 | 1994 | Official_Review 905,Rebuttal 889,Meta_Review 200 | 没有单独 Decision 记录或结构化 decision 字段 |
|
||||||
|
| 2025 | 229 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
|
||||||
|
整理版902条均被整理者标为 `accepted=true`;原始根记录的 `venue` 为相应会议名称或 oral/poster/spotlight 文本,不能替代逐次决定证据,更不能算作902条已认证的原始录用决定。这不是四届全部投稿或全部录用的完整性证明。2024年还有 `Edit` 邀请,属于同一批回复上的另一个邀请,不能再累加成1994个新回复。
|
||||||
|
|
||||||
|
为排除字段漏读,对含回复的2022、2024快照另做了全部3053个回复的字段名、包装结构、邀请角色和短决定类字段核查;重新读取的正文哈希与首次捕获完全一致。2022的230条明确字段是录用标签。2024仅保存评审、作者回复和元评审,没有可据此新增的结构化拒稿/撤稿决定。没有从元评审文本、评分、推荐或缺少录用推断负面决定,也没有保存评审正文、作者回复正文或论文全文。
|
||||||
|
|
||||||
|
**这只描述上述固定镜像子集,不证明UAI没有公开拒稿。** 本批没有生成负面决定导入队列,也未写数据库。2020、2021、2026及其他可用来源仍需分别核查;某文件未出现在这个镜像目录中不代表全网不存在。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
服务器目录均相对 `/data/paper-index/collected/`:
|
||||||
|
|
||||||
|
- `uai2022-2025-mirror-metadata-v1`:902条整理版元数据投影、字段盘点与标签计数。
|
||||||
|
- `uai2022-2025-raw-decisions-v1`:902个原始论坛的元数据、明确决定字段及来源正文/逐行哈希。
|
||||||
|
- `uai2022-2024-reply-schema-v2`:3053个回复的元数据、字段名与短标签;评审正文不保存。角色核查包含直接记录和 `note` 包装,未发生静默遗漏。
|
||||||
|
|
||||||
|
源响应正文仅在内存中读取、校验,保存元数据投影与固定来源URL/字节哈希;不是保留了含评审文本的完整原始快照。复核原始源字节时应使用固定提交,并区分投影文件哈希与源正文哈希。恢复包已在本机独立核验全部902个论坛、3053个回复的元数据结构、逐文件哈希及来源关联。
|
||||||
|
|
||||||
|
| 恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 整理版元数据恢复包 | 55889fb2b9005260cef1001b2cf36e203f3b7e1deeec0291f058b52eeae880e7 |
|
||||||
|
| 原始元数据/回复结构恢复包 | caade7df899dd3e65572d06e2bf2326fad692cc5e1c1e4ab002fdc6d7823743f |
|
||||||
|
| 原始结构独立核验报告 | 5f0ef2948d0db68fe0cd89eb4c8b57a1088b94b53bed98f180d2b96629746e72 |
|
||||||
|
|
||||||
|
服务器恢复包分别为 `/data/paper-index/validation/uai2022-2025-mirror-metadata-recovery-v1.tar.gz` 与 `uai2022-2025-raw-reply-recovery-v1.tar.gz`。本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。先检查实际单元、完整manifest和哈希,再恢复;不要因进度文件旧状态或日志安静重跑。
|
||||||
|
|
||||||
|
后续继续寻找能明确覆盖作者公开拒稿选择的来源,保留每次投稿及决定原文标签、原始ID与镜像关系,核对跨来源重叠和后续录用。UAI缺口不关闭,全部43个目标的逐年公开负面决定核查也不缩减;最终全库去重仍为独立必做阶段。
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# ISMIR 2026 官方节目与摘要采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect ismir2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录需要三个冻结目录:
|
||||||
|
|
||||||
|
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||||
|
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||||
|
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||||
|
|
||||||
|
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||||
|
|
||||||
|
## 来源与版本
|
||||||
|
|
||||||
|
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||||
|
|
||||||
|
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||||
|
|
||||||
|
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||||
|
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||||
|
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||||
|
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||||
|
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||||
|
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||||
|
|
||||||
|
## 公开评审与拒稿边界
|
||||||
|
|
||||||
|
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||||
|
|
||||||
|
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||||
|
|
||||||
|
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||||
|
|
||||||
|
## 恢复与验证
|
||||||
|
|
||||||
|
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||||
|
|
||||||
|
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||||
|
|
||||||
|
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||||
|
|
||||||
|
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# MM 2026 出版条目采集与恢复
|
||||||
|
|
||||||
|
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-publications \
|
||||||
|
--registry-snapshots /data/paper-index/collected \
|
||||||
|
--program-snapshots /data/paper-index/collected \
|
||||||
|
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||||
|
--out /data/paper-index/collected/mm2026-publications-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||||
|
|
||||||
|
## 注册目录与节目分别保留
|
||||||
|
|
||||||
|
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
|
||||||
|
|
||||||
|
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
|
||||||
|
|
||||||
|
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
|
||||||
|
|
||||||
|
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
|
||||||
|
|
||||||
|
## 分轨、摘要与版本差异
|
||||||
|
|
||||||
|
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
|
||||||
|
|
||||||
|
| 输出分区 | 注册条目 |
|
||||||
|
| --- | ---: |
|
||||||
|
| main | 1533 |
|
||||||
|
| grand_challenge | 77 |
|
||||||
|
| interactive_art | 17 |
|
||||||
|
| datasets | 40 |
|
||||||
|
| reproducibility | 1 |
|
||||||
|
| bni | 22 |
|
||||||
|
| open_source | 11 |
|
||||||
|
| doctoral_symposium | 2 |
|
||||||
|
| demo_and_video | 20 |
|
||||||
|
| industry_talk | 3 |
|
||||||
|
| proceedings_unclassified | 192 |
|
||||||
|
|
||||||
|
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
|
||||||
|
|
||||||
|
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
|
||||||
|
|
||||||
|
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||||
|
|
||||||
|
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||||
|
|
||||||
|
## 只读数据库预检与导入边界
|
||||||
|
|
||||||
|
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
|
||||||
|
|
||||||
|
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
|
||||||
|
|
||||||
|
## 恢复
|
||||||
|
|
||||||
|
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
|
||||||
|
|
||||||
|
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
|
||||||
@@ -73,4 +73,67 @@
|
|||||||
|
|
||||||
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||||
|
|
||||||
本批尚未形成生产导入队列。下一步需要可复用采集器保留每个节目 ID 的全部展示上下文、明确贡献类型与来源边界,单列缺失字段,再做作品/出版身份及既往投稿审查。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
## 可复用采集命令
|
||||||
|
|
||||||
|
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/mm2026-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||||
|
|
||||||
|
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||||
|
|
||||||
|
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||||
|
|
||||||
|
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||||
|
|
||||||
|
## 直接审计节目身份与既往历史
|
||||||
|
|
||||||
|
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||||
|
paper-atlas audit source-identities \
|
||||||
|
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||||
|
|
||||||
|
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||||
|
|
||||||
|
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||||
|
|
||||||
|
## 已完成的规范化、只读审计与出版核对
|
||||||
|
|
||||||
|
采集器提交 `c8db611` 和直接节目审计提交 `09a2d4e` 已通过PR #65、#66由用户合并。授权远程runner的535/543项测试、wheel构建安装和安装后全量命令检查通过;没有本机测试或构建。服务器规范化和只读候选报告也已独立验收,并在本机备份、复核全部2143项与814个候选证据包。
|
||||||
|
|
||||||
|
| 只读候选分类 | 来源节目项数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| 标题与完整作者多重集合一致,仍需作品核查 | 660 |
|
||||||
|
| 仅部分作者重叠 | 153 |
|
||||||
|
| 无精确标题候选,不证明是新作品 | 1329 |
|
||||||
|
| 有标题候选但无作者一致性 | 1 |
|
||||||
|
|
||||||
|
全部4326个展示上下文一致,814个证据包均无截断。24篇候选作品涉及24次已有负面投稿、37条负面来源观察;37不能直接算作37次拒稿。所有候选均尚未批准身份或入库,旧负面决定必须在之后关联时保留。
|
||||||
|
|
||||||
|
对已有作品的原始arXiv记录、标题/完整作者多重集合和MM2026语境进一步筛查,得到67个 Main Track 出版DOI核对候选。独立查询其明确给出的DOI及父论文集,共68次单次请求,67次HTTP200、1次404,没有认证或重试。父注册记录为[第34届ACM Multimedia论文集](https://doi.org/10.1145/3767308),ISBN `9798400722134`,出版者ACM,出版年2026。
|
||||||
|
|
||||||
|
其中66项的原始arXiv引用DOI与注册DOI一致,按既有审计规则规范化的标题、完整作者多重集合、出版年、出版者、明确会议名和论文集标题也一致。注册文章未提供ISBN,不能伪称逐篇ISBN已核验;归属依据是文章明确给出的会议和论文集字段,以及独立的原始arXiv语境和DOI。仍需逐项作品/版本审查及新鲜数据库保护检查,**不是66项已入库或最终身份批准**。
|
||||||
|
|
||||||
|
`mfp7067` 对应既有候选作品2749947,原始arXiv提供的 `10.1145/3767308.3836169` 返回404,暂缓。404不是撤稿/拒稿证据,也不据此删除或修正既有标识;需独立核对后再处理。
|
||||||
|
|
||||||
|
完整会议出版名单仍需继续核对。实际ISBN查询只返回父论文集一个条目,父DOI的公开Unixref XML也只含该父DOI、0个 `conference_paper` 节点;都不能代表逐篇名单。按会议名检索得到的 `total-results` 是出版者全年范围内的宽泛检索量,不能当作MM篇数;必须逐项核对明确会议/论文集字段、节目与分轨归属,并保留未匹配项。全部28个来源track继续保留,不能把本批Main Track候选当成会议完整覆盖。
|
||||||
|
|
||||||
|
| 已独立验证的恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 全轨规范化恢复包 | b119081618b8ce98037dc139be159512f9fdb86a0ccfb127c8b8a1c5722f3d87 |
|
||||||
|
| 只读候选/814证据包恢复包 | 42d0d0404bba04e64e638344b050ad4e7287c3e4c75a777b7069ecc7d7a6ed60 |
|
||||||
|
| 67项DOI及父论文集原始注册回执恢复包 | dd7a0ff76f12070f403cc7352ac54269e454758d807896a086c58c1c1ac7b798 |
|
||||||
|
|
||||||
|
服务器恢复包位于 `/data/paper-index/validation/`,分别名为 `mm2026-native-recovery-v1.tar.gz`、`mm2026-preflight-recovery-v1.tar.gz`、`mm2026-publication-metadata-recovery-v1.tar.gz`;本地同名备份在 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。来源目录为 `collected/mm2026-native-v1`、`mm2026-preflight-v1`、`mm2026-publication-metadata-v1`。先核对完整manifest及实际单元结果,不能用旧进度文件的running字样重启。其他会议已消费的导入队列仍禁止重放;本轮均为来源核查、只读候选审计,没有生产写入。
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# 出版来源版本与覆盖修正
|
||||||
|
|
||||||
|
`paper-atlas maintain source-revisions` 用于同一出版清单的新版本:稳定 DOI 全集、标题、作者和决定不变,但官方分轨、摘要或对应来源证据已经过重新核对。它不导入新论文、不合并实体、不推断拒稿,不把出版分类当成研究类型。
|
||||||
|
|
||||||
|
例如,同一清单的“未分类”由 192 条减少到 166 条,而主会等分区增加。直接放松增量导入的计数保护会隐藏清单丢失。本命令先核验完整的新旧清单及数据库快照,再单独修正版本;普通 `import records` 的单调计数规则保持原样。
|
||||||
|
|
||||||
|
## 准备完整证据
|
||||||
|
|
||||||
|
在工作区外保存 inventory JSON,包含以下字段:
|
||||||
|
|
||||||
|
| 字段 | 内容 |
|
||||||
|
| --- | --- |
|
||||||
|
| `format` | `1` |
|
||||||
|
| `source`, `venue`, `year`, `event_key`, `list_url` | 明确的已有来源、会议缩写、年份、事件键和 HTTPS 出版清单地址 |
|
||||||
|
| `previous_rows`, `rows` | 完整旧版、新版规范化来源记录数组,每版最多 10,000 条;每条包含稳定 DOI、来源字段、作者、摘要、track、决定、标签、原始来源 metadata |
|
||||||
|
| `evidence` | 一个或多个 `{ "url": "https://…", "sha256": "…" }`,绑定已独立核验的原始证据文件 |
|
||||||
|
|
||||||
|
两版必须有完全相同的 DOI 集合;仅允许 `abstract`、`track`、`tags`、`metadata` 的有据变化。必须保留 collector 原始字段,不能将工作身份核查产生的额外锚点混入原始来源版本。只有明确 `accepted` 的出版观察可进入本流程。证据 URL/hash 是审核输入,本命令不联网确认它们;执行者必须另行验证文件哈希、完整性及对应关系。
|
||||||
|
|
||||||
|
数据库可以部分导入,甚至各分区的预期数暂时来自不同版本。现有覆盖必须为 `partial`、无错误,导入数与实际来源记录一致;预期数必须恰好属于旧版或新版的同一分区。已有目标 edition 和对应覆盖记录必须存在,目标中可以有其他来源及既有投稿。本版本不会新建 edition,也不会处理共享来源投稿、多次既有 scope curation 的再迁移,或成员集合变化;遇到这些情况会拒绝并要求单独核查。
|
||||||
|
|
||||||
|
## 先生成只读计划
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas maintain source-revisions \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--inventory /data/paper-index/validation/reviewed-source-version.json \
|
||||||
|
--out /data/paper-index/validation/source-revision-plan.json
|
||||||
|
```
|
||||||
|
|
||||||
|
计划包含明确数据库身份、两个完整来源版本、当前版本父节点、覆盖表原值和修正值、逐条变更,以及受影响会议 scope 内所有论文的规范记录、稳定标识、全部跨会议投稿和决定、来源关系、标签与已存在的 annotation。拒稿后录用的历史也在保护范围内,不能只校验当前会议的录用观察。
|
||||||
|
|
||||||
|
计划和原始证据应保存到服务器与独立恢复位置,核对完整差异及计划文件 SHA-256。执行前重新核对生产保护基线与单写入安排。输出文件必须是不存在的新路径,禁止覆盖旧报告。
|
||||||
|
|
||||||
|
## 按文件哈希应用并验收
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas maintain source-revisions \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--plan /data/paper-index/validation/source-revision-plan.json \
|
||||||
|
--plan-sha256 <已审核的计划文件SHA256> \
|
||||||
|
--out /data/paper-index/validation/source-revision-result.json
|
||||||
|
```
|
||||||
|
|
||||||
|
应用时重新生成计划并要求完全一致;PostgreSQL 使用已有 writer 锁及相关表写锁,SQLite 使用立即事务。任何保护数据、当前版本或目标标识冲突都使操作失败。操作保留投稿和决定 ID、原始决定证据、规范论文文本、全部稳定标识、来源关系和现有标签;只修改计划列出的投稿/投稿标识 edition,以及来源记录的摘要与 metadata 当前投影。旧投影完整保存在版本历史中,来源原始获取时间不会伪造为修正时间。
|
||||||
|
|
||||||
|
覆盖只调整明确分区的预期数和来源观察数,保留 `partial`、cursor、成功时间和错误字段原值。尚未导入的分区不产生虚构投稿或来源观察。覆盖数不能被解读为科学作品数量,或所有 track、摘要、负面决定已完整。
|
||||||
|
|
||||||
|
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
|
||||||
|
|
||||||
|
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
|
||||||
|
|
||||||
|
## 后续刷新及限制
|
||||||
|
|
||||||
|
`import records` 检查当前版本的逐条完整来源字段;`Store.record` 也检查原始文本、作者、来源 metadata 和基础标识,防止直接调用绕回旧版摘要。来源字段必须与当前版本相符,可以额外携带已经核验的工作身份/curation 证据,不能替换来源自身字段。相同来源、会议事件和清单地址下未登记的成员必须另行核验新版本。
|
||||||
|
|
||||||
|
版本表通过包内接口追加,未声明数据库管理员的任意 SQL 为安全接口。新来源或新版本在导入前仍需要独立工作身份核查;姓名、标题或无候选都不证明作品身份。实现测试和版本修正验收也不替代所有导入结束后的全库去重、标识冲突、近似标题/作者、摘要、来源覆盖与查询审计。
|
||||||
@@ -74,6 +74,74 @@ def load_inputs(files):
|
|||||||
return records, inputs
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def load_mm_program_inputs(report_path):
|
||||||
|
"""Read every MM observation without turning a program into import records.
|
||||||
|
|
||||||
|
Reconcile the complete output against its frozen raw source before any DB
|
||||||
|
connection. The synthetic unknown status is an audit placeholder only.
|
||||||
|
"""
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
|
||||||
|
report_path = Path(report_path).absolute()
|
||||||
|
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||||
|
report = json.loads(raw_report)
|
||||||
|
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||||
|
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||||
|
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||||
|
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||||
|
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Unreviewed MM observation report')
|
||||||
|
root = report_path.parent
|
||||||
|
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||||
|
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||||
|
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||||
|
raise ValueError('MM raw evidence changed')
|
||||||
|
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||||
|
expected = {r['paper_id']:r for r in expected_rows}
|
||||||
|
records, inputs, seen, tracks = [], [], set(), set()
|
||||||
|
for item in report['files']:
|
||||||
|
track = item.get('track_literal')
|
||||||
|
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||||
|
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||||
|
or item.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Duplicate or unreviewed MM track format')
|
||||||
|
tracks.add(track)
|
||||||
|
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||||
|
path = root/Path(item['file']).name
|
||||||
|
if path.suffix != '.jsonl':
|
||||||
|
raise ValueError('Invalid MM observation filename')
|
||||||
|
raw = mm2026.safe_read(path, 16*1024**2)
|
||||||
|
lines = raw.splitlines(keepends=True)
|
||||||
|
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||||
|
raise ValueError('MM observation file count/hash mismatch')
|
||||||
|
for number,line in enumerate(lines, 1):
|
||||||
|
row = json.loads(line)
|
||||||
|
pid = row.get('paper_id')
|
||||||
|
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||||
|
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||||
|
seen.add(pid)
|
||||||
|
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||||
|
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||||
|
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||||
|
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||||
|
'identifier_hits':[],'title_candidate_ids':[],
|
||||||
|
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'program_id':pid,'program_contexts':row['contexts'],
|
||||||
|
'source_observed_status':row['observed_status'],
|
||||||
|
'missing_source_fields':row['missing_fields'],
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'contribution_type_verified':False})
|
||||||
|
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||||
|
'manifest_sha256':digest(raw_report),'manifest':{
|
||||||
|
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||||
|
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||||
|
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||||
|
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||||
|
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
def _lookup(connection, values, columns, lateral):
|
def _lookup(connection, values, columns, lateral):
|
||||||
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||||
width = len(columns)
|
width = len(columns)
|
||||||
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
|
|||||||
temporary.replace(path)
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
def audit(files, out, *, pg_service, pg_schema):
|
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||||
# Reject input corruption before opening a database connection or reserving output.
|
# Reject input corruption before opening a database connection or reserving output.
|
||||||
records, inputs = load_inputs(files)
|
if bool(files) == bool(mm_program_report):
|
||||||
|
raise ValueError('Choose source files or one MM program report')
|
||||||
|
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||||
out = Path(out)
|
out = Path(out)
|
||||||
out.mkdir(parents=True, exist_ok=False)
|
out.mkdir(parents=True, exist_ok=False)
|
||||||
(out / 'audit.lock').write_text(timestamp() + '\n')
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||||
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
'Name agreement does not prove person or scientific-work identity.',
|
'Name agreement does not prove person or scientific-work identity.',
|
||||||
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||||
'Source/history limits are explicit in compressed entity packets.',
|
'Source/history limits are explicit in compressed entity packets.',
|
||||||
|
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||||
'Not a final whole-database deduplication or coverage audit.']}
|
'Not a final whole-database deduplication or coverage audit.']}
|
||||||
_atomic_json(out / 'postgresql-preflight.json', report)
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||||
return report
|
return report
|
||||||
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
def main(argv=None, *, prog=None):
|
||||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
inputs.add_argument('--files', nargs='+', type=Path)
|
||||||
|
inputs.add_argument('--mm-program-report', type=Path,
|
||||||
|
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||||
parser.add_argument('--pg-service', required=True)
|
parser.add_argument('--pg-service', required=True)
|
||||||
parser.add_argument('--pg-schema', required=True)
|
parser.add_argument('--pg-schema', required=True)
|
||||||
parser.add_argument('--out', required=True, type=Path)
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
mm_program_report=args.mm_program_report)
|
||||||
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||||
@@ -31,6 +32,8 @@ COMMANDS = {
|
|||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
|
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
@@ -43,6 +46,7 @@ COMMANDS = {
|
|||||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||||
|
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
|
||||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||||
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
||||||
|
|||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'ismir_program'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||||
|
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||||
|
'research_completed_not_coverage_or_import_approval', {
|
||||||
|
'ismir-accepted.html': ACCEPTED,
|
||||||
|
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||||
|
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||||
|
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||||
|
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||||
|
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||||
|
}
|
||||||
|
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def title_key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshots(root):
|
||||||
|
root = Path(root)
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ValueError('Unsafe snapshot root')
|
||||||
|
result = {}
|
||||||
|
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
files = {'manifest.json': raw}; receipts = {}
|
||||||
|
for filename, url in urls.items():
|
||||||
|
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||||
|
path = directory/filename
|
||||||
|
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body disagree')
|
||||||
|
files[filename] = body; receipts[filename] = receipt
|
||||||
|
result[name] = (files, receipts)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(literal):
|
||||||
|
# Institution names can contain semicolons; split only outside parentheses.
|
||||||
|
parts = []; depth = 0; start = 0
|
||||||
|
for index, char in enumerate(literal):
|
||||||
|
if char == '(': depth += 1
|
||||||
|
elif char == ')':
|
||||||
|
depth -= 1
|
||||||
|
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||||
|
elif char == ';' and depth == 0:
|
||||||
|
parts.append(literal[start:index]); start = index+1
|
||||||
|
if depth:
|
||||||
|
raise ValueError('Unbalanced author affiliation')
|
||||||
|
parts.append(literal[start:])
|
||||||
|
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||||
|
if any(not name for name in names):
|
||||||
|
raise ValueError('Missing accepted-list author')
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_listing(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); tables = []
|
||||||
|
for table in doc.root.walk():
|
||||||
|
if table.tag != 'table': continue
|
||||||
|
rows = []
|
||||||
|
for row in table.walk():
|
||||||
|
if row.tag == 'tr':
|
||||||
|
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||||
|
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||||
|
tables.append(rows)
|
||||||
|
rows = one(tables)[1:]; result = {}
|
||||||
|
if len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete official accepted-list census')
|
||||||
|
for row in rows:
|
||||||
|
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||||
|
raise ValueError('Missing or ambiguous accepted-list title')
|
||||||
|
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(rows, listing):
|
||||||
|
if not isinstance(rows, list) or len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete public program census')
|
||||||
|
observations = []; seen = set(); used = set()
|
||||||
|
for original in rows:
|
||||||
|
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||||
|
raise ValueError('Invalid public program record')
|
||||||
|
pid = original.get('id'); content = original.get('content', {})
|
||||||
|
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||||
|
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||||
|
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||||
|
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||||
|
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||||
|
raise ValueError('Missing program title, author sequence or abstract')
|
||||||
|
key = title_key(title)
|
||||||
|
if key not in listing or key in used:
|
||||||
|
raise ValueError('Program/accepted-list title membership mismatch')
|
||||||
|
used.add(key); accepted = listing[key]
|
||||||
|
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||||
|
session = content.get('session'); reviews = original.get('reviews')
|
||||||
|
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||||
|
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||||
|
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||||
|
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||||
|
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||||
|
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||||
|
or any(not isinstance(v, str) for v in reviews.values())
|
||||||
|
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||||
|
raise ValueError('Unreviewed opt-in review structure')
|
||||||
|
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||||
|
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||||
|
raise ValueError('Review publication preference/body disagree')
|
||||||
|
flags = []
|
||||||
|
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||||
|
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||||
|
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||||
|
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||||
|
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||||
|
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||||
|
raise ValueError('Unreviewed source-version or opt-in census')
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
sources = load_snapshots(snapshots)
|
||||||
|
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||||
|
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||||
|
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||||
|
raise ValueError('Unreviewed public program data path or poster link template')
|
||||||
|
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, (files, _) in sources.items():
|
||||||
|
directory = evidence/name; directory.mkdir()
|
||||||
|
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for observation in observations:
|
||||||
|
original = observation['original']; content = original['content']; pid = original['id']
|
||||||
|
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||||
|
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||||
|
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||||
|
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||||
|
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||||
|
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||||
|
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||||
|
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||||
|
'original_decision_signature_verified':False,'paper_type':None,
|
||||||
|
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||||
|
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||||
|
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||||
|
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||||
|
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||||
|
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||||
|
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||||
|
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||||
|
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||||
|
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||||
|
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||||
|
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
SOURCE = 'mm_program'
|
||||||
|
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||||
|
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||||
|
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||||
|
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||||
|
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||||
|
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||||
|
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||||
|
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||||
|
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||||
|
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||||
|
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||||
|
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def safe_read(path, limit):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||||
|
raise ValueError('Unsafe snapshot path or size')
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||||
|
raise ValueError('Unreviewed manifest scope')
|
||||||
|
evidence = {'manifest.json':raw}
|
||||||
|
for receipt in manifest['entries']:
|
||||||
|
name = receipt['file']
|
||||||
|
if Path(name).name != name or name in evidence:
|
||||||
|
raise ValueError('Unsafe or duplicate receipt name')
|
||||||
|
body = safe_read(directory/name, 12*1024**2)
|
||||||
|
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['content_type'].startswith('application/json')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if name == 'program-data.json':
|
||||||
|
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||||
|
raise ValueError('Wrong program source')
|
||||||
|
elif receipt['http_status'] != 403:
|
||||||
|
raise ValueError('Unreviewed public-access result')
|
||||||
|
evidence[name] = body
|
||||||
|
if 'program-data.json' not in evidence:
|
||||||
|
raise ValueError('Missing program source')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def without_emails(value):
|
||||||
|
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [without_emails(v) for v in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def program_locations(node, pointer=''):
|
||||||
|
"""Independent structural guard against silently skipping a new nesting form."""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
if 'paper_id' in node:
|
||||||
|
yield pointer
|
||||||
|
for key,value in node.items():
|
||||||
|
escaped = key.replace('~','~0').replace('/','~1')
|
||||||
|
yield from program_locations(value, pointer+'/'+escaped)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for i,value in enumerate(node):
|
||||||
|
yield from program_locations(value, pointer+'/'+str(i))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
data = json.loads(raw)
|
||||||
|
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||||
|
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||||
|
or len(data.get('entries',[])) != 157):
|
||||||
|
raise ValueError('Unreviewed program edition or census')
|
||||||
|
records = {}; sessions = []; seen = []
|
||||||
|
|
||||||
|
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||||
|
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||||
|
raise ValueError('Unreviewed program fields')
|
||||||
|
for key in ('paper_id','title','abstract','track'):
|
||||||
|
if not isinstance(paper[key],str):
|
||||||
|
raise ValueError('Wrong program field type')
|
||||||
|
for key in ('authors','affiliations'):
|
||||||
|
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||||
|
raise ValueError('Wrong author or affiliation list')
|
||||||
|
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||||
|
raise ValueError('Missing identity/title or unreviewed source track')
|
||||||
|
core = {k:paper[k] for k in CORE}
|
||||||
|
pid = paper['paper_id']; seen.append(pointer)
|
||||||
|
if pid not in records:
|
||||||
|
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||||
|
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||||
|
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||||
|
'observed_status':'listed_in_official_program',
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'approved_for_import':False,'missing_fields':missing,
|
||||||
|
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||||
|
'contexts':[]}
|
||||||
|
elif {k:records[pid][k] for k in CORE} != core:
|
||||||
|
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||||
|
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||||
|
'subsession_pointer':subsession_pointer,
|
||||||
|
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||||
|
|
||||||
|
for i,entry in enumerate(data['entries']):
|
||||||
|
pointer = '/entries/'+str(i)
|
||||||
|
if entry['track'] != '':
|
||||||
|
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||||
|
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||||
|
session.update(source_pointer=pointer,subsessions=[])
|
||||||
|
for j,paper in enumerate(entry.get('papers') or []):
|
||||||
|
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||||
|
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||||
|
sp = pointer+'/subsessions/'+str(j)
|
||||||
|
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||||
|
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||||
|
for k,item in enumerate(sub.get('items') or []):
|
||||||
|
ip = sp+'/items/'+str(k)
|
||||||
|
if 'paper_id' in item:
|
||||||
|
observe(item,ip,pointer,sp)
|
||||||
|
else:
|
||||||
|
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||||
|
session['subsessions'].append(saved)
|
||||||
|
sessions.append(session)
|
||||||
|
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||||
|
raise ValueError('Unhandled or duplicated program location')
|
||||||
|
items = list(records.values())
|
||||||
|
if (len(seen) != 4326 or len(items) != 2143
|
||||||
|
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||||
|
or sum(not r['abstract'] for r in items) != 216
|
||||||
|
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||||
|
raise ValueError('Program count or missing-field census changed')
|
||||||
|
return items,sessions
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; use a fresh directory')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = load_snapshot(snapshots)
|
||||||
|
rows,sessions = parse_program(evidence['program-data.json'])
|
||||||
|
(out/'evidence').mkdir()
|
||||||
|
for name,body in evidence.items():
|
||||||
|
(out/'evidence'/name).write_bytes(body)
|
||||||
|
outputs = []
|
||||||
|
for track,count in TRACK_COUNTS.items():
|
||||||
|
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||||
|
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||||
|
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||||
|
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||||
|
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||||
|
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||||
|
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||||
|
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||||
|
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||||
|
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||||
|
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||||
|
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||||
|
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||||
|
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||||
|
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||||
|
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||||
|
part = out/'collection-report.json.part'
|
||||||
|
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,227 @@
|
|||||||
|
"""Frozen MM 2026 registered publications with separately attributed program evidence.
|
||||||
|
|
||||||
|
All publications survive normalization. A program metadata match is not a
|
||||||
|
scientific-work identity decision; publisher article type is not research type.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as program
|
||||||
|
|
||||||
|
SOURCE = 'mm_registry_program'
|
||||||
|
SNAPSHOT = 'mm2026-container-registry-v2'
|
||||||
|
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
|
||||||
|
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
|
||||||
|
PARENT_DOI = '10.1145/3767308'
|
||||||
|
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
|
||||||
|
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
|
||||||
|
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
|
||||||
|
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
|
||||||
|
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
|
||||||
|
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
|
||||||
|
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
|
||||||
|
'publisher','member','type','container-title','short-container-title','event','abstract',
|
||||||
|
'page','ISBN','isbn-type','published','published-print','published-online','issued',
|
||||||
|
'license','resource','relation','alternative-id'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(values):
|
||||||
|
return Counter(key(v) for v in values)
|
||||||
|
|
||||||
|
|
||||||
|
def load_registry(root, parent_path):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
raw_parent = program.safe_read(Path(parent_path), 128*1024)
|
||||||
|
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
|
||||||
|
raise ValueError('Reviewed registry or parent hash changed')
|
||||||
|
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
|
||||||
|
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
|
||||||
|
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
|
||||||
|
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
|
||||||
|
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
|
||||||
|
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
|
||||||
|
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
|
||||||
|
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
|
||||||
|
raise ValueError('Incomplete reviewed registry or wrong proceedings')
|
||||||
|
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
|
||||||
|
for number,receipt in enumerate(manifest['pages'],1):
|
||||||
|
name=receipt['file']
|
||||||
|
if name != f'page-{number:03}.json':
|
||||||
|
raise ValueError('Unexpected registry page filename')
|
||||||
|
body=program.safe_read(directory/name,8*1024**2)
|
||||||
|
url=urlsplit(receipt['url']);query=parse_qs(url.query)
|
||||||
|
expected_filter='container-title:'+CONTAINER+',member:320'
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
|
||||||
|
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
|
||||||
|
or query['rows'] != ['250'] or not query['cursor'][0]
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Registry receipt and source body disagree')
|
||||||
|
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
|
||||||
|
if (obj['status'] != 'ok' or message['total-results'] != 1918
|
||||||
|
or len(message['items']) != expected or receipt['items'] != expected):
|
||||||
|
raise ValueError('Incomplete registry pagination')
|
||||||
|
evidence[name]=body
|
||||||
|
for position,original in enumerate(message['items']):
|
||||||
|
doi=original.get('DOI')
|
||||||
|
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
|
||||||
|
raise ValueError('Missing, foreign or repeated publication DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
|
||||||
|
or original.get('member') != '320' or original.get('publisher') != 'ACM'
|
||||||
|
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
|
||||||
|
raise ValueError('Wrong registered publication context or changed abstract availability')
|
||||||
|
titles=original.get('title');authors=original.get('author')
|
||||||
|
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
|
||||||
|
or not isinstance(authors,list) or not authors):
|
||||||
|
raise ValueError('Missing publication title or authors')
|
||||||
|
names=[];flags=[]
|
||||||
|
for i,author in enumerate(authors,1):
|
||||||
|
family=author.get('family');given=author.get('given','')
|
||||||
|
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
|
||||||
|
raise ValueError('Invalid registered author fields')
|
||||||
|
name=' '.join(x.strip() for x in (given,family) if x.strip())
|
||||||
|
if '@' in name or name.casefold()=='anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
names.append(name)
|
||||||
|
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
|
||||||
|
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
|
||||||
|
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
|
||||||
|
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
|
||||||
|
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
|
||||||
|
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
|
||||||
|
'receipt':receipt,'source_position':position})
|
||||||
|
if len(rows)!=1918:
|
||||||
|
raise ValueError('Incomplete registered publication census')
|
||||||
|
return evidence,rows
|
||||||
|
|
||||||
|
|
||||||
|
def correlate(rows, observations):
|
||||||
|
titles=defaultdict(list)
|
||||||
|
for row in observations:titles[key(row['title'])].append(row)
|
||||||
|
matched=set();kinds=Counter()
|
||||||
|
for row in rows:
|
||||||
|
candidates=titles[key(row['title'])]
|
||||||
|
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||||
|
if len(exact)==1:
|
||||||
|
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
|
||||||
|
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||||
|
if exact[0]['track'] not in TRACKS:
|
||||||
|
raise ValueError('New registered program track requires review')
|
||||||
|
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||||
|
elif candidates:row['kind']='program_author_version_requires_review'
|
||||||
|
else:row['kind']='no_exact_program_title_requires_review'
|
||||||
|
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||||
|
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||||
|
kinds[row['kind']]+=1
|
||||||
|
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||||
|
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
|
||||||
|
if kinds != expected:
|
||||||
|
raise ValueError('Reviewed program/publication correspondence changed')
|
||||||
|
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||||
|
out=Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; inspect prior operation, never overwrite')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
|
||||||
|
program_evidence=program.load_snapshot(program_snapshots)
|
||||||
|
observations,_=program.parse_program(program_evidence['program-data.json'])
|
||||||
|
kinds,unmatched=correlate(publications,observations)
|
||||||
|
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
|
||||||
|
directory=out/'evidence'/group;directory.mkdir(parents=True)
|
||||||
|
for name,body in files.items():(directory/name).write_bytes(body)
|
||||||
|
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
|
||||||
|
for item in sorted(publications,key=lambda r:r['doi']):
|
||||||
|
p=item.get('program');doi=item['doi'];receipt=item['receipt']
|
||||||
|
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
|
||||||
|
abstract=p['abstract'] if p and p['abstract'] else None
|
||||||
|
missing+=abstract is None;copied+=abstract is not None
|
||||||
|
if p:known_tracks[p['track']]+=1
|
||||||
|
tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}]
|
||||||
|
if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']})
|
||||||
|
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
|
||||||
|
'title':item['title'],'authors':item['authors'],'abstract':abstract,
|
||||||
|
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||||
|
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||||
|
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||||
|
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||||
|
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||||
|
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
|
||||||
|
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
|
||||||
|
'program_fields':p,'program_match_kind':item['kind'],
|
||||||
|
'program_title_candidates':item['program_title_candidates'],
|
||||||
|
'program_exact_candidates':item['program_exact_candidates'],
|
||||||
|
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
|
||||||
|
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||||
|
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||||
|
'review_flags':item['flags'],
|
||||||
|
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
|
||||||
|
outputs=[]
|
||||||
|
for track,rows in sorted(buckets.items()):
|
||||||
|
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||||
|
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||||
|
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||||
|
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
|
||||||
|
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
|
||||||
|
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||||
|
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||||
|
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||||
|
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||||
|
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||||
|
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||||
|
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||||
|
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
|
||||||
|
raise ValueError('Reviewed output partition or abstract census changed')
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--registry-snapshots',required=True)
|
||||||
|
parser.add_argument('--program-snapshots',required=True)
|
||||||
|
parser.add_argument('--parent-snapshot',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv)
|
||||||
|
report=collect(args.out,registry_snapshots=args.registry_snapshots,
|
||||||
|
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
|
||||||
|
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
|
||||||
|
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
|
||||||
|
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
|
||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
|
TABLE = 'source_scope_revisions'
|
||||||
|
MARKER = 'source_scope_revisions_v1'
|
||||||
|
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
||||||
|
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
||||||
|
|
||||||
|
|
||||||
|
def available(c):
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
|
||||||
|
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(c):
|
||||||
|
exists = available(c)
|
||||||
|
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
|
||||||
|
if exists != marker:
|
||||||
|
raise ValueError('Source revision table/marker mismatch')
|
||||||
|
if exists:
|
||||||
|
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
|
||||||
|
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
|
||||||
|
if tuple(x[0] for x in description) != COLUMNS:
|
||||||
|
raise ValueError('Unexpected source revision schema')
|
||||||
|
return
|
||||||
|
ddl = '''CREATE TABLE source_scope_revisions (
|
||||||
|
version_sha256 TEXT PRIMARY KEY,
|
||||||
|
source_id INTEGER NOT NULL REFERENCES sources(id),
|
||||||
|
venue_id INTEGER NOT NULL REFERENCES venues(id),
|
||||||
|
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
|
||||||
|
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
|
||||||
|
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
|
||||||
|
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
|
||||||
|
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
|
||||||
|
c.execute(ddl)
|
||||||
|
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
|
||||||
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
|
||||||
|
|
||||||
|
|
||||||
|
def heads(c, source_id):
|
||||||
|
if not available(c):
|
||||||
|
return []
|
||||||
|
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
|
||||||
|
raise ValueError('Source revision table/marker mismatch')
|
||||||
|
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
|
||||||
|
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
|
||||||
|
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
|
||||||
|
ORDER BY v.version_sha256''', (source_id,)).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
def current_rows(c, source_id, cache):
|
||||||
|
active = heads(c, source_id)
|
||||||
|
signatures = tuple(r[0] for r in active)
|
||||||
|
# A new head invalidates the cache, even when maintenance uses the same Store.
|
||||||
|
if cache.get('signatures') != (source_id, signatures):
|
||||||
|
records = {}; scopes = {}
|
||||||
|
for signature in signatures:
|
||||||
|
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
||||||
|
scopes[signature] = json.loads(raw)
|
||||||
|
for row in scopes[signature]:
|
||||||
|
ext = row['external_id']
|
||||||
|
if ext in records and records[ext] != row:
|
||||||
|
raise ValueError('Conflicting active source snapshots')
|
||||||
|
records[ext] = row
|
||||||
|
cache.clear()
|
||||||
|
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
|
||||||
|
return cache['records']
|
||||||
|
|
||||||
|
|
||||||
|
def subset(expected, actual):
|
||||||
|
"""Reviewed source fields must survive additional work-identity/curation evidence."""
|
||||||
|
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
|
||||||
|
|
||||||
|
|
||||||
|
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
|
||||||
|
expected = current_rows(c, source_id, cache).get(external)
|
||||||
|
if expected is None:
|
||||||
|
return
|
||||||
|
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
|
||||||
|
if any(actual[k] != expected.get(k) for k in actual):
|
||||||
|
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
|
||||||
|
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
|
||||||
|
raise ValueError('Stale source snapshot metadata or identifiers')
|
||||||
|
|
||||||
|
|
||||||
|
def scope_head(c, source_id, manifest):
|
||||||
|
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
|
||||||
|
if not venue:
|
||||||
|
return None
|
||||||
|
matches = [r for r in heads(c, source_id)
|
||||||
|
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
|
||||||
|
if len(matches) > 1:
|
||||||
|
raise ValueError('Forked source revision history')
|
||||||
|
return matches[0][0] if matches else None
|
||||||
|
|
||||||
|
|
||||||
|
def require_manifest(c, source_id, manifest, cache):
|
||||||
|
current_rows(c, source_id, cache)
|
||||||
|
signature = scope_head(c, source_id, manifest)
|
||||||
|
if signature is None:
|
||||||
|
return
|
||||||
|
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
|
||||||
|
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
|
||||||
|
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
|
||||||
|
raise ValueError('Source coverage manifest differs from current source snapshot')
|
||||||
|
|
||||||
|
|
||||||
|
def require_row(c, source_id, row, manifest, cache):
|
||||||
|
expected = current_rows(c, source_id, cache).get(row['external_id'])
|
||||||
|
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
|
||||||
|
signature = scope_head(c, source_id, scoped_manifest)
|
||||||
|
if expected is None:
|
||||||
|
if signature is not None:
|
||||||
|
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
|
||||||
|
return
|
||||||
|
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
|
||||||
|
raise ValueError('Source listing differs from current source snapshot')
|
||||||
|
for key, value in expected.items():
|
||||||
|
if key in ('metadata', 'identifiers'):
|
||||||
|
okay = subset(value, row.get(key))
|
||||||
|
else:
|
||||||
|
okay = row.get(key) == value
|
||||||
|
if not okay:
|
||||||
|
raise ValueError('Stale source snapshot field: ' + key)
|
||||||
@@ -96,6 +96,10 @@ class Store:
|
|||||||
if found:
|
if found:
|
||||||
matches.add(found[0])
|
matches.add(found[0])
|
||||||
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
||||||
|
from paper_atlas.db.source_revisions import require_record
|
||||||
|
if not hasattr(self, '_source_revision_cache'):
|
||||||
|
self._source_revision_cache = {}
|
||||||
|
require_record(c, source_id, external_id, url, title, abstract, authors, identifiers, metadata, self._source_revision_cache)
|
||||||
from paper_atlas.db.edition_scopes import preserve
|
from paper_atlas.db.edition_scopes import preserve
|
||||||
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
||||||
if old:
|
if old:
|
||||||
|
|||||||
@@ -101,6 +101,9 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
|||||||
c.commit()
|
c.commit()
|
||||||
if incremental is not None:
|
if incremental is not None:
|
||||||
require_incremental_parent(c, source_id, manifest, incremental)
|
require_incremental_parent(c, source_id, manifest, incremental)
|
||||||
|
from paper_atlas.db.source_revisions import require_manifest
|
||||||
|
store._source_revision_cache = {}
|
||||||
|
require_manifest(c, source_id, manifest, store._source_revision_cache)
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
with path.open('rb') as stream:
|
with path.open('rb') as stream:
|
||||||
for line in stream:
|
for line in stream:
|
||||||
@@ -113,6 +116,10 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
|||||||
require_track_record(row, manifest)
|
require_track_record(row, manifest)
|
||||||
from paper_atlas.db.events import validate_event_key
|
from paper_atlas.db.events import validate_event_key
|
||||||
validate_event_key(row.get('event_key'))
|
validate_event_key(row.get('event_key'))
|
||||||
|
from paper_atlas.db.source_revisions import require_row
|
||||||
|
if not hasattr(store, '_source_revision_cache'):
|
||||||
|
store._source_revision_cache = {}
|
||||||
|
require_row(c, source_id, row, manifest, store._source_revision_cache)
|
||||||
external = row['external_id']
|
external = row['external_id']
|
||||||
if external in seen:
|
if external in seen:
|
||||||
raise ValueError('Duplicate external ID in one collected file: '+external)
|
raise ValueError('Duplicate external ID in one collected file: '+external)
|
||||||
|
|||||||
@@ -0,0 +1,327 @@
|
|||||||
|
"""Plan-bound publication scope revisions with complete before-images, never reimport."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from contextlib import closing
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.db import source_revisions as versions
|
||||||
|
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
|
||||||
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
|
from paper_atlas.ingestion.edition_tracks import curation
|
||||||
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||||
|
|
||||||
|
MAX_ROWS = 10000
|
||||||
|
|
||||||
|
|
||||||
|
def validate(inventory):
|
||||||
|
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
|
||||||
|
'previous_rows', 'rows', 'evidence'}
|
||||||
|
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
|
||||||
|
raise ValueError('Unsupported source revision inventory')
|
||||||
|
if (type(inventory['year']) is not int or inventory['year'] < 2000
|
||||||
|
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
|
||||||
|
or not isinstance(inventory['event_key'], str)):
|
||||||
|
raise ValueError('Explicit publication source scope required')
|
||||||
|
if not inventory['list_url'].startswith('https://'):
|
||||||
|
raise ValueError('HTTPS source listing required')
|
||||||
|
evidence = inventory['evidence']
|
||||||
|
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
|
||||||
|
raise ValueError('Bounded source evidence required')
|
||||||
|
for item in evidence:
|
||||||
|
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
|
||||||
|
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
|
||||||
|
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
|
||||||
|
raise ValueError('Source evidence URL/hash required')
|
||||||
|
snapshots = []
|
||||||
|
for key in ('previous_rows', 'rows'):
|
||||||
|
items = inventory[key]
|
||||||
|
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
|
||||||
|
raise ValueError('Complete bounded source snapshots required')
|
||||||
|
records = {}
|
||||||
|
for row in items:
|
||||||
|
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
|
||||||
|
raise ValueError('Source snapshot scope mismatch')
|
||||||
|
ext = row.get('external_id')
|
||||||
|
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
|
||||||
|
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
|
||||||
|
or row.get('identifiers', {}).get('doi') != ext
|
||||||
|
or not isinstance(row.get('track'), str) or not row['track']
|
||||||
|
or row.get('event_key', '') != inventory['event_key']
|
||||||
|
or not isinstance(row.get('title'), str) or not row['title']
|
||||||
|
or not isinstance(row.get('authors'), list)
|
||||||
|
or not isinstance(row.get('metadata'), dict)):
|
||||||
|
raise ValueError('Unique DOI-bearing accepted publication records required')
|
||||||
|
records[ext] = row
|
||||||
|
snapshots.append(records)
|
||||||
|
previous, current = snapshots
|
||||||
|
if set(previous) != set(current):
|
||||||
|
raise ValueError('Source membership changed; this operation cannot add or remove publications')
|
||||||
|
for ext, row in current.items():
|
||||||
|
before = previous[ext]
|
||||||
|
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
|
||||||
|
if any(before.get(k) != row.get(k) for k in stable):
|
||||||
|
raise ValueError('Stable publication identity or decision changed')
|
||||||
|
return previous, current
|
||||||
|
|
||||||
|
|
||||||
|
def bounded(c, sql, params):
|
||||||
|
result = values(c.execute(sql + ' LIMIT 100001', params))
|
||||||
|
if len(result) > 100000:
|
||||||
|
raise ValueError('Source revision protection exceeds bounded scope')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def capture(c, inventory):
|
||||||
|
previous, current = validate(inventory)
|
||||||
|
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
|
||||||
|
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
|
||||||
|
if not source or not venue:
|
||||||
|
raise ValueError('Existing source and venue required')
|
||||||
|
sid, vid = source[0], venue[0]
|
||||||
|
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
|
||||||
|
(vid, inventory['year'], inventory['event_key']))
|
||||||
|
by_track = {r[3]: r[0] for r in editions}
|
||||||
|
# The reviewed scope includes all its existing target attempts, even when
|
||||||
|
# they originate from another source. No empty-target shortcut is used.
|
||||||
|
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
|
||||||
|
scope_params = (vid, inventory['year'], inventory['event_key'])
|
||||||
|
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
|
||||||
|
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
|
||||||
|
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
|
||||||
|
if not own_records:
|
||||||
|
raise ValueError('Existing source observations required')
|
||||||
|
if any(r[2] not in current for r in own_records):
|
||||||
|
raise ValueError('Snapshot omits an existing source observation')
|
||||||
|
pids = sorted({r[1] for r in scoped})
|
||||||
|
if not pids or len(pids) > MAX_ROWS:
|
||||||
|
raise ValueError('Bounded nonempty protected paper scope required')
|
||||||
|
marks = ','.join('?' for _ in pids)
|
||||||
|
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
|
||||||
|
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
|
||||||
|
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
|
||||||
|
queries = {
|
||||||
|
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
|
||||||
|
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
|
||||||
|
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
|
||||||
|
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
|
||||||
|
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
|
||||||
|
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
|
||||||
|
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
|
||||||
|
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
|
||||||
|
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
|
||||||
|
}
|
||||||
|
from paper_atlas.curation.annotations import available
|
||||||
|
if available(c):
|
||||||
|
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
|
||||||
|
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
|
||||||
|
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
|
||||||
|
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
|
||||||
|
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
|
||||||
|
raise ValueError('Unambiguous source listing required')
|
||||||
|
if len({r[0] for r in coverage}) != len(coverage):
|
||||||
|
raise ValueError('One source coverage entry per edition required')
|
||||||
|
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
|
||||||
|
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
|
||||||
|
if len(own_heads) > 1:
|
||||||
|
raise ValueError('Forked source revision history')
|
||||||
|
parent = own_heads[0][0] if own_heads else None
|
||||||
|
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
|
||||||
|
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
|
||||||
|
'parent_sha256': parent, 'by_track': by_track,
|
||||||
|
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
|
||||||
|
|
||||||
|
|
||||||
|
def changes(c, inventory, before):
|
||||||
|
previous, current = validate(inventory)
|
||||||
|
packet = before['packet']; result = []; sid = before['source_id']
|
||||||
|
editions = {r[0]: r for r in before['editions']}
|
||||||
|
for raw in before['own_records']:
|
||||||
|
ext = raw[2]; old = previous[ext]; new = current[ext]
|
||||||
|
metadata = json.loads(raw[8])
|
||||||
|
candidates = [old, new]
|
||||||
|
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
|
||||||
|
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
|
||||||
|
raise ValueError('Current raw source differs from both reviewed snapshots')
|
||||||
|
if metadata.get('record_identifiers', {}).get('doi') != ext:
|
||||||
|
raise ValueError('Current stable DOI differs from source evidence')
|
||||||
|
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
|
||||||
|
if len(attempts) != 1:
|
||||||
|
raise ValueError('Exactly one publication attempt per source record required')
|
||||||
|
sub = attempts[0]; origin = editions[sub[2]]
|
||||||
|
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
|
||||||
|
raise ValueError('Current source attempt identity/scope differs')
|
||||||
|
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
|
||||||
|
raise ValueError('Shared-source attempt requires separate reconciliation')
|
||||||
|
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
|
||||||
|
if not decisions or any(d[2] != 'accepted' for d in decisions):
|
||||||
|
raise ValueError('Only accepted publication observations can be revised')
|
||||||
|
target = before['by_track'].get(new['track'])
|
||||||
|
if target is None:
|
||||||
|
raise ValueError('Reviewed target edition must already exist')
|
||||||
|
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
|
||||||
|
if not identities or any(x[0] != sub[2] for x in identities):
|
||||||
|
raise ValueError('Inconsistent scoped submission identifiers')
|
||||||
|
for _, scheme, value, _ in identities:
|
||||||
|
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
|
||||||
|
if conflict and conflict[0] != sub[0]:
|
||||||
|
raise ValueError('Target submission identifier collision')
|
||||||
|
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
|
||||||
|
if sub[2] != target and KEY in metadata:
|
||||||
|
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
|
||||||
|
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
|
||||||
|
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
|
||||||
|
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
|
||||||
|
'from_track': origin[3], 'track': new['track'],
|
||||||
|
'new_abstract': new.get('abstract'), 'metadata': replacement})
|
||||||
|
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
|
||||||
|
observed = Counter()
|
||||||
|
own_ids = {r[0] for r in before['own_records']}
|
||||||
|
for s in packet['submissions']:
|
||||||
|
if s[3] in own_ids and s[2] in editions:
|
||||||
|
observed[s[2]] += 1
|
||||||
|
coverage = []
|
||||||
|
for raw in before['coverage']:
|
||||||
|
track = editions[raw[0]][3]
|
||||||
|
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
|
||||||
|
raise ValueError('Coverage parent is not the verified partial source observation count')
|
||||||
|
if raw[4] not in {old_expected[track], new_expected[track]}:
|
||||||
|
raise ValueError('Coverage expected count differs from both source versions')
|
||||||
|
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
|
||||||
|
if count < 0 or count > new_expected[track]:
|
||||||
|
raise ValueError('Corrected coverage exceeds current source membership')
|
||||||
|
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
|
||||||
|
covered = {r['before'][0] for r in coverage}
|
||||||
|
if any(r['target_edition_id'] not in covered for r in result):
|
||||||
|
raise ValueError('Target source coverage must already exist')
|
||||||
|
return result, coverage
|
||||||
|
|
||||||
|
|
||||||
|
def proposal(c, inventory, identity):
|
||||||
|
before = capture(c, inventory)
|
||||||
|
if before['parent_sha256']:
|
||||||
|
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
||||||
|
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
|
||||||
|
raise ValueError('Previous snapshot differs from active source version')
|
||||||
|
records, coverage = changes(c, inventory, before)
|
||||||
|
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
|
||||||
|
'changes': records, 'coverage': coverage}
|
||||||
|
|
||||||
|
|
||||||
|
def verify(c, plan, signature):
|
||||||
|
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
|
||||||
|
for change in plan['changes']:
|
||||||
|
for row in expected['submissions']:
|
||||||
|
if row[0] == change['submission_id']:
|
||||||
|
row[2] = change['target_edition_id']
|
||||||
|
for row in expected['submission_identifiers']:
|
||||||
|
if row[3] == change['submission_id']:
|
||||||
|
row[0] = change['target_edition_id']
|
||||||
|
for row in expected['source_records']:
|
||||||
|
if row[0] == change['source_record_id']:
|
||||||
|
row[6] = change['new_abstract']
|
||||||
|
row[8] = encoded_metadata(plan, change, signature)
|
||||||
|
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
|
||||||
|
if after['packet'] != expected:
|
||||||
|
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
|
||||||
|
if after['editions'] != plan['before']['editions']:
|
||||||
|
raise ValueError('Edition identities changed')
|
||||||
|
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
|
||||||
|
inventory = plan['inventory']
|
||||||
|
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
|
||||||
|
if after['revision_heads'] != sorted(expected_heads):
|
||||||
|
raise ValueError('Active source version differs')
|
||||||
|
expected_coverage = []
|
||||||
|
for item in plan['coverage']:
|
||||||
|
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
|
||||||
|
expected_coverage.append(row)
|
||||||
|
if after['coverage'] != expected_coverage:
|
||||||
|
raise ValueError('Reconciled coverage differs; roll back')
|
||||||
|
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
|
||||||
|
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
|
||||||
|
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
||||||
|
raise ValueError('Versioned source or complete before-images changed')
|
||||||
|
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
|
||||||
|
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
|
||||||
|
|
||||||
|
|
||||||
|
def encoded_metadata(plan, change, signature):
|
||||||
|
metadata = copy.deepcopy(change['metadata'])
|
||||||
|
if change['from_edition_id'] != change['target_edition_id']:
|
||||||
|
inventory = plan['inventory']
|
||||||
|
evidence = inventory['evidence'][0]
|
||||||
|
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
|
||||||
|
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
|
||||||
|
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
|
||||||
|
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
|
||||||
|
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
|
||||||
|
return json.dumps(metadata, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_plan(c, plan):
|
||||||
|
if plan.get('format') != 1:
|
||||||
|
raise ValueError('Unsupported source revision plan')
|
||||||
|
if not getattr(c, 'postgres', False) and not c.in_transaction:
|
||||||
|
c.execute('BEGIN IMMEDIATE')
|
||||||
|
signature = digest(plan)
|
||||||
|
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
|
||||||
|
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
|
||||||
|
if proposal(c, plan['inventory'], plan['database']) != plan:
|
||||||
|
raise ValueError('Stale source revision plan; create and review a fresh plan')
|
||||||
|
versions.initialize(c)
|
||||||
|
for row in plan['changes']:
|
||||||
|
if row['from_edition_id'] != row['target_edition_id']:
|
||||||
|
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
|
||||||
|
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
|
||||||
|
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
|
||||||
|
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
|
||||||
|
for row in plan['coverage']:
|
||||||
|
before = row['before']
|
||||||
|
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||||
|
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
|
||||||
|
inventory = plan['inventory']
|
||||||
|
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
|
||||||
|
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
|
||||||
|
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
|
||||||
|
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
|
||||||
|
|
||||||
|
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or not out.parent.is_dir():
|
||||||
|
raise ValueError('New output in existing directory required')
|
||||||
|
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
|
||||||
|
raise ValueError('Choose inventory preflight or plan plus exact file hash')
|
||||||
|
if inventory:
|
||||||
|
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
|
||||||
|
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
|
||||||
|
else:
|
||||||
|
raw = Path(plan).read_bytes()
|
||||||
|
if hashlib.sha256(raw).hexdigest() != plan_sha256:
|
||||||
|
raise ValueError('Plan file hash differs')
|
||||||
|
proposed = json.loads(raw)
|
||||||
|
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
|
||||||
|
try:
|
||||||
|
if proposed['database'] != database_identity(c, db):
|
||||||
|
raise ValueError('Plan belongs to another database')
|
||||||
|
if not getattr(c, 'postgres', False):
|
||||||
|
c.commit(); c.execute('BEGIN IMMEDIATE')
|
||||||
|
result = apply_plan(c, proposed); c.commit()
|
||||||
|
except BaseException:
|
||||||
|
c.rollback(); raise
|
||||||
|
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
|
||||||
|
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
|
||||||
|
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)
|
||||||
@@ -8,7 +8,7 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
|
|||||||
'submission_sources', 'submission_identifiers', 'decision_sources',
|
'submission_sources', 'submission_identifiers', 'decision_sources',
|
||||||
'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
|
'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
|
||||||
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
|
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
|
||||||
OPTIONAL_TABLES = ('annotation_events',)
|
OPTIONAL_TABLES = ('annotation_events', 'source_scope_revisions')
|
||||||
CHECKS = {
|
CHECKS = {
|
||||||
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
|
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
|
||||||
'venues': ["venue_type IN ('conference','journal','review_platform')"],
|
'venues': ["venue_type IN ('conference','journal','review_platform')"],
|
||||||
|
|||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,92 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import ismir2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ISMIR2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||||
|
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||||
|
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||||
|
|
||||||
|
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||||
|
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||||
|
self.assertEqual(len(observations),140)
|
||||||
|
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||||
|
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||||
|
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||||
|
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||||
|
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||||
|
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||||
|
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||||
|
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||||
|
|
||||||
|
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||||
|
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||||
|
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||||
|
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||||
|
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||||
|
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||||
|
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||||
|
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||||
|
rows=copy.deepcopy(self.program)
|
||||||
|
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||||
|
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||||
|
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||||
|
if kind=='missing_row':rows.pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||||
|
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||||
|
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_source_optin_no_cannot_publish_review_text(self):
|
||||||
|
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||||
|
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||||
|
|
||||||
|
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||||
|
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||||
|
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||||
|
with self.assertRaises(ValueError):c.author_names(raw)
|
||||||
|
|
||||||
|
def test_ambiguous_official_title_is_not_fused(self):
|
||||||
|
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||||
|
|
||||||
|
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||||
|
for kind in ('manifest','body','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MM2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||||
|
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||||
|
cls.data = json.loads(cls.raw)
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name,body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def parse_changed(self, data):
|
||||||
|
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||||
|
|
||||||
|
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||||
|
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||||
|
actual = {r['paper_id']:r for r in rows}
|
||||||
|
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||||
|
for gold in self.gold['inventory']:
|
||||||
|
r = actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||||
|
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||||
|
|
||||||
|
def test_every_presentation_and_session_context_is_preserved(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
for r in rows:
|
||||||
|
for context in r['contexts']:
|
||||||
|
node = self.data
|
||||||
|
for key in context['source_pointer'].split('/')[1:]:
|
||||||
|
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||||
|
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||||
|
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||||
|
for i,session in enumerate(sessions):
|
||||||
|
original = self.data['entries'][i]
|
||||||
|
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||||
|
self.assertEqual(session['track'],'')
|
||||||
|
for key in original.keys()-{'papers','subsessions'}:
|
||||||
|
self.assertEqual(session[key],original[key])
|
||||||
|
for j,sub in enumerate(original.get('subsessions') or []):
|
||||||
|
saved = session['subsessions'][j]
|
||||||
|
self.assertEqual(saved['title'],sub['title'])
|
||||||
|
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||||
|
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||||
|
|
||||||
|
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||||
|
rows,_ = c.parse_program(self.raw)
|
||||||
|
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||||
|
for r in rows:
|
||||||
|
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['publication_identity_verified'])
|
||||||
|
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||||
|
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||||
|
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||||
|
missing = [r for r in rows if not r['authors']]
|
||||||
|
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||||
|
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||||
|
|
||||||
|
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src')
|
||||||
|
report=c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||||
|
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||||
|
for name,body in self.files.items():
|
||||||
|
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||||
|
n=0
|
||||||
|
for item in report['files']:
|
||||||
|
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||||
|
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||||
|
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||||
|
self.assertEqual(n,2143)
|
||||||
|
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
first=data['entries'][0]['papers'][0]
|
||||||
|
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||||
|
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
moved=data['entries'][0]['papers'].pop()
|
||||||
|
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||||
|
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||||
|
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||||
|
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||||
|
elif kind=='paper_track':paper['track']='New Track'
|
||||||
|
elif kind=='field':paper['doi']='unreviewed'
|
||||||
|
elif kind=='title':paper['title']=''
|
||||||
|
else:paper['authors']='Unparsed author string'
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_edition_or_census_change_requires_review(self):
|
||||||
|
for kind in ('edition','count'):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||||
|
else:data['entries'][0]['papers'].pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||||
|
for kind in ('manifest','program','probe','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||||
|
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||||
|
path=directory/name
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':
|
||||||
|
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||||
|
rows,sessions=c.parse_program(self.raw)
|
||||||
|
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||||
|
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||||
|
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
from contextlib import redirect_stdout
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm_publications2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MMPublications2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
fixtures=Path(__file__).parent/'fixtures'
|
||||||
|
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||||
|
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||||
|
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||||
|
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||||
|
cls.out=cls.root/'out'
|
||||||
|
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
||||||
|
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
||||||
|
cls.by_doi={r['external_id']:r for r in cls.rows}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.temp.cleanup()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def materialize(root,files):
|
||||||
|
for name,body in files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||||
|
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||||
|
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
|
||||||
|
for g in self.fixture['independent_gold']:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||||
|
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
||||||
|
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
||||||
|
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
||||||
|
|
||||||
|
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||||
|
count=0
|
||||||
|
for g in self.fixture['independent_gold']:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata']
|
||||||
|
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||||
|
if g['abstract_sha256'] is not None:
|
||||||
|
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||||
|
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||||
|
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
|
||||||
|
else:
|
||||||
|
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||||
|
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
|
||||||
|
|
||||||
|
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||||
|
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||||
|
for p in self.report['unmatched_program']:
|
||||||
|
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||||
|
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
|
||||||
|
gold=self.program_fixture['independent_inventory']['inventory']
|
||||||
|
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||||
|
for g in gold:
|
||||||
|
p=programs[g['paper_id']]
|
||||||
|
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
||||||
|
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
||||||
|
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
||||||
|
|
||||||
|
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
||||||
|
for r in self.rows:
|
||||||
|
doi=r['external_id'];m=r['metadata']
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
||||||
|
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
||||||
|
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
||||||
|
self.assertFalse(m['scientific_work_identity_approved'])
|
||||||
|
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
||||||
|
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
|
||||||
|
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||||
|
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||||
|
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||||
|
self.assertEqual(len(variants),192)
|
||||||
|
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||||
|
|
||||||
|
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||||
|
authors_by_page={}
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
||||||
|
self.assertEqual(actual.read_bytes(),body.encode())
|
||||||
|
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
||||||
|
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
||||||
|
for name,body in self.program_fixture['files'].items():
|
||||||
|
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
||||||
|
for r in self.rows:
|
||||||
|
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
||||||
|
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
||||||
|
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
||||||
|
for f in self.report['files']:
|
||||||
|
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
||||||
|
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
||||||
|
|
||||||
|
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||||
|
n=0
|
||||||
|
for g in self.fixture['independent_gold']:
|
||||||
|
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||||
|
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||||
|
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||||
|
for i in actual:
|
||||||
|
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
||||||
|
self.assertEqual(n,8)
|
||||||
|
|
||||||
|
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||||
|
count=0
|
||||||
|
for g in self.fixture['independent_gold']:
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||||
|
count+=1;r=self.by_doi[g['doi']]
|
||||||
|
self.assertEqual(r['title'],g['title'])
|
||||||
|
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||||
|
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||||
|
|
||||||
|
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||||
|
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
||||||
|
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
||||||
|
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||||
|
|
||||||
|
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||||
|
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
|
||||||
|
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||||
|
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
|
||||||
|
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||||
|
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||||
|
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||||
|
self.assertIn('unknown',self.report['public_negative_availability'])
|
||||||
|
|
||||||
|
|
||||||
|
def assert_native_ingestion(self, pg_service=None):
|
||||||
|
# An isolated synthetic database exercises the writer contract, not a
|
||||||
|
# production identity approval. All source rows/track partitions run.
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.db.backend import open_store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
from paper_atlas.postgres.client import connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
|
||||||
|
store=Store(db)
|
||||||
|
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
|
||||||
|
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
|
||||||
|
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
|
||||||
|
'https://example.org/decision','rejected','Reject')
|
||||||
|
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
|
||||||
|
store.conn.commit();store.conn.close()
|
||||||
|
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
|
||||||
|
try:
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
|
||||||
|
for f in self.report['files']:
|
||||||
|
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
|
||||||
|
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
|
||||||
|
try:
|
||||||
|
conn=store.conn
|
||||||
|
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
|
||||||
|
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
|
||||||
|
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
|
||||||
|
self.assertEqual(set(actual),set(self.by_doi))
|
||||||
|
self.assertEqual(actual[seed['external_id']][1],paper)
|
||||||
|
for doi,row in self.by_doi.items():
|
||||||
|
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
|
||||||
|
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
|
||||||
|
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726)
|
||||||
|
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
|
||||||
|
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
|
||||||
|
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
|
||||||
|
finally:store.conn.close()
|
||||||
|
finally:
|
||||||
|
if pg_service:
|
||||||
|
with connect(pg_service) as pg:
|
||||||
|
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
|
||||||
|
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||||
|
|
||||||
|
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion()
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
|
||||||
|
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||||
|
|
||||||
|
|
||||||
|
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
src = self.root/'source'
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||||
|
mm2026.collect(self.root/'program', snapshots=src)
|
||||||
|
self.report_path = self.root/'program/collection-report.json'
|
||||||
|
|
||||||
|
def change_report(self, change):
|
||||||
|
report=json.loads(self.report_path.read_text());change(report)
|
||||||
|
self.report_path.write_text(json.dumps(report))
|
||||||
|
|
||||||
|
def assert_rejected_before_database(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||||
|
connection.assert_not_called()
|
||||||
|
self.assertFalse((self.root/'audit').exists())
|
||||||
|
|
||||||
|
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||||
|
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||||
|
actual={r['program_id']:r for r in rows}
|
||||||
|
for gold in self.fixture['independent_inventory']['inventory']:
|
||||||
|
r=actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal']))
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||||
|
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||||
|
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||||
|
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['contribution_type_verified'])
|
||||||
|
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||||
|
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||||
|
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||||
|
|
||||||
|
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||||
|
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143)
|
||||||
|
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||||
|
|
||||||
|
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||||
|
report=json.loads(self.report_path.read_text())
|
||||||
|
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||||
|
|
||||||
|
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||||
|
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||||
|
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||||
|
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||||
|
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||||
|
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||||
|
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||||
|
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||||
|
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_missing_or_repeated_track_rejected(self):
|
||||||
|
original=self.report_path.read_bytes()
|
||||||
|
for kind in ('missing','repeated'):
|
||||||
|
self.report_path.write_bytes(original)
|
||||||
|
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||||
|
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||||
|
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||||
|
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||||
|
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||||
|
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_mixed_input_modes_rejected_before_connection(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||||
|
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||||
|
connection.assert_not_called()
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
|
||||||
|
from contextlib import closing, redirect_stdout
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from paper_atlas.db.backend import open_store
|
||||||
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
|
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||||
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||||
|
from paper_atlas.ingestion.records import coverage_options, ingest_file
|
||||||
|
from paper_atlas.postgres.client import connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
|
||||||
|
URL = 'https://example.org/volume'
|
||||||
|
|
||||||
|
|
||||||
|
def seed(path):
|
||||||
|
old = []
|
||||||
|
for i in range(4):
|
||||||
|
doi = '10.1234/publication'+str(i)
|
||||||
|
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
|
||||||
|
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
|
||||||
|
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
|
||||||
|
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
|
||||||
|
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
|
||||||
|
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
|
||||||
|
current = copy.deepcopy(old)
|
||||||
|
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
|
||||||
|
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
|
||||||
|
current[1]['tags'][0]['value'] = 'main'
|
||||||
|
store = Store(path)
|
||||||
|
for row in old[:3]:
|
||||||
|
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||||
|
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
|
||||||
|
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
|
||||||
|
store.record_tags(p, r, 'fixture', row['tags'])
|
||||||
|
if row == old[1]:
|
||||||
|
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
|
||||||
|
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
|
||||||
|
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
|
||||||
|
# An unrelated source already has an attempt in the populated target edition.
|
||||||
|
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
|
||||||
|
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
|
||||||
|
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||||
|
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
|
||||||
|
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
|
||||||
|
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
|
||||||
|
store.conn.commit(); store.conn.close()
|
||||||
|
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
|
||||||
|
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
|
||||||
|
|
||||||
|
|
||||||
|
class RevisionCases:
|
||||||
|
def store(self):
|
||||||
|
return open_store(**self.target)
|
||||||
|
|
||||||
|
def plan(self, c):
|
||||||
|
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||||
|
|
||||||
|
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||||
|
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
|
||||||
|
before_papers = plan['before']['packet']['papers']
|
||||||
|
before_tags = plan['before']['packet']['paper_tags']
|
||||||
|
result = apply_plan(c, plan); c.commit()
|
||||||
|
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
|
||||||
|
self.assertEqual(result['protected_decisions'], 5)
|
||||||
|
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
|
||||||
|
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
|
||||||
|
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
|
||||||
|
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
|
||||||
|
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
|
||||||
|
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
|
||||||
|
self.assertEqual(saved, plan)
|
||||||
|
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||||
|
|
||||||
|
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
store = object.__new__(Store); store.conn = c; store.event_identity = True
|
||||||
|
old = self.inventory['previous_rows'][1]
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||||
|
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
|
||||||
|
c.rollback()
|
||||||
|
row = self.inventory['rows'][1]
|
||||||
|
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||||
|
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
|
||||||
|
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
|
||||||
|
c.rollback()
|
||||||
|
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
|
||||||
|
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'source snapshot'):
|
||||||
|
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
|
||||||
|
|
||||||
|
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
apply_plan(c, self.plan(c)); c.commit()
|
||||||
|
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||||
|
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'absent from current'):
|
||||||
|
require_row(c, sid, unknown, {'list_url': URL}, cache)
|
||||||
|
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||||
|
require_row(c, sid, bad, {'list_url': URL}, cache)
|
||||||
|
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
|
||||||
|
require_row(c, sid, good, {'list_url': URL}, cache)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
|
||||||
|
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
|
||||||
|
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
|
||||||
|
|
||||||
|
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c)
|
||||||
|
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale'):
|
||||||
|
apply_plan(c, plan)
|
||||||
|
c.rollback()
|
||||||
|
move = plan['changes'][0]
|
||||||
|
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
|
||||||
|
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'collision'):
|
||||||
|
self.plan(c)
|
||||||
|
c.rollback()
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
|
||||||
|
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c)
|
||||||
|
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Injected'):
|
||||||
|
apply_plan(c, plan)
|
||||||
|
c.rollback()
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
self.assertEqual(self.plan(c), plan)
|
||||||
|
|
||||||
|
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
|
||||||
|
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
|
||||||
|
path = self.root/'plan.json'
|
||||||
|
args = {k: v for k, v in self.target.items() if k != 'db'}
|
||||||
|
run(self.target.get('db'), path, inventory=inventory, **args)
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash'):
|
||||||
|
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
|
||||||
|
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
|
||||||
|
self.assertEqual(report['events_moved'], 1)
|
||||||
|
|
||||||
|
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
|
||||||
|
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
|
||||||
|
lambda x: x['rows'][0].update(status='rejected')]:
|
||||||
|
bad = copy.deepcopy(self.inventory); mutate(bad)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
validate(bad)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'incremental'):
|
||||||
|
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
|
||||||
|
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
|
||||||
|
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
|
||||||
|
|
||||||
|
|
||||||
|
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||||
|
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||||
|
def test_optional_revision_table_migrates_with_full_history(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
|
||||||
|
self.addCleanup(drop_schema, service, schema)
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
|
||||||
|
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
|
||||||
|
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||||
|
|
||||||
|
|
||||||
|
def drop_schema(service, schema):
|
||||||
|
with connect(service) as c:
|
||||||
|
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
|
||||||
|
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||||
|
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||||
|
db = self.root/'source.sqlite'; self.inventory = seed(db)
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
|
||||||
|
self.addCleanup(drop_schema, service, schema)
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
|
||||||
|
self.target = {'pg_service': service, 'pg_schema': schema}
|
||||||
Reference in New Issue
Block a user