Compare commits

..

15 Commits

Author SHA1 Message Date
codex-bot bcb8b2ad7e Record verified IROS arXiv work bindings
Metadata integrity checks / metadata (push) Successful in 11m14s
2026-11-18 17:28:03 +08:00
codex-bot 95ff77d77f Document IROS 2026 publication and presentation sources
Metadata integrity checks / metadata (push) Successful in 7m56s
2026-11-18 17:24:06 +08:00
Zayden-Jung 9da6a45ec0 Merge pull request '记录公开拒稿镜像差集与五条待核验 ID' (#72) from codex/negative-source-audit into main
Metadata integrity checks / metadata (push) Successful in 10m59s
Reviewed-on: #72
2026-11-18 08:54:56 +00:00
Zayden-Jung 7407f97ef2 Merge pull request '仅投影公开评审镜像索引字段并保留决定来源' (#71) from codex/reviewarena-metadata into main
Metadata integrity checks / metadata (push) Successful in 10m0s
Reviewed-on: #71
2026-11-18 08:49:53 +00:00
Zayden-Jung 31ff7d1626 Merge pull request '修复MM标题排版导致的26条节目摘要与分轨缺口' (#69) from codex/mm-title-formatting into main
Metadata integrity checks / metadata (push) Successful in 12m47s
Reviewed-on: #69
2026-11-18 08:23:13 +00:00
Zayden-Jung 8a7896d352 Merge pull request '版本化修正出版来源分轨与覆盖并保护旧版刷新' (#70) from codex/source-scope-revisions into main
Metadata integrity checks / metadata (push) Successful in 10m54s
Reviewed-on: #70
2026-11-18 08:23:02 +00:00
Zayden-Jung 35ff09a463 Merge pull request '核对 CIKM 2026 出版、附属议程与既有投稿历史' (#76) from codex/cikm-source-reconciliation into main
Metadata integrity checks / metadata (push) Successful in 7m38s
Reviewed-on: #76
2026-11-18 08:22:49 +00:00
codex-bot c469785a64 Reconcile CIKM supplemental sessions and existing DOI history
Metadata integrity checks / metadata (push) Successful in 7m35s
2026-11-18 15:41:50 +08:00
codex-bot 2f445eddc8 Record CIKM programme and publisher registry reconciliation
Metadata integrity checks / metadata (push) Successful in 7m6s
2026-11-18 15:28:08 +08:00
codex-bot 7d9c693761 Review mirror publication literals and replay frozen metadata offline
Metadata integrity checks / metadata (push) Successful in 6m40s
2026-11-18 09:34:10 +08:00
codex-bot 98ef415912 Use Arrow binary wheels compatible with runner Python 3.14
Metadata integrity checks / metadata (push) Successful in 7m13s
2026-11-18 09:14:27 +08:00
codex-bot 90fd789a17 Project pinned ReviewArena metadata without full-text payload pages
Metadata integrity checks / metadata (push) Failing after 47s
2026-11-18 09:10:29 +08:00
codex-bot 2000d340fa Add snapshot-bound publication source scope revisions
Metadata integrity checks / metadata (push) Successful in 7m16s
2026-11-18 06:59:21 +08:00
codex-bot 12f95cff91 Reject incomplete MM title markup before HTML parsing
Metadata integrity checks / metadata (push) Successful in 6m59s
2026-11-18 04:21:31 +08:00
codex-bot 9864632774 Recover MM program evidence through reviewed title formatting
Metadata integrity checks / metadata (push) Failing after 5m46s
2026-11-18 04:14:41 +08:00
25 changed files with 4627 additions and 43 deletions
+8 -4
View File
@@ -35,7 +35,7 @@ jobs:
venv = root / 'venv' venv = root / 'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python = str(venv / 'bin/python') python = str(venv / 'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True) subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True)
testenv = os.environ.copy() testenv = os.environ.copy()
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
if service_file.is_file(): if service_file.is_file():
@@ -75,11 +75,15 @@ jobs:
path.write_bytes(body.encode('utf-8')) path.write_bytes(body.encode('utf-8'))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text()) mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201) assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0 assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())} mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
assert len(mm_pub_rows)==1918 assert len(mm_pub_rows)==1918
for gold in mm_pub_fixture['independent_gold']: mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
for original_gold in mm_pub_fixture['independent_gold']:
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
row=mm_pub_rows[gold['doi']] row=mm_pub_rows[gold['doi']]
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track']) assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256'] assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
+88
View File
@@ -0,0 +1,88 @@
name: ReviewArena metadata projection
on:
workflow_dispatch:
inputs:
split:
description: 'Pinned public mirror family; metadata columns only'
required: true
default: emnlp
projection_name:
description: 'Optional existing metadata output directory name for offline replay'
required: false
default: ''
projection_sha256:
description: 'Exact cached collection-report.json SHA256, required for offline replay'
required: false
default: ''
jobs:
project:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
MIRROR_SPLIT: ${{ inputs.split }}
MIRROR_PROJECTION: ${{ inputs.projection_name }}
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Project permitted metadata on the approved remote runner
run: |
python3 -B - <<'PY'
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
split = os.environ['MIRROR_SPLIT']
cached = os.environ.get('MIRROR_PROJECTION','')
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
raise SystemExit('Explicit approved source commit/split required')
base = pathlib.Path('/data/paper-index')
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
if out.exists():
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
extra=[]
if cached:
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
raise SystemExit('Explicit permitted cached projection name and report hash required')
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
elif cached_sha:
raise SystemExit('Cached projection name required')
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
with urllib.request.urlopen(url,timeout=60) as response:
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Oversized source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
tar.extractall(root,filter='data')
candidates=list(root.glob('*/pyproject.toml'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive')
src=candidates[0].parent
venv=root/'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python=str(venv/'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
wheels=root/'wheels'
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
wheel,=wheels.glob('paper_atlas-*.whl')
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside=root/'outside';outside.mkdir()
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
report=json.loads((out/'collection-report.json').read_bytes())
assert report['database_written'] is False and report['approved_for_import']==0
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
backup=base/'validation'/(out.name+'-recovery.tar.gz')
if backup.exists():
raise SystemExit('Recovery archive already exists; inspect')
with tarfile.open(backup,'w:gz') as tar:
tar.add(out,arcname=str(out.relative_to(base)))
backup.chmod(0o444)
for path in out.iterdir():
if path.is_file():path.chmod(0o444)
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
PY
+5 -1
View File
@@ -2,6 +2,8 @@
**长期维护、按需运行的论文元数据与公开投稿历史工具。** **长期维护、按需运行的论文元数据与公开投稿历史工具。**
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
@@ -117,6 +119,8 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。 已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
完整出版来源的新旧版本导致分轨或摘要变化时,使用 [source-revisions 流程](docs/source-revisions.md):核对同一 DOI 全集,保存完整修正前数据与版本链,支持已有记录的目标 track,按新版本重算分区覆盖数,并阻止旧版元数据刷新。普通增量导入仍禁止缩小预期数量。
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。 入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。 RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
@@ -143,4 +147,4 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。 `collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,16 @@
# CIKM 2026 节目与出版登记核对
官方节目列有 1,008 项带 DOI 的条目,逐篇均有摘要。五个标签的数量分别为 Full Research 596、Short Research 234、Applied Research 86、Resources 46、Demo 46。另一份官方节目快照少三个节目编号,其余 1,005 个编号重合。两份原始快照和响应哈希保存在本机的 `cikm2026-official-detailed-program-v1` 工作目录。[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html) · [另一份官方节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)
正式卷 DOI 为 `10.1145/3799682`,ISBN 为 `9798400725395`。ISBN 查询只返回卷级登记。按卷名及出版日期分页取得 1,081 条登记,逐条核对卷 DOI 前缀、完整容器标题及来源类型。1,008 个节目 DOI 全部匹配,另有 73 条登记位于此前带 DOI 节目分区之外。附属议程核对结果见下文。该次 Crossref 响应中摘要字段均缺失。[ACM 出版条目](https://doi.org/10.1145/3799682.3840615) · [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)
按已知排版标记、Unicode 和标点变化比较,同 DOI 的条目有 4 处标题差异、11 处有序作者列表差异。差异条目保留两份原文,等后续核对后再决定如何关联。当前 PostgreSQL 快照中,97 个节目 DOI 已绑定论文,304 个节目标题有候选,其中 95 个 DOI 绑定与标题候选重合。仍需核查已有作品与历次投稿历史,当前尚未写入。
[逐条差集、字段差异及证据哈希](cikm2026-source-reconciliation-2026-11-18.json)。先将出版登记与节目摘要作为各自来源采集,再核查两者的作品关系和轨道。公开拒稿、撤回及桌拒的可得性继续单独核查。
## 附属议程与已有 DOI 的进一步核查
在官方节目页面的附属议程及主题演讲中,73 条登记有 72 条找到唯一规范标题对应,70 条同时通过有序作者比较。对应项包括 Industry Day 20 条、博士论坛 19 条、教程 14 条、工作坊条目 15 条、主题演讲 3 条及 AnalytiCup 1 条。工作坊条目记录该场活动对应的出版材料,独立 workshop 论文仍需各自目录。两条工作坊条目有作者写法差异,另有一条 LASS 标题带不同前后缀,三条均保留待核对。[官方完整节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)
全卷 1,081 个 DOI 的 PostgreSQL 只读复查找到 98 个已有绑定,其中 97 个来自带 DOI 节目分区。89 项在现有论文、Crossref 和节目中有一致的规范标题、有序作者及明确节目标签,可优先准备来源关联。其余 9 项保留待核查。98 项的现有历史中无拒稿、桌拒或撤回记录,这一观察仅覆盖已有绑定的作品。此次仍未写入,入库时须重新确认绑定及保护既有历史。
@@ -0,0 +1,204 @@
{
"checked_on": "2026-11-18",
"venue": "IROS",
"year": 2026,
"conference_program": {
"url": "https://2026.ieee-iros.org/program/paper_index/",
"status": "unavailable_in_checked_environment",
"paper_level_full_index_verified": false
},
"publisher_registration_check": {
"status": "no_target_container_in_first_25_relevance_results",
"rows_checked": 25,
"observed_container": "2026 IEEE 13th International Conference on Intelligent Systems (IS)",
"complete_registry_search": false
},
"journal_presentations": {
"journal": "IEEE Transactions on Robotics",
"scope": "IROS 2026 conference presentation of journal papers",
"rendered_index_and_embedded_entries_match": true,
"count": 30,
"xplore_document_ids": [
"10989573",
"11251175",
"11488837",
"11268969",
"11098164",
"11219220",
"11304178",
"11267071",
"11304172",
"11455384",
"11397863",
"11288096",
"11544114",
"11534108",
"11488912",
"11535315",
"11342351",
"11353077",
"11429535",
"11364144",
"11427341",
"11427344",
"11455389",
"11488904",
"11535317",
"11554436",
"11495270",
"11575122",
"11488843",
"11581355"
],
"conference_submission_decision_verified": false,
"publication_dois_verified": false
},
"author_reported_conference_acceptance_candidates": [
{
"arxiv": "2607.06499",
"acceptance_evidence": [
"uw-roundup.html",
"uw-faculty.html"
],
"author_count_current": 3,
"work_identity_status": "existing_work_binding_verified_read_only",
"database_paper_id": 2731624,
"ordered_authors_match": true,
"arxiv_versions_stored": [
"v1"
]
},
{
"arxiv": "2505.04999",
"acceptance_evidence": [
"usc-acceptance.html"
],
"author_list_evidence": "usc-clam-project.html",
"author_count_current": 7,
"work_identity_status": "existing_work_binding_verified_read_only",
"database_paper_id": 2381843,
"ordered_authors_match": true,
"arxiv_versions_stored": [
"v1",
"v2"
]
},
{
"acceptance_evidence": [
"ntt-contributed.html"
],
"author_count_current": 6,
"work_identity_status": "stable_identifier_and_track_pending"
}
],
"purdue_scope_note": {
"evidence": "purdue-mars.html",
"reported_combined_count": 10,
"reported_tro_subset": 2
},
"public_rejected_desk_rejected_withdrawn": "not_located_in_checked_public_sources",
"coverage_status": "partial_source_audit",
"database_written": false,
"receipts": [
{
"name": "ieee-tro-list.html",
"url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
"final_url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
"status": 200,
"bytes": 610713,
"sha256": "2f2bb466446c62c37dc0927f8df45b161a096cbf8ab1aeffc4f43fa5d27e6358"
},
{
"name": "ieee-tro-policy.html",
"url": "https://www.ieee-ras.org/publications/t-ro/",
"final_url": "https://www.ieee-ras.org/publications/t-ro/",
"status": 200,
"bytes": 566001,
"sha256": "37b8877babb62e69d61e8b32193378afad4372117025a139cf70ffc26fcada5c"
},
{
"name": "ntt-contributed.html",
"url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
"final_url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
"status": 200,
"bytes": 221293,
"sha256": "ade0a0320a83e1d4504ff5da24c9f495aed581cffa01aa38b1e4cbec331f6dbc"
},
{
"name": "crossref-proceedings-query.json",
"url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
"final_url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
"status": 200,
"bytes": 13634,
"sha256": "415bf105de8006a42d3f2f8e6eb0078eb54c51bd65315296c0735e853431b228"
},
{
"name": "purdue-mars.html",
"url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
"final_url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
"status": 200,
"bytes": 60386,
"sha256": "fae6ba9637ea141ff3fda93643b97ddfb65d6ea3689f1bca3580956272b2d373"
},
{
"name": "uw-acceptance.html",
"url": "https://faculty.washington.edu/chx/post/2026-06-17-iros-mppi/",
"status": 404,
"error": "HTTP Error 404: "
},
{
"name": "usc-acceptance.html",
"url": "https://liralab.usc.edu/news/",
"final_url": "https://liralab.usc.edu/news/",
"status": 200,
"bytes": 25381,
"sha256": "9fa6b272b7995448a4318dd91cc94b93eeb514ddaa92d7df6880b3d93d91fceb"
},
{
"name": "arxiv-2607.06499.html",
"url": "https://arxiv.org/abs/2607.06499",
"final_url": "https://arxiv.org/abs/2607.06499",
"status": 200,
"bytes": 43362,
"sha256": "98fcbcaf39832f3002fa2910f69ca650481bb473cdb450290e5f9ca9245646a2"
},
{
"name": "arxiv-2505.04999.html",
"url": "https://arxiv.org/abs/2505.04999",
"final_url": "https://arxiv.org/abs/2505.04999",
"status": 200,
"bytes": 42506,
"sha256": "26ea0fd9ac8753b911ff97f1aaf29ea495f50b5c1e106b33c1336ea87c03b0df"
},
{
"name": "uw-faculty.html",
"url": "https://www.me.washington.edu/facultyfinder/xu-chen",
"final_url": "https://www.me.washington.edu/facultyfinder/xu-chen",
"status": 200,
"bytes": 99274,
"sha256": "877af368b564ba26a62b1ce4f947b439cceada2490dfb446b8730ba132c72c78"
},
{
"name": "uw-roundup.html",
"url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
"final_url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
"status": 200,
"bytes": 180954,
"sha256": "bc9ef19a07ea2875f8cfffe9afd62bcc2998fbe7c4bc8a4440c1bc87bae3856b"
},
{
"name": "usc-clam-project.html",
"url": "https://clamrobot.github.io/",
"final_url": "https://clamrobot.github.io/",
"status": 200,
"bytes": 20098,
"sha256": "3b3ea3f817ce6483807ace9b5a42ccf9835b811837346b63411190316fe2819c"
}
],
"work_identity_review": {
"snapshot_at": "2026-11-18 09:26:04.582744+00",
"evidence_sha256": "5e5cfb57525a369e8c81e1a9ea6edeb4f5e83a33e1961eeead8a014e3d59ad6d",
"existing_work_matches": 2,
"database_written": false
}
}
@@ -0,0 +1,21 @@
# IROS 2026 来源核查
2026-11-18 核查了会议节目、期刊报告名单、作者机构公布的录用信息和 DOI 登记线索。逐项来源、响应哈希与文献定位符见[机器可读清单](iros2026-source-audit-2026-11-18.json)。本次来源核查未写入数据库。
## 期刊论文在会上报告
IEEE RAS 的 T-RO 名单在 IROS 2026 一节列出 30 个不同的 IEEE Xplore 文献编号。页面渲染内容和内嵌数据的编号及顺序完全一致。T-RO 政策允许已录用的期刊文章到 IROS 报告。后续应保留期刊出版与会议报告两种关系,并在确认期刊 DOI 后核对现有论文身份。本清单没有逐条验证 DOI 或会议投稿决定。[T-RO 报告名单](https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/) · [T-RO 报告政策](https://www.ieee-ras.org/publications/t-ro/)
普渡大学的通告将团队的 10 项成果一同介绍,其中 2 项属于 T-RO。这个团队样本可以帮助核查来源口径,暂不用于生成逐篇录用记录。[普渡大学通告](https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab)
## 作者机构公布的录用线索
- 华盛顿大学的作者通告列出 IROS 2026 待出版论文,标题与 arXiv 2607.06499 对应,三位作者一致。只读查询确认对应论文 2731624,库内标题与三位作者的顺序一致。[作者通告](https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/) · [arXiv](https://arxiv.org/abs/2607.06499)
- USC 实验室确认 CLAM 在 IROS 2026 录用。项目页指向 arXiv 2505.04999,当前作者名单为七人。库内 arXiv 保留 v1、v2,现有论文 2381843 的标题和七位作者顺序与当前详情页一致。[实验室通告](https://liralab.usc.edu/news/) · [作者项目页](https://clamrobot.github.io/) · [arXiv](https://arxiv.org/abs/2505.04999)
- NTT 公布一条题名及六位作者,待核实稳定标识和会议 track。[NTT 通告](https://group.ntt/en/topics/2026/09/28/iros2026.html)
这两条工作身份已核实,尚未录入 IROS 录用证据。
## 待补证据
[会议论文索引](https://2026.ieee-iros.org/program/paper_index/)在受检环境未能读取。一次限定 IEEE、2026 年及会议论文类型的 Crossref 相关性查询返回的前 25 条属于另一会议。实际会议论文集 DOI、完整出版清单、正式分轨和逐篇覆盖仍待核查。已检查的公开来源中未找到逐篇拒稿、桌拒或撤回清单,继续保持这些决定的公开可得性待查。
+16 -10
View File
@@ -11,7 +11,7 @@ paper-atlas collect mm2026-publications \
--registry-snapshots /data/paper-index/collected \ --registry-snapshots /data/paper-index/collected \
--program-snapshots /data/paper-index/collected \ --program-snapshots /data/paper-index/collected \
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \ --parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
--out /data/paper-index/collected/mm2026-publications-native-v1 --out /data/paper-index/collected/mm2026-publications-native-v3
``` ```
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。 输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
@@ -28,27 +28,31 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
## 分轨、摘要与版本差异 ## 分轨、摘要与版本差异
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。 按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
| 输出分区 | 注册条目 | | 输出分区 | 注册条目 |
| --- | ---: | | --- | ---: |
| main | 1533 | | main | 1555 |
| grand_challenge | 77 | | grand_challenge | 77 |
| interactive_art | 17 | | interactive_art | 17 |
| datasets | 40 | | datasets | 41 |
| reproducibility | 1 | | reproducibility | 1 |
| bni | 22 | | bni | 22 |
| open_source | 11 | | open_source | 11 |
| doctoral_symposium | 2 | | doctoral_symposium | 2 |
| demo_and_video | 20 | | demo_and_video | 23 |
| industry_talk | 3 | | industry_talk | 3 |
| proceedings_unclassified | 192 | | proceedings_unclassified | 166 |
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。 注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。 其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。 8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。 注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
@@ -64,4 +68,6 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。 节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。 实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
+52
View File
@@ -0,0 +1,52 @@
# ReviewArena 镜像的索引字段投影
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
## 运行
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
包内命令如下,`--out` 必须是既有父目录内的新目录:
```sh
paper-atlas collect reviewarena-metadata \
--split emnlp \
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
```
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
```sh
paper-atlas collect reviewarena-metadata --split emnlp \
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
--projection-sha256 <已验证的原报告SHA256> \
--out /data/paper-index/collected/<新解释版本>
```
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
## 实际读取和保存范围
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
## 决定和身份边界
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
+57
View File
@@ -0,0 +1,57 @@
# 出版来源版本与覆盖修正
`paper-atlas maintain source-revisions` 用于同一出版清单的新版本:稳定 DOI 全集、标题、作者和决定不变,但官方分轨、摘要或对应来源证据已经过重新核对。它不导入新论文、不合并实体、不推断拒稿,不把出版分类当成研究类型。
例如,同一清单的“未分类”由 192 条减少到 166 条,而主会等分区增加。直接放松增量导入的计数保护会隐藏清单丢失。本命令先核验完整的新旧清单及数据库快照,再单独修正版本;普通 `import records` 的单调计数规则保持原样。
## 准备完整证据
在工作区外保存 inventory JSON,包含以下字段:
| 字段 | 内容 |
| --- | --- |
| `format` | `1` |
| `source`, `venue`, `year`, `event_key`, `list_url` | 明确的已有来源、会议缩写、年份、事件键和 HTTPS 出版清单地址 |
| `previous_rows`, `rows` | 完整旧版、新版规范化来源记录数组,每版最多 10,000 条;每条包含稳定 DOI、来源字段、作者、摘要、track、决定、标签、原始来源 metadata |
| `evidence` | 一个或多个 `{ "url": "https://…", "sha256": "…" }`,绑定已独立核验的原始证据文件 |
两版必须有完全相同的 DOI 集合;仅允许 `abstract`、`track`、`tags`、`metadata` 的有据变化。必须保留 collector 原始字段,不能将工作身份核查产生的额外锚点混入原始来源版本。只有明确 `accepted` 的出版观察可进入本流程。证据 URL/hash 是审核输入,本命令不联网确认它们;执行者必须另行验证文件哈希、完整性及对应关系。
数据库可以部分导入,甚至各分区的预期数暂时来自不同版本。现有覆盖必须为 `partial`、无错误,导入数与实际来源记录一致;预期数必须恰好属于旧版或新版的同一分区。已有目标 edition 和对应覆盖记录必须存在,目标中可以有其他来源及既有投稿。本版本不会新建 edition,也不会处理共享来源投稿、多次既有 scope curation 的再迁移,或成员集合变化;遇到这些情况会拒绝并要求单独核查。
## 先生成只读计划
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--inventory /data/paper-index/validation/reviewed-source-version.json \
--out /data/paper-index/validation/source-revision-plan.json
```
计划包含明确数据库身份、两个完整来源版本、当前版本父节点、覆盖表原值和修正值、逐条变更,以及受影响会议 scope 内所有论文的规范记录、稳定标识、全部跨会议投稿和决定、来源关系、标签与已存在的 annotation。拒稿后录用的历史也在保护范围内,不能只校验当前会议的录用观察。
计划和原始证据应保存到服务器与独立恢复位置,核对完整差异及计划文件 SHA-256。执行前重新核对生产保护基线与单写入安排。输出文件必须是不存在的新路径,禁止覆盖旧报告。
## 按文件哈希应用并验收
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--plan /data/paper-index/validation/source-revision-plan.json \
--plan-sha256 <已审核的计划文件SHA256> \
--out /data/paper-index/validation/source-revision-result.json
```
应用时重新生成计划并要求完全一致;PostgreSQL 使用已有 writer 锁及相关表写锁,SQLite 使用立即事务。任何保护数据、当前版本或目标标识冲突都使操作失败。操作保留投稿和决定 ID、原始决定证据、规范论文文本、全部稳定标识、来源关系和现有标签;只修改计划列出的投稿/投稿标识 edition,以及来源记录的摘要与 metadata 当前投影。旧投影完整保存在版本历史中,来源原始获取时间不会伪造为修正时间。
覆盖只调整明确分区的预期数和来源观察数,保留 `partial`、cursor、成功时间和错误字段原值。尚未导入的分区不产生虚构投稿或来源观察。覆盖数不能被解读为科学作品数量,或所有 track、摘要、负面决定已完整。
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
## 后续刷新及限制
`import records` 检查当前版本的逐条完整来源字段;`Store.record` 也检查原始文本、作者、来源 metadata 和基础标识,防止直接调用绕回旧版摘要。来源字段必须与当前版本相符,可以额外携带已经核验的工作身份/curation 证据,不能替换来源自身字段。相同来源、会议事件和清单地址下未登记的成员必须另行核验新版本。
版本表通过包内接口追加,未声明数据库管理员的任意 SQL 为安全接口。新来源或新版本在导入前仍需要独立工作身份核查;姓名、标题或无候选都不证明作品身份。实现测试和版本修正验收也不替代所有导入结束后的全库去重、标识冲突、近似标题/作者、摘要、来源覆盖与查询审计。
+2
View File
@@ -20,6 +20,7 @@ COMMANDS = {
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'), ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'), ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
@@ -46,6 +47,7 @@ COMMANDS = {
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'), ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
+123 -14
View File
@@ -7,6 +7,7 @@ import argparse
from collections import Counter, defaultdict from collections import Counter, defaultdict
from datetime import datetime from datetime import datetime
import hashlib import hashlib
from html.parser import HTMLParser
import json import json
from pathlib import Path from pathlib import Path
import re import re
@@ -115,31 +116,138 @@ def load_registry(root, parent_path):
return evidence,rows return evidence,rows
def correlate(rows, observations):
titles=defaultdict(list) class _TitleFormatting(HTMLParser):
for row in observations:titles[key(row['title'])].append(row) """Decode text once, removing only balanced, attribute-free reviewed tags."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts=[];self.stack=[];self.valid=True
def handle_data(self, text):
self.parts.append(text)
def handle_starttag(self, tag, attrs):
if tag not in {'i','scp','sup','sub'} or attrs:
self.valid=False
else:
self.stack.append(tag)
def handle_endtag(self, tag):
if not self.stack or self.stack[-1]!=tag:
self.valid=False
else:
self.stack.pop()
def handle_startendtag(self, tag, attrs):
self.valid=False
def handle_comment(self, text):
self.valid=False
def handle_decl(self, text):
self.valid=False
def handle_pi(self, text):
self.valid=False
def unknown_decl(self, text):
self.valid=False
def formatting_text(title):
# HTMLParser may silently discard a trailing incomplete start tag.
# Check lexical coverage before parsing so no title content can disappear.
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
if '<' in literal or '>' in literal:
return None
parser=_TitleFormatting()
parser.feed(title);parser.close()
text=''.join(parser.parts)
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
'10.1145/3767308.3834936': 'mfp0229',
'10.1145/3767308.3834946': 'mfp0297',
'10.1145/3767308.3834956': 'mfp0339',
'10.1145/3767308.3835035': 'mfp0798',
'10.1145/3767308.3835055': 'mfp0903',
'10.1145/3767308.3835101': 'mfp1175',
'10.1145/3767308.3835374': 'mfp2617',
'10.1145/3767308.3835424': 'mfp2909',
'10.1145/3767308.3835452': 'mfp3048',
'10.1145/3767308.3835460': 'mfp3109',
'10.1145/3767308.3835544': 'mfp3554',
'10.1145/3767308.3835591': 'mfp3783',
'10.1145/3767308.3835667': 'mfp4173',
'10.1145/3767308.3835763': 'mfp4778',
'10.1145/3767308.3835895': 'mfp5537',
'10.1145/3767308.3835959': 'mfp5823',
'10.1145/3767308.3836215': 'mfp7332',
'10.1145/3767308.3836272': 'mfp7635',
'10.1145/3767308.3836293': 'mfp7732',
'10.1145/3767308.3836447': 'mfp8648',
'10.1145/3767308.3836600': 'mfp9815',
'10.1145/3767308.3838656': 'mds177',
'10.1145/3767308.3839012': 'mde41',
'10.1145/3767308.3839024': 'mde15',
'10.1145/3767308.3839028': 'mde02'}
def match_program(rows, observations):
titles=defaultdict(list);formatted=defaultdict(list)
for p in observations:
titles[key(p['title'])].append(p)
text=formatting_text(p['title'])
if text is not None:formatted[key(text)].append(p)
matched=set();kinds=Counter() matched=set();kinds=Counter()
for row in rows: for row in rows:
row.pop('program',None);row.pop('formatting_comparison',None)
candidates=titles[key(row['title'])] candidates=titles[key(row['title'])]
method='literal_title'
# A literal-title author disagreement must not be bypassed by fallback.
if not candidates:
text=formatting_text(row['title'])
if text is not None:
candidates=formatted[key(text)]
if candidates:method='reviewed_title_formatting'
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])] exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
if len(exact)==1: if len(exact)==1:
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval' row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
row['program']=exact[0];matched.add(exact[0]['paper_id']) row['program']=exact[0];matched.add(exact[0]['paper_id'])
if exact[0]['track'] not in TRACKS: if exact[0]['track'] not in TRACKS:
raise ValueError('New registered program track requires review') raise ValueError('New registered program track requires review')
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold' elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
elif candidates:row['kind']='program_author_version_requires_review' elif candidates:
row['kind']=('program_format_author_version_requires_review'
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
else:row['kind']='no_exact_program_title_requires_review' else:row['kind']='no_exact_program_title_requires_review'
row['program_title_candidates']=[p['paper_id'] for p in candidates] row['program_title_candidates']=[p['paper_id'] for p in candidates]
row['program_exact_candidates']=[p['paper_id'] for p in exact] row['program_exact_candidates']=[p['paper_id'] for p in exact]
if method=='reviewed_title_formatting':
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
'registry_comparison_text':formatting_text(row['title']),
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
'raw_titles_preserved':True,'work_identity_approved':False}
kinds[row['kind']]+=1 kinds[row['kind']]+=1
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
if kinds != expected:
raise ValueError('Reviewed program/publication correspondence changed')
return dict(kinds),[p for p in observations if p['paper_id'] not in matched] return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
def correlate(rows, observations):
kinds,unmatched=match_program(rows,observations)
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
'program_author_version_requires_review':122,
'program_format_author_version_requires_review':1,
'no_exact_program_title_requires_review':43}
actual={r['doi']:r['program']['paper_id'] for r in rows
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
if kinds != expected or actual != FORMATTING_MATCHES:
raise ValueError('Reviewed program/publication correspondence changed')
return kinds,unmatched
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot): def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
out=Path(out) out=Path(out)
if out.exists() or out.is_symlink(): if out.exists() or out.is_symlink():
@@ -170,7 +278,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'venue':'MM','year':2026,'track':track,'status':'accepted', 'venue':'MM','year':2026,'track':track,'status':'accepted',
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.', 'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags, 'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'], 'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'], 'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']), 'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
'canonical_object_json_sha256':item['original_canonical_json_sha256']}, 'canonical_object_json_sha256':item['original_canonical_json_sha256']},
@@ -180,7 +288,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'program_fields':p,'program_match_kind':item['kind'], 'program_fields':p,'program_match_kind':item['kind'],
'program_title_candidates':item['program_title_candidates'], 'program_title_candidates':item['program_title_candidates'],
'program_exact_candidates':item['program_exact_candidates'], 'program_exact_candidates':item['program_exact_candidates'],
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources', 'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
'registry_abstract_available':False,'original_decision_signature_verified':False, 'registry_abstract_available':False,'original_decision_signature_verified':False,
'scientific_work_identity_approved':False,'research_type_classified':False, 'scientific_work_identity_approved':False,'research_type_classified':False,
'review_flags':item['flags'], 'review_flags':item['flags'],
@@ -189,7 +297,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
for track,rows in sorted(buckets.items()): for track,rows in sorted(buckets.items()):
path=out/f'mm-publications-2026-{track}.jsonl' path=out/f'mm-publications-2026-{track}.jsonl'
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw) raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows), manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI, 'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows), 'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'} 'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
@@ -199,12 +307,13 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched, 'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918, 'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing, 'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']), 'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications), 'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False, 'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
'public_negative_availability':'unknown_no_negative_decisions_inferred', 'public_negative_availability':'unknown_no_negative_decisions_inferred',
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'} 'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417): if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
raise ValueError('Reviewed output partition or abstract census changed') raise ValueError('Reviewed output partition or abstract census changed')
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json') part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
return report return report
+216
View File
@@ -0,0 +1,216 @@
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
from importlib.resources import files
import json
from pathlib import Path
import re
from paper_atlas.io.parquet_metadata import RangeSource, projection
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
SOURCE = 'reviewarena_mirror'
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
'title': 'string', 'abstract': 'string', 'authors': 'strings',
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
def inventory(split):
if split not in SPLITS:
raise ValueError('Explicit supported mirror split required')
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
raise ValueError('Pinned mirror file inventory changed')
items = json.loads(raw)
if len(items) != 15:
raise ValueError('Pinned mirror shard inventory changed')
selected = []
for item in items:
path = item['path']
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
raise ValueError('Unexpected mirror shard path')
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
raise ValueError('Missing source object size/checksum claim')
if path.startswith('data/'+split+'-'):
selected.append(item)
parts = sorted(selected, key=lambda x: x['path'])
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
raise ValueError('Incomplete selected mirror split inventory')
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
raise ValueError('Mirror shard sequence changed')
return parts
def decision_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'reject', 'rejected'}:
return 'rejected'
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
return 'desk_rejected'
if value in {'withdraw', 'withdrawn'}:
return 'withdrawn'
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
return 'accepted'
return 'unknown'
def decision_track_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'accept-findings', 'accept (findings)'}:
return 'findings'
if value in {'accept-main', 'accept (main)'}:
return 'main'
return None
def record(row, split, item, row_index):
if set(row) != set(COLUMNS):
raise ValueError('Unexpected projected metadata fields')
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
raise ValueError('Mirror conference/year outside pinned source scope')
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
'mirror_decision_category': decision_claim(row['decision']),
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
'source_locator': locator, 'identifiers': {},
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
'scientific_work_identity_approved': False, 'research_type_classified': False,
'approved_for_import': False,
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
def cached_projections(path, expected_sha256, split, items):
path = Path(path)
raw = path.read_bytes()
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
or hashlib.sha256(raw).hexdigest() != expected_sha256):
raise ValueError('Cached projection report hash differs')
report = json.loads(raw)
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
or report.get('approved_for_import') != 0):
raise ValueError('Cached report scope differs from permitted mirror projection')
declared = report.get('files', [])
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
raise ValueError('Incomplete cached mirror shard set')
result = {}
for item in items:
fm = next((f for f in declared if f['source_file'] == item['path']), None)
filename = Path(item['path']).stem+'.metadata.jsonl'
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
raise ValueError('Cached mirror source differs')
file = path.parent/filename
if file.is_symlink():
raise ValueError('Symlink metadata cache forbidden')
content = file.read_bytes()
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
raise ValueError('Cached metadata file hash differs')
rows = []
for index, line in enumerate(content.splitlines()):
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
if previous.get(key) != expected[key]:
raise ValueError('Cached raw metadata/identity differs: '+key)
rows.append(row)
proof = fm['projection']
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
or proof['full_file_checksum_verified'] is not False):
raise ValueError('Cached projection proof differs')
result[item['path']] = (rows, proof)
if sum(len(rows) for rows, _ in result.values()) != report['records']:
raise ValueError('Cached projection row count differs')
return result
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
items = inventory(split); out = Path(out)
if bool(from_projection) != bool(projection_sha256):
raise ValueError('Cached projection and exact report hash required together')
if out.exists() or not out.parent.is_dir() or out.is_symlink():
raise ValueError('New output directory in existing parent required')
out.mkdir()
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
observed = datetime.now(timezone.utc).isoformat()
try:
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
for item in items:
if cached is None:
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
source = source_factory(source_url, item['size'])
rows, proof = projection(source, COLUMNS, progress=progress)
else:
rows, proof = cached[item['path']]
if progress:
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
path = out/(Path(item['path']).stem+'.metadata.jsonl')
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
with part.open('wb') as stream:
for index, row in enumerate(rows):
item_record = record(row, split, item, index)
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
stream.write(raw); h.update(raw); records += 1
counts[item_record['mirror_decision_category']] += 1
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
years[(row['year'], row['track'])] += 1
part.rename(path)
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
'projection': proof})
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
'mirror_decision_track_claim_counts': dict(track_claims),
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
'cached_projection_report_sha256': projection_sha256,
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
'approved_for_import': 0,
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
'Every selected row and row locator is retained, including missing or anonymous author data.',
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
'Rows are not independent scientific works or a complete venue/year public negative census.',
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except BaseException as error:
# No source-specific body or signed redirect URLs in failure reports.
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
'completed_files': reports, 'error_type': type(error).__name__,
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--split', required=True, choices=SPLITS)
parser.add_argument('--out', required=True)
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
args = parser.parse_args(argv)
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
progress=lambda row: print(json.dumps(row), flush=True))
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
+128
View File
@@ -0,0 +1,128 @@
"""Versioned publication-source scopes and guards against stale source refreshes."""
import json
import re
TABLE = 'source_scope_revisions'
MARKER = 'source_scope_revisions_v1'
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
def available(c):
if getattr(c, 'postgres', False):
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
def initialize(c):
exists = available(c)
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
if exists != marker:
raise ValueError('Source revision table/marker mismatch')
if exists:
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
if tuple(x[0] for x in description) != COLUMNS:
raise ValueError('Unexpected source revision schema')
return
ddl = '''CREATE TABLE source_scope_revisions (
version_sha256 TEXT PRIMARY KEY,
source_id INTEGER NOT NULL REFERENCES sources(id),
venue_id INTEGER NOT NULL REFERENCES venues(id),
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
if getattr(c, 'postgres', False):
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
c.execute(ddl)
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
def heads(c, source_id):
if not available(c):
return []
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
raise ValueError('Source revision table/marker mismatch')
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
ORDER BY v.version_sha256''', (source_id,)).fetchall()
def current_rows(c, source_id, cache):
active = heads(c, source_id)
signatures = tuple(r[0] for r in active)
# A new head invalidates the cache, even when maintenance uses the same Store.
if cache.get('signatures') != (source_id, signatures):
records = {}; scopes = {}
for signature in signatures:
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
scopes[signature] = json.loads(raw)
for row in scopes[signature]:
ext = row['external_id']
if ext in records and records[ext] != row:
raise ValueError('Conflicting active source snapshots')
records[ext] = row
cache.clear()
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
return cache['records']
def subset(expected, actual):
"""Reviewed source fields must survive additional work-identity/curation evidence."""
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
expected = current_rows(c, source_id, cache).get(external)
if expected is None:
return
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
if any(actual[k] != expected.get(k) for k in actual):
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
raise ValueError('Stale source snapshot metadata or identifiers')
def scope_head(c, source_id, manifest):
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
if not venue:
return None
matches = [r for r in heads(c, source_id)
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
if len(matches) > 1:
raise ValueError('Forked source revision history')
return matches[0][0] if matches else None
def require_manifest(c, source_id, manifest, cache):
current_rows(c, source_id, cache)
signature = scope_head(c, source_id, manifest)
if signature is None:
return
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
raise ValueError('Source coverage manifest differs from current source snapshot')
def require_row(c, source_id, row, manifest, cache):
expected = current_rows(c, source_id, cache).get(row['external_id'])
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
signature = scope_head(c, source_id, scoped_manifest)
if expected is None:
if signature is not None:
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
return
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
raise ValueError('Source listing differs from current source snapshot')
for key, value in expected.items():
if key in ('metadata', 'identifiers'):
okay = subset(value, row.get(key))
else:
okay = row.get(key) == value
if not okay:
raise ValueError('Stale source snapshot field: ' + key)
+4
View File
@@ -96,6 +96,10 @@ class Store:
if found: if found:
matches.add(found[0]) matches.add(found[0])
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
from paper_atlas.db.source_revisions import require_record
if not hasattr(self, '_source_revision_cache'):
self._source_revision_cache = {}
require_record(c, source_id, external_id, url, title, abstract, authors, identifiers, metadata, self._source_revision_cache)
from paper_atlas.db.edition_scopes import preserve from paper_atlas.db.edition_scopes import preserve
metadata = preserve(dict(metadata or {}), old[4] if old else None) metadata = preserve(dict(metadata or {}), old[4] if old else None)
if old: if old:
+7
View File
@@ -101,6 +101,9 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
c.commit() c.commit()
if incremental is not None: if incremental is not None:
require_incremental_parent(c, source_id, manifest, incremental) require_incremental_parent(c, source_id, manifest, incremental)
from paper_atlas.db.source_revisions import require_manifest
store._source_revision_cache = {}
require_manifest(c, source_id, manifest, store._source_revision_cache)
digest = hashlib.sha256() digest = hashlib.sha256()
with path.open('rb') as stream: with path.open('rb') as stream:
for line in stream: for line in stream:
@@ -113,6 +116,10 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
require_track_record(row, manifest) require_track_record(row, manifest)
from paper_atlas.db.events import validate_event_key from paper_atlas.db.events import validate_event_key
validate_event_key(row.get('event_key')) validate_event_key(row.get('event_key'))
from paper_atlas.db.source_revisions import require_row
if not hasattr(store, '_source_revision_cache'):
store._source_revision_cache = {}
require_row(c, source_id, row, manifest, store._source_revision_cache)
external = row['external_id'] external = row['external_id']
if external in seen: if external in seen:
raise ValueError('Duplicate external ID in one collected file: '+external) raise ValueError('Duplicate external ID in one collected file: '+external)
+327
View File
@@ -0,0 +1,327 @@
"""Plan-bound publication scope revisions with complete before-images, never reimport."""
import argparse
from collections import Counter
from contextlib import closing
import copy
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db import source_revisions as versions
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MAX_ROWS = 10000
def validate(inventory):
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
'previous_rows', 'rows', 'evidence'}
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
raise ValueError('Unsupported source revision inventory')
if (type(inventory['year']) is not int or inventory['year'] < 2000
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
or not isinstance(inventory['event_key'], str)):
raise ValueError('Explicit publication source scope required')
if not inventory['list_url'].startswith('https://'):
raise ValueError('HTTPS source listing required')
evidence = inventory['evidence']
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
raise ValueError('Bounded source evidence required')
for item in evidence:
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
raise ValueError('Source evidence URL/hash required')
snapshots = []
for key in ('previous_rows', 'rows'):
items = inventory[key]
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
raise ValueError('Complete bounded source snapshots required')
records = {}
for row in items:
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
raise ValueError('Source snapshot scope mismatch')
ext = row.get('external_id')
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
or row.get('identifiers', {}).get('doi') != ext
or not isinstance(row.get('track'), str) or not row['track']
or row.get('event_key', '') != inventory['event_key']
or not isinstance(row.get('title'), str) or not row['title']
or not isinstance(row.get('authors'), list)
or not isinstance(row.get('metadata'), dict)):
raise ValueError('Unique DOI-bearing accepted publication records required')
records[ext] = row
snapshots.append(records)
previous, current = snapshots
if set(previous) != set(current):
raise ValueError('Source membership changed; this operation cannot add or remove publications')
for ext, row in current.items():
before = previous[ext]
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
if any(before.get(k) != row.get(k) for k in stable):
raise ValueError('Stable publication identity or decision changed')
return previous, current
def bounded(c, sql, params):
result = values(c.execute(sql + ' LIMIT 100001', params))
if len(result) > 100000:
raise ValueError('Source revision protection exceeds bounded scope')
return result
def capture(c, inventory):
previous, current = validate(inventory)
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
if not source or not venue:
raise ValueError('Existing source and venue required')
sid, vid = source[0], venue[0]
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
(vid, inventory['year'], inventory['event_key']))
by_track = {r[3]: r[0] for r in editions}
# The reviewed scope includes all its existing target attempts, even when
# they originate from another source. No empty-target shortcut is used.
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
scope_params = (vid, inventory['year'], inventory['event_key'])
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
if not own_records:
raise ValueError('Existing source observations required')
if any(r[2] not in current for r in own_records):
raise ValueError('Snapshot omits an existing source observation')
pids = sorted({r[1] for r in scoped})
if not pids or len(pids) > MAX_ROWS:
raise ValueError('Bounded nonempty protected paper scope required')
marks = ','.join('?' for _ in pids)
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
queries = {
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
raise ValueError('Unambiguous source listing required')
if len({r[0] for r in coverage}) != len(coverage):
raise ValueError('One source coverage entry per edition required')
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
if len(own_heads) > 1:
raise ValueError('Forked source revision history')
parent = own_heads[0][0] if own_heads else None
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
'parent_sha256': parent, 'by_track': by_track,
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
def changes(c, inventory, before):
previous, current = validate(inventory)
packet = before['packet']; result = []; sid = before['source_id']
editions = {r[0]: r for r in before['editions']}
for raw in before['own_records']:
ext = raw[2]; old = previous[ext]; new = current[ext]
metadata = json.loads(raw[8])
candidates = [old, new]
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
raise ValueError('Current raw source differs from both reviewed snapshots')
if metadata.get('record_identifiers', {}).get('doi') != ext:
raise ValueError('Current stable DOI differs from source evidence')
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
if len(attempts) != 1:
raise ValueError('Exactly one publication attempt per source record required')
sub = attempts[0]; origin = editions[sub[2]]
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
raise ValueError('Current source attempt identity/scope differs')
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
if not decisions or any(d[2] != 'accepted' for d in decisions):
raise ValueError('Only accepted publication observations can be revised')
target = before['by_track'].get(new['track'])
if target is None:
raise ValueError('Reviewed target edition must already exist')
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
if not identities or any(x[0] != sub[2] for x in identities):
raise ValueError('Inconsistent scoped submission identifiers')
for _, scheme, value, _ in identities:
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
if conflict and conflict[0] != sub[0]:
raise ValueError('Target submission identifier collision')
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
if sub[2] != target and KEY in metadata:
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
'from_track': origin[3], 'track': new['track'],
'new_abstract': new.get('abstract'), 'metadata': replacement})
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
observed = Counter()
own_ids = {r[0] for r in before['own_records']}
for s in packet['submissions']:
if s[3] in own_ids and s[2] in editions:
observed[s[2]] += 1
coverage = []
for raw in before['coverage']:
track = editions[raw[0]][3]
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
raise ValueError('Coverage parent is not the verified partial source observation count')
if raw[4] not in {old_expected[track], new_expected[track]}:
raise ValueError('Coverage expected count differs from both source versions')
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
if count < 0 or count > new_expected[track]:
raise ValueError('Corrected coverage exceeds current source membership')
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
covered = {r['before'][0] for r in coverage}
if any(r['target_edition_id'] not in covered for r in result):
raise ValueError('Target source coverage must already exist')
return result, coverage
def proposal(c, inventory, identity):
before = capture(c, inventory)
if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before)
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
'changes': records, 'coverage': coverage}
def verify(c, plan, signature):
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
for change in plan['changes']:
for row in expected['submissions']:
if row[0] == change['submission_id']:
row[2] = change['target_edition_id']
for row in expected['submission_identifiers']:
if row[3] == change['submission_id']:
row[0] = change['target_edition_id']
for row in expected['source_records']:
if row[0] == change['source_record_id']:
row[6] = change['new_abstract']
row[8] = encoded_metadata(plan, change, signature)
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
if after['packet'] != expected:
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
if after['editions'] != plan['before']['editions']:
raise ValueError('Edition identities changed')
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
inventory = plan['inventory']
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
if after['revision_heads'] != sorted(expected_heads):
raise ValueError('Active source version differs')
expected_coverage = []
for item in plan['coverage']:
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
expected_coverage.append(row)
if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
def encoded_metadata(plan, change, signature):
metadata = copy.deepcopy(change['metadata'])
if change['from_edition_id'] != change['target_edition_id']:
inventory = plan['inventory']
evidence = inventory['evidence'][0]
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
return json.dumps(metadata, ensure_ascii=False)
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported source revision plan')
if not getattr(c, 'postgres', False) and not c.in_transaction:
c.execute('BEGIN IMMEDIATE')
signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan:
raise ValueError('Stale source revision plan; create and review a fresh plan')
versions.initialize(c)
for row in plan['changes']:
if row['from_edition_id'] != row['target_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
for row in plan['coverage']:
before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
out = Path(out)
if out.exists() or not out.parent.is_dir():
raise ValueError('New output in existing directory required')
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
raise ValueError('Choose inventory preflight or plan plus exact file hash')
if inventory:
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
else:
raw = Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != plan_sha256:
raise ValueError('Plan file hash differs')
proposed = json.loads(raw)
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
try:
if proposed['database'] != database_identity(c, db):
raise ValueError('Plan belongs to another database')
if not getattr(c, 'postgres', False):
c.commit(); c.execute('BEGIN IMMEDIATE')
result = apply_plan(c, proposed); c.commit()
except BaseException:
c.rollback(); raise
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)
+195
View File
@@ -0,0 +1,195 @@
"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()
+1 -1
View File
@@ -8,7 +8,7 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
'submission_sources', 'submission_identifiers', 'decision_sources', 'submission_sources', 'submission_identifiers', 'decision_sources',
'submission_merges', 'decision_merges', 'schema_migrations', 'tags', 'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs') 'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
OPTIONAL_TABLES = ('annotation_events',) OPTIONAL_TABLES = ('annotation_events', 'source_scope_revisions')
CHECKS = { CHECKS = {
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'], 'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
'venues': ["venue_type IN ('conference','journal','review_platform')"], 'venues': ["venue_type IN ('conference','journal','review_platform')"],
@@ -0,0 +1 @@
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
+2 -1
View File
@@ -12,6 +12,7 @@ dependencies = ["PyYAML==6.0.3"]
[project.optional-dependencies] [project.optional-dependencies]
postgres = ["psycopg[binary]==3.3.8"] postgres = ["psycopg[binary]==3.3.8"]
mirrors = ["pyarrow==26.0.1"]
[project.scripts] [project.scripts]
paper-atlas = "paper_atlas.cli:main" paper-atlas = "paper_atlas.cli:main"
@@ -22,4 +23,4 @@ exclude = ["tests*"]
[tool.setuptools.package-data] [tool.setuptools.package-data]
"paper_atlas.db" = ["schema.sql", "annotations.sql"] "paper_atlas.db" = ["schema.sql", "annotations.sql"]
"paper_atlas.resources" = ["venues.json"] "paper_atlas.resources" = ["venues.json", "reviewarena-files.json"]
+706
View File
@@ -0,0 +1,706 @@
{
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
"gold": [
{
"doi": "10.1145/3767308.3834907",
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
"authors": [
"Jia Wang",
"Jiaming Cai",
"Zunying Hu",
"Zhanjie Wu",
"Jinyuan Liu",
"Hua Cheng",
"Yun Peng",
"Xin Fan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 246,
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
"program_id": "mfp0053",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
"program_context_pointers": [
"/entries/129/papers/19",
"/entries/129/subsessions/1/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834936",
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
"authors": [
"Jiacheng Lu",
"Weijian Wang",
"Mingyuan Xiao",
"Yang Hua",
"Tao Song",
"Bo Peng",
"Cheng Hua",
"Haibing Guan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 25,
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
"program_id": "mfp0229",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
"program_context_pointers": [
"/entries/109/papers/27",
"/entries/109/subsessions/2/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834946",
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 35,
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
"program_id": "mfp0297",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
"program_context_pointers": [
"/entries/61/papers/40",
"/entries/61/subsessions/3/items/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834956",
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Baojin Huang",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 235,
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
"program_id": "mfp0339",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
"program_context_pointers": [
"/entries/88/papers/48",
"/entries/88/subsessions/4/items/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835035",
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
"authors": [
"Haoqian Kang",
"Liupeng Li",
"Kuofeng Gao",
"Jinpeng Wang",
"Zhenyu Lu",
"Bin Chen",
"Ke Chen",
"Yaowei Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 8,
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
"program_id": "mfp0798",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
"program_context_pointers": [
"/entries/88/papers/39",
"/entries/88/subsessions/3/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835055",
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
"authors": [
"Qiang Qi",
"Wenqi Shang",
"Xiao Wang",
"Shuyuan Lin"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 28,
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
"program_id": "mfp0903",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
"program_context_pointers": [
"/entries/129/papers/55",
"/entries/129/subsessions/4/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835101",
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
"authors": [
"Zifan Zheng",
"Yu Luo",
"Yuxin Huang",
"Chuancheng Shi",
"Mohammad Masbaul Alam Polash",
"Zhiyong Wang",
"Kun Hu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 82,
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
"program_id": "mfp1175",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
"program_context_pointers": [
"/entries/109/papers/86",
"/entries/109/subsessions/7/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835374",
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
"authors": [
"Qianpeng Chong",
"Jiakun Wen",
"Guangyi Wei",
"Wenyi Zeng"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 59,
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
"program_id": "mfp2617",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
"program_context_pointers": [
"/entries/61/papers/80",
"/entries/61/subsessions/6/items/8"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835424",
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
"authors": [
"Evan Ram",
"Tuan Tran",
"Youngwook Son",
"Sangtae Ha"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 108,
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
"program_id": "mfp2909",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
"program_context_pointers": [
"/entries/129/papers/130",
"/entries/129/subsessions/8/items/22"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835452",
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
"authors": [
"He Peize",
"Yaodi Luo",
"Xiaoqian Liu",
"Xuyang Liu",
"Jiahang Deng",
"Yaosong Du",
"Li Bangyu",
"Xiyan Gui",
"Yuxuan Chen",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 167,
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
"program_id": "mfp3048",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
"program_context_pointers": [
"/entries/46/papers/78",
"/entries/46/subsessions/6/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835460",
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
"authors": [
"Chi Li",
"Rui Lin",
"Aobo Ji",
"Dongzhu Xu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 175,
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
"program_id": "mfp3109",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
"program_context_pointers": [
"/entries/109/papers/121",
"/entries/109/subsessions/8/items/13"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835544",
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
"authors": [
"Boyan Li",
"Ruijun Mao",
"Zhicheng Liang",
"Dayou Zhang",
"Fangxin Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 129,
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
"program_id": "mfp3554",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
"program_context_pointers": [
"/entries/46/papers/153",
"/entries/46/subsessions/9/items/21"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835591",
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
"authors": [
"Jiafeng Huang",
"Wenzhen Jia",
"Shilong Dong",
"Shengjie Zhao"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 97,
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
"program_id": "mfp3783",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
"program_context_pointers": [
"/entries/33/papers/158",
"/entries/33/subsessions/10/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835667",
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
"authors": [
"Yongqi Shan",
"Yifan Wang",
"Yunzhi Zhuge",
"Lijun Wang",
"Huchuan Lu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 209,
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
"program_id": "mfp4173",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
"program_context_pointers": [
"/entries/88/papers/114",
"/entries/88/subsessions/8/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835763",
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
"authors": [
"Yuxuan Wang",
"He Peize",
"Xiyan Gui",
"Xiaoqian Liu",
"Junhao He",
"Xuyang Liu",
"Xuming Hu",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 192,
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
"program_id": "mfp4778",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
"program_context_pointers": [
"/entries/129/papers/100",
"/entries/129/subsessions/7/items/16"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835895",
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
"authors": [
"Zan Gao",
"Xin Duan",
"Nian Hu",
"Hao Jia",
"Yibo Zhao",
"Cong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-001.json",
"source_position": 39,
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
"program_id": "mfp5537",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
"program_context_pointers": [
"/entries/88/papers/184",
"/entries/88/subsessions/11/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835959",
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
"authors": [
"Weiliang Huang",
"Weichao Cai",
"Bob Zhang",
"Chao Huang",
"Biao Xue",
"Rongxin Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 21,
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
"program_id": "mfp5823",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
"program_context_pointers": [
"/entries/88/papers/66",
"/entries/88/subsessions/5/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836215",
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
"authors": [
"Wentao Tan",
"Bowen Wang",
"Heng Zhi",
"Chenyv Liu",
"Zhe Li",
"Jian Liu",
"Zengrong Lin",
"Yukun Dai",
"Yipeng Chen",
"Wenjie Yang",
"Enci Xie",
"Hao Xue",
"Baixu Ji",
"Chen Xu",
"Zhibin Wang",
"Tianshi Wang",
"Lei Zhu",
"Heng Tao Shen"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 46,
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
"program_id": "mfp7332",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
"program_context_pointers": [
"/entries/33/papers/13",
"/entries/33/subsessions/1/items/1",
"/entries/47/papers/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836272",
"title": "SEE: Structure-aware Exploring &amp; Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
"authors": [
"Zhuohang Fan",
"Beichen Zhang",
"Yuanfa Li",
"Changqiao Wu",
"Wei Liu",
"Jian Luan",
"Weigang Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 127,
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
"program_id": "mfp7635",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
"program_context_pointers": [
"/entries/88/papers/131",
"/entries/88/subsessions/8/items/23"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836293",
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
"authors": [
"Ruizhe Chen",
"Zhiting Fan",
"Yang Shi",
"Enxin Song",
"Wenhao Chai",
"Tongkun Guan",
"Songtao Jiang",
"Ruilin Luo",
"Yuanxing Zhang",
"Shuai Bai",
"Sibo Song",
"Zhibo Yang",
"Zuozhu Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 148,
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
"program_id": "mfp7732",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
"program_context_pointers": [
"/entries/46/papers/198",
"/entries/46/subsessions/11/items/20"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836447",
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
"authors": [
"Raghu Vamsi Chittersu",
"Yuvraj Singh Rathore",
"Pranav Adlinge",
"Eswar Sai Pashaula",
"Kunal Swami"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 183,
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
"program_id": "mfp8648",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
"program_context_pointers": [
"/entries/107/papers/4",
"/entries/117/papers/30",
"/entries/117/subsessions/2/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836600",
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
"authors": [
"Mansi Sharma",
"Dinar Orazgaliyev",
"Antonio Krüger",
"Philipp Müller"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 140,
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
"program_id": "mfp9815",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
"program_context_pointers": [
"/entries/46/papers/219",
"/entries/46/subsessions/12/items/17"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3838656",
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
"authors": [
"Xiaochen Wang",
"Yuan Zhong",
"Lingwei Zhang",
"Lisong Dai",
"Ting Wang",
"Fenglong Ma"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 225,
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
"program_id": "mds177",
"track_literal": "Datasets",
"track": "datasets",
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
"program_context_pointers": [
"/entries/46/papers/23",
"/entries/46/subsessions/1/items/11"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839012",
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
"authors": [
"Abhijeet Narang",
"Hagar Elbatanouny",
"Vanishri Shekhawat",
"Hasan Al-Nashash",
"Abhinav Dhall",
"Usman Tariq"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 238,
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
"program_id": "mde41",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
"program_context_pointers": [
"/entries/94/papers/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839024",
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
"authors": [
"Jiangfan Liu",
"Zexuan Cui",
"Tianyuan Zhang",
"Zonglei Jing",
"Yaoyuan Zhang",
"Jiakai Wang",
"Jinyang Guo",
"Yuqing Ma",
"Lu Sheng",
"Aishan Liu",
"Xianglong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 0,
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
"program_id": "mde15",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
"program_context_pointers": [
"/entries/49/papers/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839028",
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
"authors": [
"Xiaoyan Yu",
"Ziya Zhao",
"Yunshan Ma",
"Yifan Wei",
"Dianbo Sui",
"Tat-Seng Chua"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 4,
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
"program_id": "mde02",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
"program_context_pointers": [
"/entries/49/papers/0"
],
"single_name_positions": []
}
],
"author_disagreement_hold": {
"doi": "10.1145/3767308.3836308",
"program_id": "mfp7801"
},
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
"held_gold": {
"doi": "10.1145/3767308.3836308",
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
"authors": [
"Zhirong Tian",
"Yi Zhang",
"Weixuan Liang",
"Miaomiao Li",
"Xinwang Liu"
],
"kind": "program_format_author_version_requires_review",
"source_page": "page-001.json",
"source_position": 82,
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
"program_id": null,
"track_literal": null,
"track": "proceedings_unclassified",
"abstract_sha256": null,
"program_context_pointers": [],
"single_name_positions": []
}
}
+67 -12
View File
@@ -19,6 +19,9 @@ class MMPublications2026Tests(unittest.TestCase):
fixtures=Path(__file__).parent/'fixtures' fixtures=Path(__file__).parent/'fixtures'
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes())) cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes())) cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name) cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
cls.materialize(cls.root/'registry',cls.fixture['files']) cls.materialize(cls.root/'registry',cls.fixture['files'])
cls.materialize(cls.root/'program',cls.program_fixture['files']) cls.materialize(cls.root/'program',cls.program_fixture['files'])
@@ -38,8 +41,8 @@ class MMPublications2026Tests(unittest.TestCase):
def test_all1918_publications_against_independent_registry_program_gold(self): def test_all1918_publications_against_independent_registry_program_gold(self):
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918) self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']}) self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
for g in self.fixture['independent_gold']: for g in self.gold:
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields'] r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track'])) self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
self.assertEqual(m['program_match_kind'],g['kind']) self.assertEqual(m['program_match_kind'],g['kind'])
@@ -51,22 +54,22 @@ class MMPublications2026Tests(unittest.TestCase):
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self): def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
count=0 count=0
for g in self.fixture['independent_gold']: for g in self.gold:
r=self.by_doi[g['doi']];m=r['metadata'] r=self.by_doi[g['doi']];m=r['metadata']
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract')) self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
if g['abstract_sha256'] is not None: if g['abstract_sha256'] is not None:
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256']) count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
self.assertEqual(r['abstract'],m['program_fields']['abstract']) self.assertEqual(r['abstract'],m['program_fields']['abstract'])
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence') self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
else: else:
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources') self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201) self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self): def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']} programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
for p in self.report['unmatched_program']: for p in self.report['unmatched_program']:
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417) self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
gold=self.program_fixture['independent_inventory']['inventory'] gold=self.program_fixture['independent_inventory']['inventory']
self.assertEqual(set(programs),{g['paper_id'] for g in gold}) self.assertEqual(set(programs),{g['paper_id'] for g in gold})
for g in gold: for g in gold:
@@ -91,7 +94,7 @@ class MMPublications2026Tests(unittest.TestCase):
for track in ('interactive_art','industry_talk','demo_and_video'): for track in ('interactive_art','industry_talk','demo_and_video'):
self.assertTrue(any(r['track']==track for r in self.rows)) self.assertTrue(any(r['track']==track for r in self.rows))
variants=[r for r in self.rows if r['track']=='proceedings_unclassified'] variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
self.assertEqual(len(variants),192) self.assertEqual(len(variants),166)
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants)) self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self): def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
@@ -113,7 +116,7 @@ class MMPublications2026Tests(unittest.TestCase):
def test_eight_single_name_fields_preserved_without_splitting(self): def test_eight_single_name_fields_preserved_without_splitting(self):
n=0 n=0
for g in self.fixture['independent_gold']: for g in self.gold:
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags'] r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split'] actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
self.assertEqual(actual,g['single_name_positions']);n+=len(actual) self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
@@ -123,13 +126,62 @@ class MMPublications2026Tests(unittest.TestCase):
def test_structured_title_markup_retained_and_flagged_without_flattening(self): def test_structured_title_markup_retained_and_flagged_without_flattening(self):
count=0 count=0
for g in self.fixture['independent_gold']: for g in self.gold:
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']): if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
count+=1;r=self.by_doi[g['doi']] count+=1;r=self.by_doi[g['doi']]
self.assertEqual(r['title'],g['title']) self.assertEqual(r['title'],g['title'])
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags']) self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags']) self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
self.assertEqual(len(self.format_gold['gold']),26)
for gold in self.format_gold['gold']:
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
comparison=m['program_title_formatting_comparison']
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
self.assertFalse(m['scientific_work_identity_approved'])
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
self.assertEqual(held['metadata']['program_exact_candidates'],[])
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> &amp; C'),'A H2 B1 & C')
self.assertEqual(c.formatting_text('A &amp;amp; B'),'A &amp; B')
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A &lt;i&gt;B&lt;/i&gt;',
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
def observation(pid,authors):
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
self.assertNotIn('program',rows[0])
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
c.match_program(rows,[observation('p1',['A','B'])])
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
self.assertNotIn('program',rows[0])
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
c.match_program(rows,observations)
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self): def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
for kind in ('registry_manifest','registry_page','parent','program'): for kind in ('registry_manifest','registry_page','parent','program'):
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
@@ -150,9 +202,12 @@ class MMPublications2026Tests(unittest.TestCase):
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json') c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self): def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}) self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
self.assertEqual(self.report['registry_missing_abstract_count'],1918) self.assertEqual(self.report['registry_missing_abstract_count'],1918)
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717) self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
self.assertEqual(self.report['unmatched_program_count'],391)
self.assertEqual(sum(f['count'] for f in self.report['files']),1918) self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written']) self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0) self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
@@ -196,7 +251,7 @@ class MMPublications2026Tests(unittest.TestCase):
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata']) meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall() tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918) self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726) self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'}) self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall() decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)]) self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
+203
View File
@@ -0,0 +1,203 @@
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
from collections import Counter
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import re
import struct
import tempfile
import unittest
from unittest.mock import patch
import urllib.error
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
class MemorySource:
def __init__(self, raw):
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
def read(self, start, length):
raw = self.raw[start:start+length]
self.bytes_read += len(raw)
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
return raw
def fixture():
import pyarrow as pa
import pyarrow.parquet as pq
rows = []
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
'track': 'main', 'venue_id': 'source-venue-claim',
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
rows.append(row)
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
class RangeHTTPTests(unittest.TestCase):
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
response = io.BytesIO(data); response.status = status
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
return response
def test_strict_range_and_changed_object_detection(self):
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
source = RangeSource('https://example.org/public', 10, opener=opener)
self.assertEqual(source.read(2, 2), b'ab')
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
with self.assertRaisesRegex(ValueError, 'ETag'):
source.read(4, 2)
self.assertEqual(source.bytes_read, 2)
def test_no_full_file_fallback_or_denied_source_retry(self):
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
opener.open.return_value = response
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
response.read.assert_not_called()
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
self.assertEqual(opener.open.call_count, 1)
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
with self.assertRaises(ValueError):
source.seek(-101, 2)
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
class ParquetProjectionTests(unittest.TestCase):
def test_complete_selected_rows_and_no_excluded_page_requests(self):
import pyarrow.parquet as pq
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
self.assertEqual(rows, expected)
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
excluded = []
for i in range(metadata.num_row_groups):
group = metadata.row_group(i)
for j in range(group.num_columns):
col = group.column(j)
if col.path_in_schema.split('.')[0] not in COLUMNS:
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
excluded.append((start, start+col.total_compressed_size))
for request in source.receipts:
a, b = request['start'], request['start']+request['length']
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
self.assertLess(source.bytes_read, len(raw))
encoded = json.dumps(rows)+json.dumps(proof)
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
raw, _ = fixture(); source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'Missing'):
projection(source, {'missing_column': 'string'})
self.assertEqual(len(source.receipts), 3)
source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'type differs'):
projection(source, {**COLUMNS, 'year': 'string'})
self.assertEqual(len(source.receipts), 3)
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
source = MemorySource(bytes(bad))
with self.assertRaisesRegex(ValueError, 'footer'):
projection(source, COLUMNS)
self.assertEqual(len(source.receipts), 2)
def test_collector_output_preserves_claims_without_original_identity(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'projected'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
for row in projected:
self.assertEqual(row['identifiers'], {})
self.assertFalse(row['metadata']['original_forum_id_verified'])
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertNotIn('status', row)
self.assertIn(COMMIT, row['external_id'])
self.assertEqual(projected[2]['authors'], ['Anonymous'])
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
self.assertEqual(projected[0]['track'], 'main')
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
all_saved=''.join(p.read_text() for p in out.iterdir())
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
with self.assertRaisesRegex(ValueError, 'New output'):
collect('emnlp', out)
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
# Mimic the first reviewed source version: new literal spellings were unknown.
for row in before[:2]:
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
with self.assertRaisesRegex(ValueError, 'hash differs'):
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
no_network.assert_not_called()
def test_failure_is_retained_without_source_payload_or_retries(self):
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'failed'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
with self.assertRaises(RuntimeError):
collect('emnlp', out, source_factory=lambda url, size: None)
report = json.loads((out/'failure.json').read_text())
self.assertEqual(report['status'], 'incomplete_do_not_import')
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
class MirrorSemanticsTests(unittest.TestCase):
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
result = {s: inventory(s) for s in splits}
self.assertEqual(sum(len(v) for v in result.values()), 15)
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
self.assertEqual(result['emnlp'][0]['size'], 90794691)
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
with self.assertRaises(ValueError):
inventory('uai')
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
self.assertEqual(decision_claim(literal), 'unknown')
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
self.assertIsNone(decision_track_claim('Accept (Poster)'))
+209
View File
@@ -0,0 +1,209 @@
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
from contextlib import closing, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest
from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values
from paper_atlas.ingestion.records import coverage_options, ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
URL = 'https://example.org/volume'
def seed(path):
old = []
for i in range(4):
doi = '10.1234/publication'+str(i)
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
current = copy.deepcopy(old)
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
current[1]['tags'][0]['value'] = 'main'
store = Store(path)
for row in old[:3]:
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
store.record_tags(p, r, 'fixture', row['tags'])
if row == old[1]:
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
# An unrelated source already has an attempt in the populated target edition.
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
store.conn.commit(); store.conn.close()
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
class RevisionCases:
def store(self):
return open_store(**self.target)
def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
before_papers = plan['before']['packet']['papers']
before_tags = plan['before']['packet']['paper_tags']
result = apply_plan(c, plan); c.commit()
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
self.assertEqual(result['protected_decisions'], 5)
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
self.assertEqual(saved, plan)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
with closing(self.store().conn) as c:
store = object.__new__(Store); store.conn = c; store.event_identity = True
old = self.inventory['previous_rows'][1]
with self.assertRaisesRegex(ValueError, 'Stale source'):
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
c.rollback()
row = self.inventory['rows'][1]
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
c.rollback()
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
with self.assertRaisesRegex(ValueError, 'source snapshot'):
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
with closing(self.store().conn) as c:
apply_plan(c, self.plan(c)); c.commit()
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
with self.assertRaisesRegex(ValueError, 'absent from current'):
require_row(c, sid, unknown, {'list_url': URL}, cache)
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
with self.assertRaisesRegex(ValueError, 'Stale source'):
require_row(c, sid, bad, {'list_url': URL}, cache)
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
require_row(c, sid, good, {'list_url': URL}, cache)
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
with self.assertRaisesRegex(ValueError, 'Stale'):
apply_plan(c, plan)
c.rollback()
move = plan['changes'][0]
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
with self.assertRaisesRegex(ValueError, 'collision'):
self.plan(c)
c.rollback()
self.assertFalse(available(c))
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
with self.assertRaisesRegex(ValueError, 'Injected'):
apply_plan(c, plan)
c.rollback()
self.assertFalse(available(c))
self.assertEqual(self.plan(c), plan)
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
path = self.root/'plan.json'
args = {k: v for k, v in self.target.items() if k != 'db'}
run(self.target.get('db'), path, inventory=inventory, **args)
with closing(self.store().conn) as c:
self.assertFalse(available(c))
with self.assertRaisesRegex(ValueError, 'hash'):
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
self.assertEqual(report['events_moved'], 1)
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
lambda x: x['rows'][0].update(status='rejected')]:
bad = copy.deepcopy(self.inventory); mutate(bad)
with self.assertRaises(ValueError):
validate(bad)
with self.assertRaisesRegex(ValueError, 'incremental'):
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
def test_optional_revision_table_migrates_with_full_history(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def drop_schema(service, schema):
with connect(service) as c:
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db)
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
self.target = {'pg_service': service, 'pg_schema': schema}