Compare commits

..

3 Commits

Author SHA1 Message Date
codex-bot 7d9c693761 Review mirror publication literals and replay frozen metadata offline
Metadata integrity checks / metadata (push) Has started running
2026-11-18 09:34:10 +08:00
codex-bot 98ef415912 Use Arrow binary wheels compatible with runner Python 3.14
Metadata integrity checks / metadata (push) Successful in 7m13s
2026-11-18 09:14:27 +08:00
codex-bot 90fd789a17 Project pinned ReviewArena metadata without full-text payload pages
Metadata integrity checks / metadata (push) Failing after 47s
2026-11-18 09:10:29 +08:00
14 changed files with 800 additions and 921 deletions
+4 -8
View File
@@ -35,7 +35,7 @@ jobs:
venv = root / 'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python = str(venv / 'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True)
testenv = os.environ.copy()
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
if service_file.is_file():
@@ -75,15 +75,11 @@ jobs:
path.write_bytes(body.encode('utf-8'))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
assert len(mm_pub_rows)==1918
mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
for original_gold in mm_pub_fixture['independent_gold']:
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
for gold in mm_pub_fixture['independent_gold']:
row=mm_pub_rows[gold['doi']]
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
+88
View File
@@ -0,0 +1,88 @@
name: ReviewArena metadata projection
on:
workflow_dispatch:
inputs:
split:
description: 'Pinned public mirror family; metadata columns only'
required: true
default: emnlp
projection_name:
description: 'Optional existing metadata output directory name for offline replay'
required: false
default: ''
projection_sha256:
description: 'Exact cached collection-report.json SHA256, required for offline replay'
required: false
default: ''
jobs:
project:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
MIRROR_SPLIT: ${{ inputs.split }}
MIRROR_PROJECTION: ${{ inputs.projection_name }}
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Project permitted metadata on the approved remote runner
run: |
python3 -B - <<'PY'
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
split = os.environ['MIRROR_SPLIT']
cached = os.environ.get('MIRROR_PROJECTION','')
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
raise SystemExit('Explicit approved source commit/split required')
base = pathlib.Path('/data/paper-index')
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
if out.exists():
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
extra=[]
if cached:
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
raise SystemExit('Explicit permitted cached projection name and report hash required')
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
elif cached_sha:
raise SystemExit('Cached projection name required')
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
with urllib.request.urlopen(url,timeout=60) as response:
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Oversized source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
tar.extractall(root,filter='data')
candidates=list(root.glob('*/pyproject.toml'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive')
src=candidates[0].parent
venv=root/'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python=str(venv/'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
wheels=root/'wheels'
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
wheel,=wheels.glob('paper_atlas-*.whl')
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside=root/'outside';outside.mkdir()
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
report=json.loads((out/'collection-report.json').read_bytes())
assert report['database_written'] is False and report['approved_for_import']==0
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
backup=base/'validation'/(out.name+'-recovery.tar.gz')
if backup.exists():
raise SystemExit('Recovery archive already exists; inspect')
with tarfile.open(backup,'w:gz') as tar:
tar.add(out,arcname=str(out.relative_to(base)))
backup.chmod(0o444)
for path in out.iterdir():
if path.is_file():path.chmod(0o444)
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
PY
+3 -1
View File
@@ -2,6 +2,8 @@
**长期维护、按需运行的论文元数据与公开投稿历史工具。**
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
@@ -143,4 +145,4 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
+10 -16
View File
@@ -11,7 +11,7 @@ paper-atlas collect mm2026-publications \
--registry-snapshots /data/paper-index/collected \
--program-snapshots /data/paper-index/collected \
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
--out /data/paper-index/collected/mm2026-publications-native-v3
--out /data/paper-index/collected/mm2026-publications-native-v1
```
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
@@ -28,31 +28,27 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
## 分轨、摘要与版本差异
按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
| 输出分区 | 注册条目 |
| --- | ---: |
| main | 1555 |
| main | 1533 |
| grand_challenge | 77 |
| interactive_art | 17 |
| datasets | 41 |
| datasets | 40 |
| reproducibility | 1 |
| bni | 22 |
| open_source | 11 |
| doctoral_symposium | 2 |
| demo_and_video | 23 |
| demo_and_video | 20 |
| industry_talk | 3 |
| proceedings_unclassified | 166 |
| proceedings_unclassified | 192 |
注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
@@ -68,6 +64,4 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
+52
View File
@@ -0,0 +1,52 @@
# ReviewArena 镜像的索引字段投影
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
## 运行
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
包内命令如下,`--out` 必须是既有父目录内的新目录:
```sh
paper-atlas collect reviewarena-metadata \
--split emnlp \
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
```
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
```sh
paper-atlas collect reviewarena-metadata --split emnlp \
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
--projection-sha256 <已验证的原报告SHA256> \
--out /data/paper-index/collected/<新解释版本>
```
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
## 实际读取和保存范围
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
## 决定和身份边界
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
+1
View File
@@ -20,6 +20,7 @@ COMMANDS = {
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
+13 -122
View File
@@ -7,7 +7,6 @@ import argparse
from collections import Counter, defaultdict
from datetime import datetime
import hashlib
from html.parser import HTMLParser
import json
from pathlib import Path
import re
@@ -116,136 +115,29 @@ def load_registry(root, parent_path):
return evidence,rows
class _TitleFormatting(HTMLParser):
"""Decode text once, removing only balanced, attribute-free reviewed tags."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts=[];self.stack=[];self.valid=True
def handle_data(self, text):
self.parts.append(text)
def handle_starttag(self, tag, attrs):
if tag not in {'i','scp','sup','sub'} or attrs:
self.valid=False
else:
self.stack.append(tag)
def handle_endtag(self, tag):
if not self.stack or self.stack[-1]!=tag:
self.valid=False
else:
self.stack.pop()
def handle_startendtag(self, tag, attrs):
self.valid=False
def handle_comment(self, text):
self.valid=False
def handle_decl(self, text):
self.valid=False
def handle_pi(self, text):
self.valid=False
def unknown_decl(self, text):
self.valid=False
def formatting_text(title):
# HTMLParser may silently discard a trailing incomplete start tag.
# Check lexical coverage before parsing so no title content can disappear.
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
if '<' in literal or '>' in literal:
return None
parser=_TitleFormatting()
parser.feed(title);parser.close()
text=''.join(parser.parts)
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
'10.1145/3767308.3834936': 'mfp0229',
'10.1145/3767308.3834946': 'mfp0297',
'10.1145/3767308.3834956': 'mfp0339',
'10.1145/3767308.3835035': 'mfp0798',
'10.1145/3767308.3835055': 'mfp0903',
'10.1145/3767308.3835101': 'mfp1175',
'10.1145/3767308.3835374': 'mfp2617',
'10.1145/3767308.3835424': 'mfp2909',
'10.1145/3767308.3835452': 'mfp3048',
'10.1145/3767308.3835460': 'mfp3109',
'10.1145/3767308.3835544': 'mfp3554',
'10.1145/3767308.3835591': 'mfp3783',
'10.1145/3767308.3835667': 'mfp4173',
'10.1145/3767308.3835763': 'mfp4778',
'10.1145/3767308.3835895': 'mfp5537',
'10.1145/3767308.3835959': 'mfp5823',
'10.1145/3767308.3836215': 'mfp7332',
'10.1145/3767308.3836272': 'mfp7635',
'10.1145/3767308.3836293': 'mfp7732',
'10.1145/3767308.3836447': 'mfp8648',
'10.1145/3767308.3836600': 'mfp9815',
'10.1145/3767308.3838656': 'mds177',
'10.1145/3767308.3839012': 'mde41',
'10.1145/3767308.3839024': 'mde15',
'10.1145/3767308.3839028': 'mde02'}
def match_program(rows, observations):
titles=defaultdict(list);formatted=defaultdict(list)
for p in observations:
titles[key(p['title'])].append(p)
text=formatting_text(p['title'])
if text is not None:formatted[key(text)].append(p)
def correlate(rows, observations):
titles=defaultdict(list)
for row in observations:titles[key(row['title'])].append(row)
matched=set();kinds=Counter()
for row in rows:
row.pop('program',None);row.pop('formatting_comparison',None)
candidates=titles[key(row['title'])]
method='literal_title'
# A literal-title author disagreement must not be bypassed by fallback.
if not candidates:
text=formatting_text(row['title'])
if text is not None:
candidates=formatted[key(text)]
if candidates:method='reviewed_title_formatting'
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
if len(exact)==1:
row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
row['program']=exact[0];matched.add(exact[0]['paper_id'])
if exact[0]['track'] not in TRACKS:
raise ValueError('New registered program track requires review')
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
elif candidates:
row['kind']=('program_format_author_version_requires_review'
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
elif candidates:row['kind']='program_author_version_requires_review'
else:row['kind']='no_exact_program_title_requires_review'
row['program_title_candidates']=[p['paper_id'] for p in candidates]
row['program_exact_candidates']=[p['paper_id'] for p in exact]
if method=='reviewed_title_formatting':
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
'registry_comparison_text':formatting_text(row['title']),
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
'raw_titles_preserved':True,'work_identity_approved':False}
kinds[row['kind']]+=1
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
def correlate(rows, observations):
kinds,unmatched=match_program(rows,observations)
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
'program_author_version_requires_review':122,
'program_format_author_version_requires_review':1,
'no_exact_program_title_requires_review':43}
actual={r['doi']:r['program']['paper_id'] for r in rows
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
if kinds != expected or actual != FORMATTING_MATCHES:
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
if kinds != expected:
raise ValueError('Reviewed program/publication correspondence changed')
return kinds,unmatched
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
@@ -278,7 +170,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'venue':'MM','year':2026,'track':track,'status':'accepted',
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
@@ -288,7 +180,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'program_fields':p,'program_match_kind':item['kind'],
'program_title_candidates':item['program_title_candidates'],
'program_exact_candidates':item['program_exact_candidates'],
'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
'registry_abstract_available':False,'original_decision_signature_verified':False,
'scientific_work_identity_approved':False,'research_type_classified':False,
'review_flags':item['flags'],
@@ -297,7 +189,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
for track,rows in sorted(buckets.items()):
path=out/f'mm-publications-2026-{track}.jsonl'
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
@@ -307,13 +199,12 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
'public_negative_availability':'unknown_no_negative_decisions_inferred',
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
raise ValueError('Reviewed output partition or abstract census changed')
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
return report
+216
View File
@@ -0,0 +1,216 @@
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
from importlib.resources import files
import json
from pathlib import Path
import re
from paper_atlas.io.parquet_metadata import RangeSource, projection
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
SOURCE = 'reviewarena_mirror'
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
'title': 'string', 'abstract': 'string', 'authors': 'strings',
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
def inventory(split):
if split not in SPLITS:
raise ValueError('Explicit supported mirror split required')
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
raise ValueError('Pinned mirror file inventory changed')
items = json.loads(raw)
if len(items) != 15:
raise ValueError('Pinned mirror shard inventory changed')
selected = []
for item in items:
path = item['path']
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
raise ValueError('Unexpected mirror shard path')
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
raise ValueError('Missing source object size/checksum claim')
if path.startswith('data/'+split+'-'):
selected.append(item)
parts = sorted(selected, key=lambda x: x['path'])
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
raise ValueError('Incomplete selected mirror split inventory')
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
raise ValueError('Mirror shard sequence changed')
return parts
def decision_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'reject', 'rejected'}:
return 'rejected'
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
return 'desk_rejected'
if value in {'withdraw', 'withdrawn'}:
return 'withdrawn'
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
return 'accepted'
return 'unknown'
def decision_track_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'accept-findings', 'accept (findings)'}:
return 'findings'
if value in {'accept-main', 'accept (main)'}:
return 'main'
return None
def record(row, split, item, row_index):
if set(row) != set(COLUMNS):
raise ValueError('Unexpected projected metadata fields')
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
raise ValueError('Mirror conference/year outside pinned source scope')
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
'mirror_decision_category': decision_claim(row['decision']),
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
'source_locator': locator, 'identifiers': {},
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
'scientific_work_identity_approved': False, 'research_type_classified': False,
'approved_for_import': False,
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
def cached_projections(path, expected_sha256, split, items):
path = Path(path)
raw = path.read_bytes()
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
or hashlib.sha256(raw).hexdigest() != expected_sha256):
raise ValueError('Cached projection report hash differs')
report = json.loads(raw)
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
or report.get('approved_for_import') != 0):
raise ValueError('Cached report scope differs from permitted mirror projection')
declared = report.get('files', [])
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
raise ValueError('Incomplete cached mirror shard set')
result = {}
for item in items:
fm = next((f for f in declared if f['source_file'] == item['path']), None)
filename = Path(item['path']).stem+'.metadata.jsonl'
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
raise ValueError('Cached mirror source differs')
file = path.parent/filename
if file.is_symlink():
raise ValueError('Symlink metadata cache forbidden')
content = file.read_bytes()
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
raise ValueError('Cached metadata file hash differs')
rows = []
for index, line in enumerate(content.splitlines()):
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
if previous.get(key) != expected[key]:
raise ValueError('Cached raw metadata/identity differs: '+key)
rows.append(row)
proof = fm['projection']
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
or proof['full_file_checksum_verified'] is not False):
raise ValueError('Cached projection proof differs')
result[item['path']] = (rows, proof)
if sum(len(rows) for rows, _ in result.values()) != report['records']:
raise ValueError('Cached projection row count differs')
return result
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
items = inventory(split); out = Path(out)
if bool(from_projection) != bool(projection_sha256):
raise ValueError('Cached projection and exact report hash required together')
if out.exists() or not out.parent.is_dir() or out.is_symlink():
raise ValueError('New output directory in existing parent required')
out.mkdir()
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
observed = datetime.now(timezone.utc).isoformat()
try:
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
for item in items:
if cached is None:
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
source = source_factory(source_url, item['size'])
rows, proof = projection(source, COLUMNS, progress=progress)
else:
rows, proof = cached[item['path']]
if progress:
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
path = out/(Path(item['path']).stem+'.metadata.jsonl')
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
with part.open('wb') as stream:
for index, row in enumerate(rows):
item_record = record(row, split, item, index)
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
stream.write(raw); h.update(raw); records += 1
counts[item_record['mirror_decision_category']] += 1
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
years[(row['year'], row['track'])] += 1
part.rename(path)
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
'projection': proof})
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
'mirror_decision_track_claim_counts': dict(track_claims),
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
'cached_projection_report_sha256': projection_sha256,
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
'approved_for_import': 0,
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
'Every selected row and row locator is retained, including missing or anonymous author data.',
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
'Rows are not independent scientific works or a complete venue/year public negative census.',
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except BaseException as error:
# No source-specific body or signed redirect URLs in failure reports.
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
'completed_files': reports, 'error_type': type(error).__name__,
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--split', required=True, choices=SPLITS)
parser.add_argument('--out', required=True)
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
args = parser.parse_args(argv)
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
progress=lambda row: print(json.dumps(row), flush=True))
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
+195
View File
@@ -0,0 +1,195 @@
"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()
@@ -0,0 +1 @@
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
+2 -1
View File
@@ -12,6 +12,7 @@ dependencies = ["PyYAML==6.0.3"]
[project.optional-dependencies]
postgres = ["psycopg[binary]==3.3.8"]
mirrors = ["pyarrow==26.0.1"]
[project.scripts]
paper-atlas = "paper_atlas.cli:main"
@@ -22,4 +23,4 @@ exclude = ["tests*"]
[tool.setuptools.package-data]
"paper_atlas.db" = ["schema.sql", "annotations.sql"]
"paper_atlas.resources" = ["venues.json"]
"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"]
-706
View File
@@ -1,706 +0,0 @@
{
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
"gold": [
{
"doi": "10.1145/3767308.3834907",
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
"authors": [
"Jia Wang",
"Jiaming Cai",
"Zunying Hu",
"Zhanjie Wu",
"Jinyuan Liu",
"Hua Cheng",
"Yun Peng",
"Xin Fan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 246,
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
"program_id": "mfp0053",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
"program_context_pointers": [
"/entries/129/papers/19",
"/entries/129/subsessions/1/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834936",
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
"authors": [
"Jiacheng Lu",
"Weijian Wang",
"Mingyuan Xiao",
"Yang Hua",
"Tao Song",
"Bo Peng",
"Cheng Hua",
"Haibing Guan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 25,
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
"program_id": "mfp0229",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
"program_context_pointers": [
"/entries/109/papers/27",
"/entries/109/subsessions/2/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834946",
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 35,
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
"program_id": "mfp0297",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
"program_context_pointers": [
"/entries/61/papers/40",
"/entries/61/subsessions/3/items/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834956",
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Baojin Huang",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 235,
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
"program_id": "mfp0339",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
"program_context_pointers": [
"/entries/88/papers/48",
"/entries/88/subsessions/4/items/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835035",
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
"authors": [
"Haoqian Kang",
"Liupeng Li",
"Kuofeng Gao",
"Jinpeng Wang",
"Zhenyu Lu",
"Bin Chen",
"Ke Chen",
"Yaowei Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 8,
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
"program_id": "mfp0798",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
"program_context_pointers": [
"/entries/88/papers/39",
"/entries/88/subsessions/3/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835055",
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
"authors": [
"Qiang Qi",
"Wenqi Shang",
"Xiao Wang",
"Shuyuan Lin"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 28,
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
"program_id": "mfp0903",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
"program_context_pointers": [
"/entries/129/papers/55",
"/entries/129/subsessions/4/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835101",
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
"authors": [
"Zifan Zheng",
"Yu Luo",
"Yuxin Huang",
"Chuancheng Shi",
"Mohammad Masbaul Alam Polash",
"Zhiyong Wang",
"Kun Hu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 82,
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
"program_id": "mfp1175",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
"program_context_pointers": [
"/entries/109/papers/86",
"/entries/109/subsessions/7/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835374",
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
"authors": [
"Qianpeng Chong",
"Jiakun Wen",
"Guangyi Wei",
"Wenyi Zeng"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 59,
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
"program_id": "mfp2617",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
"program_context_pointers": [
"/entries/61/papers/80",
"/entries/61/subsessions/6/items/8"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835424",
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
"authors": [
"Evan Ram",
"Tuan Tran",
"Youngwook Son",
"Sangtae Ha"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 108,
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
"program_id": "mfp2909",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
"program_context_pointers": [
"/entries/129/papers/130",
"/entries/129/subsessions/8/items/22"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835452",
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
"authors": [
"He Peize",
"Yaodi Luo",
"Xiaoqian Liu",
"Xuyang Liu",
"Jiahang Deng",
"Yaosong Du",
"Li Bangyu",
"Xiyan Gui",
"Yuxuan Chen",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 167,
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
"program_id": "mfp3048",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
"program_context_pointers": [
"/entries/46/papers/78",
"/entries/46/subsessions/6/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835460",
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
"authors": [
"Chi Li",
"Rui Lin",
"Aobo Ji",
"Dongzhu Xu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 175,
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
"program_id": "mfp3109",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
"program_context_pointers": [
"/entries/109/papers/121",
"/entries/109/subsessions/8/items/13"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835544",
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
"authors": [
"Boyan Li",
"Ruijun Mao",
"Zhicheng Liang",
"Dayou Zhang",
"Fangxin Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 129,
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
"program_id": "mfp3554",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
"program_context_pointers": [
"/entries/46/papers/153",
"/entries/46/subsessions/9/items/21"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835591",
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
"authors": [
"Jiafeng Huang",
"Wenzhen Jia",
"Shilong Dong",
"Shengjie Zhao"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 97,
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
"program_id": "mfp3783",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
"program_context_pointers": [
"/entries/33/papers/158",
"/entries/33/subsessions/10/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835667",
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
"authors": [
"Yongqi Shan",
"Yifan Wang",
"Yunzhi Zhuge",
"Lijun Wang",
"Huchuan Lu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 209,
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
"program_id": "mfp4173",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
"program_context_pointers": [
"/entries/88/papers/114",
"/entries/88/subsessions/8/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835763",
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
"authors": [
"Yuxuan Wang",
"He Peize",
"Xiyan Gui",
"Xiaoqian Liu",
"Junhao He",
"Xuyang Liu",
"Xuming Hu",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 192,
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
"program_id": "mfp4778",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
"program_context_pointers": [
"/entries/129/papers/100",
"/entries/129/subsessions/7/items/16"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835895",
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
"authors": [
"Zan Gao",
"Xin Duan",
"Nian Hu",
"Hao Jia",
"Yibo Zhao",
"Cong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-001.json",
"source_position": 39,
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
"program_id": "mfp5537",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
"program_context_pointers": [
"/entries/88/papers/184",
"/entries/88/subsessions/11/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835959",
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
"authors": [
"Weiliang Huang",
"Weichao Cai",
"Bob Zhang",
"Chao Huang",
"Biao Xue",
"Rongxin Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 21,
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
"program_id": "mfp5823",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
"program_context_pointers": [
"/entries/88/papers/66",
"/entries/88/subsessions/5/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836215",
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
"authors": [
"Wentao Tan",
"Bowen Wang",
"Heng Zhi",
"Chenyv Liu",
"Zhe Li",
"Jian Liu",
"Zengrong Lin",
"Yukun Dai",
"Yipeng Chen",
"Wenjie Yang",
"Enci Xie",
"Hao Xue",
"Baixu Ji",
"Chen Xu",
"Zhibin Wang",
"Tianshi Wang",
"Lei Zhu",
"Heng Tao Shen"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 46,
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
"program_id": "mfp7332",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
"program_context_pointers": [
"/entries/33/papers/13",
"/entries/33/subsessions/1/items/1",
"/entries/47/papers/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836272",
"title": "SEE: Structure-aware Exploring &amp; Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
"authors": [
"Zhuohang Fan",
"Beichen Zhang",
"Yuanfa Li",
"Changqiao Wu",
"Wei Liu",
"Jian Luan",
"Weigang Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 127,
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
"program_id": "mfp7635",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
"program_context_pointers": [
"/entries/88/papers/131",
"/entries/88/subsessions/8/items/23"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836293",
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
"authors": [
"Ruizhe Chen",
"Zhiting Fan",
"Yang Shi",
"Enxin Song",
"Wenhao Chai",
"Tongkun Guan",
"Songtao Jiang",
"Ruilin Luo",
"Yuanxing Zhang",
"Shuai Bai",
"Sibo Song",
"Zhibo Yang",
"Zuozhu Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 148,
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
"program_id": "mfp7732",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
"program_context_pointers": [
"/entries/46/papers/198",
"/entries/46/subsessions/11/items/20"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836447",
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
"authors": [
"Raghu Vamsi Chittersu",
"Yuvraj Singh Rathore",
"Pranav Adlinge",
"Eswar Sai Pashaula",
"Kunal Swami"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 183,
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
"program_id": "mfp8648",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
"program_context_pointers": [
"/entries/107/papers/4",
"/entries/117/papers/30",
"/entries/117/subsessions/2/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836600",
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
"authors": [
"Mansi Sharma",
"Dinar Orazgaliyev",
"Antonio Krüger",
"Philipp Müller"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 140,
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
"program_id": "mfp9815",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
"program_context_pointers": [
"/entries/46/papers/219",
"/entries/46/subsessions/12/items/17"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3838656",
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
"authors": [
"Xiaochen Wang",
"Yuan Zhong",
"Lingwei Zhang",
"Lisong Dai",
"Ting Wang",
"Fenglong Ma"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 225,
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
"program_id": "mds177",
"track_literal": "Datasets",
"track": "datasets",
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
"program_context_pointers": [
"/entries/46/papers/23",
"/entries/46/subsessions/1/items/11"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839012",
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
"authors": [
"Abhijeet Narang",
"Hagar Elbatanouny",
"Vanishri Shekhawat",
"Hasan Al-Nashash",
"Abhinav Dhall",
"Usman Tariq"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 238,
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
"program_id": "mde41",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
"program_context_pointers": [
"/entries/94/papers/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839024",
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
"authors": [
"Jiangfan Liu",
"Zexuan Cui",
"Tianyuan Zhang",
"Zonglei Jing",
"Yaoyuan Zhang",
"Jiakai Wang",
"Jinyang Guo",
"Yuqing Ma",
"Lu Sheng",
"Aishan Liu",
"Xianglong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 0,
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
"program_id": "mde15",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
"program_context_pointers": [
"/entries/49/papers/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839028",
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
"authors": [
"Xiaoyan Yu",
"Ziya Zhao",
"Yunshan Ma",
"Yifan Wei",
"Dianbo Sui",
"Tat-Seng Chua"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 4,
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
"program_id": "mde02",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
"program_context_pointers": [
"/entries/49/papers/0"
],
"single_name_positions": []
}
],
"author_disagreement_hold": {
"doi": "10.1145/3767308.3836308",
"program_id": "mfp7801"
},
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
"held_gold": {
"doi": "10.1145/3767308.3836308",
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
"authors": [
"Zhirong Tian",
"Yi Zhang",
"Weixuan Liang",
"Miaomiao Li",
"Xinwang Liu"
],
"kind": "program_format_author_version_requires_review",
"source_page": "page-001.json",
"source_position": 82,
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
"program_id": null,
"track_literal": null,
"track": "proceedings_unclassified",
"abstract_sha256": null,
"program_context_pointers": [],
"single_name_positions": []
}
}
+12 -67
View File
@@ -19,9 +19,6 @@ class MMPublications2026Tests(unittest.TestCase):
fixtures=Path(__file__).parent/'fixtures'
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
cls.materialize(cls.root/'registry',cls.fixture['files'])
cls.materialize(cls.root/'program',cls.program_fixture['files'])
@@ -41,8 +38,8 @@ class MMPublications2026Tests(unittest.TestCase):
def test_all1918_publications_against_independent_registry_program_gold(self):
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
for g in self.gold:
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
self.assertEqual(m['program_match_kind'],g['kind'])
@@ -54,22 +51,22 @@ class MMPublications2026Tests(unittest.TestCase):
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
count=0
for g in self.gold:
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];m=r['metadata']
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
if g['abstract_sha256'] is not None:
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
else:
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
for p in self.report['unmatched_program']:
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
gold=self.program_fixture['independent_inventory']['inventory']
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
for g in gold:
@@ -94,7 +91,7 @@ class MMPublications2026Tests(unittest.TestCase):
for track in ('interactive_art','industry_talk','demo_and_video'):
self.assertTrue(any(r['track']==track for r in self.rows))
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
self.assertEqual(len(variants),166)
self.assertEqual(len(variants),192)
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
@@ -116,7 +113,7 @@ class MMPublications2026Tests(unittest.TestCase):
def test_eight_single_name_fields_preserved_without_splitting(self):
n=0
for g in self.gold:
for g in self.fixture['independent_gold']:
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
@@ -126,62 +123,13 @@ class MMPublications2026Tests(unittest.TestCase):
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
count=0
for g in self.gold:
for g in self.fixture['independent_gold']:
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
count+=1;r=self.by_doi[g['doi']]
self.assertEqual(r['title'],g['title'])
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
self.assertEqual(len(self.format_gold['gold']),26)
for gold in self.format_gold['gold']:
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
comparison=m['program_title_formatting_comparison']
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
self.assertFalse(m['scientific_work_identity_approved'])
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
self.assertEqual(held['metadata']['program_exact_candidates'],[])
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> &amp; C'),'A H2 B1 & C')
self.assertEqual(c.formatting_text('A &amp;amp; B'),'A &amp; B')
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A &lt;i&gt;B&lt;/i&gt;',
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
def observation(pid,authors):
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
self.assertNotIn('program',rows[0])
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
c.match_program(rows,[observation('p1',['A','B'])])
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
self.assertNotIn('program',rows[0])
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
c.match_program(rows,observations)
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
for kind in ('registry_manifest','registry_page','parent','program'):
with tempfile.TemporaryDirectory() as tmp:
@@ -202,12 +150,9 @@ class MMPublications2026Tests(unittest.TestCase):
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
self.assertEqual(self.report['unmatched_program_count'],391)
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
@@ -251,7 +196,7 @@ class MMPublications2026Tests(unittest.TestCase):
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726)
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
+203
View File
@@ -0,0 +1,203 @@
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
from collections import Counter
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import re
import struct
import tempfile
import unittest
from unittest.mock import patch
import urllib.error
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
class MemorySource:
def __init__(self, raw):
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
def read(self, start, length):
raw = self.raw[start:start+length]
self.bytes_read += len(raw)
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
return raw
def fixture():
import pyarrow as pa
import pyarrow.parquet as pq
rows = []
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
'track': 'main', 'venue_id': 'source-venue-claim',
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
rows.append(row)
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
class RangeHTTPTests(unittest.TestCase):
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
response = io.BytesIO(data); response.status = status
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
return response
def test_strict_range_and_changed_object_detection(self):
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
source = RangeSource('https://example.org/public', 10, opener=opener)
self.assertEqual(source.read(2, 2), b'ab')
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
with self.assertRaisesRegex(ValueError, 'ETag'):
source.read(4, 2)
self.assertEqual(source.bytes_read, 2)
def test_no_full_file_fallback_or_denied_source_retry(self):
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
opener.open.return_value = response
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
response.read.assert_not_called()
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
self.assertEqual(opener.open.call_count, 1)
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
with self.assertRaises(ValueError):
source.seek(-101, 2)
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
class ParquetProjectionTests(unittest.TestCase):
def test_complete_selected_rows_and_no_excluded_page_requests(self):
import pyarrow.parquet as pq
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
self.assertEqual(rows, expected)
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
excluded = []
for i in range(metadata.num_row_groups):
group = metadata.row_group(i)
for j in range(group.num_columns):
col = group.column(j)
if col.path_in_schema.split('.')[0] not in COLUMNS:
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
excluded.append((start, start+col.total_compressed_size))
for request in source.receipts:
a, b = request['start'], request['start']+request['length']
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
self.assertLess(source.bytes_read, len(raw))
encoded = json.dumps(rows)+json.dumps(proof)
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
raw, _ = fixture(); source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'Missing'):
projection(source, {'missing_column': 'string'})
self.assertEqual(len(source.receipts), 3)
source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'type differs'):
projection(source, {**COLUMNS, 'year': 'string'})
self.assertEqual(len(source.receipts), 3)
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
source = MemorySource(bytes(bad))
with self.assertRaisesRegex(ValueError, 'footer'):
projection(source, COLUMNS)
self.assertEqual(len(source.receipts), 2)
def test_collector_output_preserves_claims_without_original_identity(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'projected'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
for row in projected:
self.assertEqual(row['identifiers'], {})
self.assertFalse(row['metadata']['original_forum_id_verified'])
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertNotIn('status', row)
self.assertIn(COMMIT, row['external_id'])
self.assertEqual(projected[2]['authors'], ['Anonymous'])
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
self.assertEqual(projected[0]['track'], 'main')
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
all_saved=''.join(p.read_text() for p in out.iterdir())
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
with self.assertRaisesRegex(ValueError, 'New output'):
collect('emnlp', out)
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
# Mimic the first reviewed source version: new literal spellings were unknown.
for row in before[:2]:
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
with self.assertRaisesRegex(ValueError, 'hash differs'):
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
no_network.assert_not_called()
def test_failure_is_retained_without_source_payload_or_retries(self):
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'failed'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
with self.assertRaises(RuntimeError):
collect('emnlp', out, source_factory=lambda url, size: None)
report = json.loads((out/'failure.json').read_text())
self.assertEqual(report['status'], 'incomplete_do_not_import')
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
class MirrorSemanticsTests(unittest.TestCase):
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
result = {s: inventory(s) for s in splits}
self.assertEqual(sum(len(v) for v in result.values()), 15)
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
self.assertEqual(result['emnlp'][0]['size'], 90794691)
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
with self.assertRaises(ValueError):
inventory('uai')
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
self.assertEqual(decision_claim(literal), 'unknown')
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
self.assertIsNone(decision_track_claim('Accept (Poster)'))