Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 12f95cff91 | |||
| 9864632774 |
@@ -35,7 +35,7 @@ jobs:
|
||||
venv = root / 'venv'
|
||||
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
||||
python = str(venv / 'bin/python')
|
||||
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True)
|
||||
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
|
||||
testenv = os.environ.copy()
|
||||
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
|
||||
if service_file.is_file():
|
||||
@@ -75,11 +75,15 @@ jobs:
|
||||
path.write_bytes(body.encode('utf-8'))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
|
||||
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
|
||||
assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||
assert len(mm_pub_rows)==1918
|
||||
for gold in mm_pub_fixture['independent_gold']:
|
||||
mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
|
||||
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
|
||||
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
|
||||
for original_gold in mm_pub_fixture['independent_gold']:
|
||||
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
|
||||
row=mm_pub_rows[gold['doi']]
|
||||
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
name: ReviewArena metadata projection
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
split:
|
||||
description: 'Pinned public mirror family; metadata columns only'
|
||||
required: true
|
||||
default: emnlp
|
||||
projection_name:
|
||||
description: 'Optional existing metadata output directory name for offline replay'
|
||||
required: false
|
||||
default: ''
|
||||
projection_sha256:
|
||||
description: 'Exact cached collection-report.json SHA256, required for offline replay'
|
||||
required: false
|
||||
default: ''
|
||||
jobs:
|
||||
project:
|
||||
runs-on: ces-control
|
||||
env:
|
||||
ARCHIVE_SHA: ${{ github.sha }}
|
||||
MIRROR_SPLIT: ${{ inputs.split }}
|
||||
MIRROR_PROJECTION: ${{ inputs.projection_name }}
|
||||
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
|
||||
PYTHONDONTWRITEBYTECODE: '1'
|
||||
steps:
|
||||
- name: Project permitted metadata on the approved remote runner
|
||||
run: |
|
||||
python3 -B - <<'PY'
|
||||
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
sha = os.environ['ARCHIVE_SHA']
|
||||
split = os.environ['MIRROR_SPLIT']
|
||||
cached = os.environ.get('MIRROR_PROJECTION','')
|
||||
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
|
||||
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
|
||||
raise SystemExit('Explicit approved source commit/split required')
|
||||
base = pathlib.Path('/data/paper-index')
|
||||
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
|
||||
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
|
||||
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
|
||||
if out.exists():
|
||||
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
|
||||
extra=[]
|
||||
if cached:
|
||||
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
|
||||
raise SystemExit('Explicit permitted cached projection name and report hash required')
|
||||
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
|
||||
elif cached_sha:
|
||||
raise SystemExit('Cached projection name required')
|
||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
|
||||
with urllib.request.urlopen(url,timeout=60) as response:
|
||||
archive = response.read(24*1024*1024+1)
|
||||
if len(archive)>24*1024*1024:
|
||||
raise SystemExit('Oversized source archive')
|
||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
|
||||
tar.extractall(root,filter='data')
|
||||
candidates=list(root.glob('*/pyproject.toml'))
|
||||
if len(candidates)!=1:
|
||||
raise SystemExit('Unexpected repository archive')
|
||||
src=candidates[0].parent
|
||||
venv=root/'venv'
|
||||
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
||||
python=str(venv/'bin/python')
|
||||
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
|
||||
wheels=root/'wheels'
|
||||
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
|
||||
wheel,=wheels.glob('paper_atlas-*.whl')
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside=root/'outside';outside.mkdir()
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
|
||||
report=json.loads((out/'collection-report.json').read_bytes())
|
||||
assert report['database_written'] is False and report['approved_for_import']==0
|
||||
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
|
||||
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
|
||||
backup=base/'validation'/(out.name+'-recovery.tar.gz')
|
||||
if backup.exists():
|
||||
raise SystemExit('Recovery archive already exists; inspect')
|
||||
with tarfile.open(backup,'w:gz') as tar:
|
||||
tar.add(out,arcname=str(out.relative_to(base)))
|
||||
backup.chmod(0o444)
|
||||
for path in out.iterdir():
|
||||
if path.is_file():path.chmod(0o444)
|
||||
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
|
||||
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
|
||||
print(json.dumps(result),flush=True)
|
||||
PY
|
||||
@@ -2,8 +2,6 @@
|
||||
|
||||
**长期维护、按需运行的论文元数据与公开投稿历史工具。**
|
||||
|
||||
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
|
||||
|
||||
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
|
||||
|
||||
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
||||
@@ -145,4 +143,4 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,717 份节目摘要明确标注来源,192 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
+16
-10
@@ -11,7 +11,7 @@ paper-atlas collect mm2026-publications \
|
||||
--registry-snapshots /data/paper-index/collected \
|
||||
--program-snapshots /data/paper-index/collected \
|
||||
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||
--out /data/paper-index/collected/mm2026-publications-native-v1
|
||||
--out /data/paper-index/collected/mm2026-publications-native-v3
|
||||
```
|
||||
|
||||
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||
@@ -28,27 +28,31 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
|
||||
|
||||
## 分轨、摘要与版本差异
|
||||
|
||||
按原始题名的Unicode/字母数字规范化与完整作者多重集合比对,1726项有唯一节目对应。源track映射到对应输出分区,并保留原始标签、节目ID、机构和所有展示上下文;不生成全局人物别名。其余122项作者版本差异、70项题名版本差异保留在 `proceedings_unclassified`,不强配分轨或静默排除。
|
||||
按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
|
||||
|
||||
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
|
||||
|
||||
| 输出分区 | 注册条目 |
|
||||
| --- | ---: |
|
||||
| main | 1533 |
|
||||
| main | 1555 |
|
||||
| grand_challenge | 77 |
|
||||
| interactive_art | 17 |
|
||||
| datasets | 40 |
|
||||
| datasets | 41 |
|
||||
| reproducibility | 1 |
|
||||
| bni | 22 |
|
||||
| open_source | 11 |
|
||||
| doctoral_symposium | 2 |
|
||||
| demo_and_video | 20 |
|
||||
| demo_and_video | 23 |
|
||||
| industry_talk | 3 |
|
||||
| proceedings_unclassified | 192 |
|
||||
| proceedings_unclassified | 166 |
|
||||
|
||||
注册摘要1918项全缺失。1717项有明确对应的节目摘要,逐字保留并标明 `abstract_source=official_program_exact_metadata_correspondence`;注册原始摘要仍标为缺失,绝不伪称由出版者提供。其余201项保持缺失,包括9项已对应节目本身没有摘要的条目及192个版本待核条目。没有用模糊匹配把待核摘要强加给论文。
|
||||
注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
|
||||
|
||||
8个作者仅有单一姓名字段,保留登记值并标记,不猜测given/family;37个含排版标记或实体的题名保留字面原文并标记,不能损失数学上下标或假定排版版本已经解决。
|
||||
其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
|
||||
|
||||
完整2143个节目ID和4326个展示位置仍在匹配证据或417个未匹配节目中,28个原始track均保留。417个节目与192个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||
8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
|
||||
|
||||
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||
|
||||
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||
|
||||
@@ -64,4 +68,6 @@ ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `
|
||||
|
||||
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||
|
||||
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1726个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
|
||||
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
|
||||
|
||||
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
# ReviewArena 镜像的索引字段投影
|
||||
|
||||
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
|
||||
|
||||
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
|
||||
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
|
||||
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
|
||||
|
||||
## 运行
|
||||
|
||||
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
|
||||
|
||||
包内命令如下,`--out` 必须是既有父目录内的新目录:
|
||||
|
||||
```sh
|
||||
paper-atlas collect reviewarena-metadata \
|
||||
--split emnlp \
|
||||
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
|
||||
```
|
||||
|
||||
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
|
||||
|
||||
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
|
||||
|
||||
```sh
|
||||
paper-atlas collect reviewarena-metadata --split emnlp \
|
||||
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
|
||||
--projection-sha256 <已验证的原报告SHA256> \
|
||||
--out /data/paper-index/collected/<新解释版本>
|
||||
```
|
||||
|
||||
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
|
||||
|
||||
## 实际读取和保存范围
|
||||
|
||||
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
|
||||
|
||||
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
|
||||
|
||||
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
|
||||
|
||||
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
|
||||
|
||||
## 决定和身份边界
|
||||
|
||||
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
|
||||
|
||||
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
|
||||
|
||||
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
|
||||
|
||||
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
|
||||
@@ -20,7 +20,6 @@ COMMANDS = {
|
||||
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
|
||||
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
|
||||
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
|
||||
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
|
||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
|
||||
@@ -7,6 +7,7 @@ import argparse
|
||||
from collections import Counter, defaultdict
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
@@ -115,31 +116,138 @@ def load_registry(root, parent_path):
|
||||
return evidence,rows
|
||||
|
||||
|
||||
def correlate(rows, observations):
|
||||
titles=defaultdict(list)
|
||||
for row in observations:titles[key(row['title'])].append(row)
|
||||
|
||||
class _TitleFormatting(HTMLParser):
|
||||
"""Decode text once, removing only balanced, attribute-free reviewed tags."""
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.parts=[];self.stack=[];self.valid=True
|
||||
|
||||
def handle_data(self, text):
|
||||
self.parts.append(text)
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag not in {'i','scp','sup','sub'} or attrs:
|
||||
self.valid=False
|
||||
else:
|
||||
self.stack.append(tag)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if not self.stack or self.stack[-1]!=tag:
|
||||
self.valid=False
|
||||
else:
|
||||
self.stack.pop()
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
self.valid=False
|
||||
|
||||
def handle_comment(self, text):
|
||||
self.valid=False
|
||||
|
||||
def handle_decl(self, text):
|
||||
self.valid=False
|
||||
|
||||
def handle_pi(self, text):
|
||||
self.valid=False
|
||||
|
||||
def unknown_decl(self, text):
|
||||
self.valid=False
|
||||
|
||||
|
||||
def formatting_text(title):
|
||||
# HTMLParser may silently discard a trailing incomplete start tag.
|
||||
# Check lexical coverage before parsing so no title content can disappear.
|
||||
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
|
||||
if '<' in literal or '>' in literal:
|
||||
return None
|
||||
parser=_TitleFormatting()
|
||||
parser.feed(title);parser.close()
|
||||
text=''.join(parser.parts)
|
||||
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
|
||||
|
||||
|
||||
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
|
||||
'10.1145/3767308.3834936': 'mfp0229',
|
||||
'10.1145/3767308.3834946': 'mfp0297',
|
||||
'10.1145/3767308.3834956': 'mfp0339',
|
||||
'10.1145/3767308.3835035': 'mfp0798',
|
||||
'10.1145/3767308.3835055': 'mfp0903',
|
||||
'10.1145/3767308.3835101': 'mfp1175',
|
||||
'10.1145/3767308.3835374': 'mfp2617',
|
||||
'10.1145/3767308.3835424': 'mfp2909',
|
||||
'10.1145/3767308.3835452': 'mfp3048',
|
||||
'10.1145/3767308.3835460': 'mfp3109',
|
||||
'10.1145/3767308.3835544': 'mfp3554',
|
||||
'10.1145/3767308.3835591': 'mfp3783',
|
||||
'10.1145/3767308.3835667': 'mfp4173',
|
||||
'10.1145/3767308.3835763': 'mfp4778',
|
||||
'10.1145/3767308.3835895': 'mfp5537',
|
||||
'10.1145/3767308.3835959': 'mfp5823',
|
||||
'10.1145/3767308.3836215': 'mfp7332',
|
||||
'10.1145/3767308.3836272': 'mfp7635',
|
||||
'10.1145/3767308.3836293': 'mfp7732',
|
||||
'10.1145/3767308.3836447': 'mfp8648',
|
||||
'10.1145/3767308.3836600': 'mfp9815',
|
||||
'10.1145/3767308.3838656': 'mds177',
|
||||
'10.1145/3767308.3839012': 'mde41',
|
||||
'10.1145/3767308.3839024': 'mde15',
|
||||
'10.1145/3767308.3839028': 'mde02'}
|
||||
|
||||
|
||||
def match_program(rows, observations):
|
||||
titles=defaultdict(list);formatted=defaultdict(list)
|
||||
for p in observations:
|
||||
titles[key(p['title'])].append(p)
|
||||
text=formatting_text(p['title'])
|
||||
if text is not None:formatted[key(text)].append(p)
|
||||
matched=set();kinds=Counter()
|
||||
for row in rows:
|
||||
row.pop('program',None);row.pop('formatting_comparison',None)
|
||||
candidates=titles[key(row['title'])]
|
||||
method='literal_title'
|
||||
# A literal-title author disagreement must not be bypassed by fallback.
|
||||
if not candidates:
|
||||
text=formatting_text(row['title'])
|
||||
if text is not None:
|
||||
candidates=formatted[key(text)]
|
||||
if candidates:method='reviewed_title_formatting'
|
||||
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||
if len(exact)==1:
|
||||
row['kind']='unique_program_title_fullauthor_agreement_not_identity_approval'
|
||||
row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
|
||||
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
|
||||
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||
if exact[0]['track'] not in TRACKS:
|
||||
raise ValueError('New registered program track requires review')
|
||||
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||
elif candidates:row['kind']='program_author_version_requires_review'
|
||||
elif candidates:
|
||||
row['kind']=('program_format_author_version_requires_review'
|
||||
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
|
||||
else:row['kind']='no_exact_program_title_requires_review'
|
||||
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||
if method=='reviewed_title_formatting':
|
||||
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
|
||||
'registry_comparison_text':formatting_text(row['title']),
|
||||
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
|
||||
'raw_titles_preserved':True,'work_identity_approved':False}
|
||||
kinds[row['kind']]+=1
|
||||
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||
'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70}
|
||||
if kinds != expected:
|
||||
raise ValueError('Reviewed program/publication correspondence changed')
|
||||
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||
|
||||
|
||||
def correlate(rows, observations):
|
||||
kinds,unmatched=match_program(rows,observations)
|
||||
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
|
||||
'program_author_version_requires_review':122,
|
||||
'program_format_author_version_requires_review':1,
|
||||
'no_exact_program_title_requires_review':43}
|
||||
actual={r['doi']:r['program']['paper_id'] for r in rows
|
||||
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
|
||||
if kinds != expected or actual != FORMATTING_MATCHES:
|
||||
raise ValueError('Reviewed program/publication correspondence changed')
|
||||
return kinds,unmatched
|
||||
|
||||
|
||||
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
out=Path(out)
|
||||
if out.exists() or out.is_symlink():
|
||||
@@ -170,7 +278,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||
'metadata':{'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||
'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||
@@ -180,7 +288,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
'program_fields':p,'program_match_kind':item['kind'],
|
||||
'program_title_candidates':item['program_title_candidates'],
|
||||
'program_exact_candidates':item['program_exact_candidates'],
|
||||
'abstract_source':'official_program_exact_metadata_correspondence' if abstract is not None else 'not_available_from_reviewed_sources',
|
||||
'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
|
||||
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||
'review_flags':item['flags'],
|
||||
@@ -189,7 +297,7 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
for track,rows in sorted(buckets.items()):
|
||||
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||
manifest={'source':SOURCE,'parser_version':1,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||
manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||
@@ -199,12 +307,13 @@ def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
|
||||
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.192title/authorvariants remainunclassified,417unmatchedprogram items retained. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||
if (sum(f['count'] for f in outputs)!=1918 or copied!=1717 or missing!=201 or len(unmatched)!=417):
|
||||
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||
if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
|
||||
raise ValueError('Reviewed output partition or abstract census changed')
|
||||
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||
return report
|
||||
|
||||
@@ -1,216 +0,0 @@
|
||||
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
from importlib.resources import files
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.parquet_metadata import RangeSource, projection
|
||||
|
||||
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
|
||||
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
|
||||
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
|
||||
SOURCE = 'reviewarena_mirror'
|
||||
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
|
||||
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
|
||||
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
|
||||
'title': 'string', 'abstract': 'string', 'authors': 'strings',
|
||||
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
|
||||
|
||||
|
||||
def inventory(split):
|
||||
if split not in SPLITS:
|
||||
raise ValueError('Explicit supported mirror split required')
|
||||
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
|
||||
raise ValueError('Pinned mirror file inventory changed')
|
||||
items = json.loads(raw)
|
||||
if len(items) != 15:
|
||||
raise ValueError('Pinned mirror shard inventory changed')
|
||||
selected = []
|
||||
for item in items:
|
||||
path = item['path']
|
||||
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
|
||||
raise ValueError('Unexpected mirror shard path')
|
||||
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
|
||||
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
|
||||
raise ValueError('Missing source object size/checksum claim')
|
||||
if path.startswith('data/'+split+'-'):
|
||||
selected.append(item)
|
||||
parts = sorted(selected, key=lambda x: x['path'])
|
||||
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
|
||||
raise ValueError('Incomplete selected mirror split inventory')
|
||||
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
|
||||
raise ValueError('Mirror shard sequence changed')
|
||||
return parts
|
||||
|
||||
|
||||
def decision_claim(raw):
|
||||
value = raw.strip().casefold() if isinstance(raw, str) else ''
|
||||
if value in {'reject', 'rejected'}:
|
||||
return 'rejected'
|
||||
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
|
||||
return 'desk_rejected'
|
||||
if value in {'withdraw', 'withdrawn'}:
|
||||
return 'withdrawn'
|
||||
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
|
||||
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
|
||||
return 'accepted'
|
||||
return 'unknown'
|
||||
|
||||
|
||||
def decision_track_claim(raw):
|
||||
value = raw.strip().casefold() if isinstance(raw, str) else ''
|
||||
if value in {'accept-findings', 'accept (findings)'}:
|
||||
return 'findings'
|
||||
if value in {'accept-main', 'accept (main)'}:
|
||||
return 'main'
|
||||
return None
|
||||
|
||||
|
||||
def record(row, split, item, row_index):
|
||||
if set(row) != set(COLUMNS):
|
||||
raise ValueError('Unexpected projected metadata fields')
|
||||
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
|
||||
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
|
||||
raise ValueError('Mirror conference/year outside pinned source scope')
|
||||
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
|
||||
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
|
||||
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
|
||||
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
|
||||
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
|
||||
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
|
||||
'mirror_decision_category': decision_claim(row['decision']),
|
||||
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
|
||||
'source_locator': locator, 'identifiers': {},
|
||||
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
|
||||
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
|
||||
'scientific_work_identity_approved': False, 'research_type_classified': False,
|
||||
'approved_for_import': False,
|
||||
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
|
||||
|
||||
|
||||
def cached_projections(path, expected_sha256, split, items):
|
||||
path = Path(path)
|
||||
raw = path.read_bytes()
|
||||
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
|
||||
or hashlib.sha256(raw).hexdigest() != expected_sha256):
|
||||
raise ValueError('Cached projection report hash differs')
|
||||
report = json.loads(raw)
|
||||
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
|
||||
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
|
||||
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
|
||||
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
|
||||
or report.get('approved_for_import') != 0):
|
||||
raise ValueError('Cached report scope differs from permitted mirror projection')
|
||||
declared = report.get('files', [])
|
||||
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
|
||||
raise ValueError('Incomplete cached mirror shard set')
|
||||
result = {}
|
||||
for item in items:
|
||||
fm = next((f for f in declared if f['source_file'] == item['path']), None)
|
||||
filename = Path(item['path']).stem+'.metadata.jsonl'
|
||||
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
|
||||
raise ValueError('Cached mirror source differs')
|
||||
file = path.parent/filename
|
||||
if file.is_symlink():
|
||||
raise ValueError('Symlink metadata cache forbidden')
|
||||
content = file.read_bytes()
|
||||
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
|
||||
raise ValueError('Cached metadata file hash differs')
|
||||
rows = []
|
||||
for index, line in enumerate(content.splitlines()):
|
||||
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
|
||||
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
|
||||
if previous.get(key) != expected[key]:
|
||||
raise ValueError('Cached raw metadata/identity differs: '+key)
|
||||
rows.append(row)
|
||||
proof = fm['projection']
|
||||
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
|
||||
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
|
||||
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
|
||||
or proof['full_file_checksum_verified'] is not False):
|
||||
raise ValueError('Cached projection proof differs')
|
||||
result[item['path']] = (rows, proof)
|
||||
if sum(len(rows) for rows, _ in result.values()) != report['records']:
|
||||
raise ValueError('Cached projection row count differs')
|
||||
return result
|
||||
|
||||
|
||||
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
|
||||
items = inventory(split); out = Path(out)
|
||||
if bool(from_projection) != bool(projection_sha256):
|
||||
raise ValueError('Cached projection and exact report hash required together')
|
||||
if out.exists() or not out.parent.is_dir() or out.is_symlink():
|
||||
raise ValueError('New output directory in existing parent required')
|
||||
out.mkdir()
|
||||
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
|
||||
observed = datetime.now(timezone.utc).isoformat()
|
||||
try:
|
||||
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
|
||||
for item in items:
|
||||
if cached is None:
|
||||
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
|
||||
source = source_factory(source_url, item['size'])
|
||||
rows, proof = projection(source, COLUMNS, progress=progress)
|
||||
else:
|
||||
rows, proof = cached[item['path']]
|
||||
if progress:
|
||||
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
|
||||
path = out/(Path(item['path']).stem+'.metadata.jsonl')
|
||||
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
|
||||
with part.open('wb') as stream:
|
||||
for index, row in enumerate(rows):
|
||||
item_record = record(row, split, item, index)
|
||||
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
|
||||
stream.write(raw); h.update(raw); records += 1
|
||||
counts[item_record['mirror_decision_category']] += 1
|
||||
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
|
||||
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
|
||||
years[(row['year'], row['track'])] += 1
|
||||
part.rename(path)
|
||||
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
|
||||
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
|
||||
'projection': proof})
|
||||
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
|
||||
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
|
||||
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
|
||||
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
|
||||
'mirror_decision_track_claim_counts': dict(track_claims),
|
||||
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
|
||||
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
|
||||
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
|
||||
'cached_projection_report_sha256': projection_sha256,
|
||||
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
|
||||
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
|
||||
'approved_for_import': 0,
|
||||
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
|
||||
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
|
||||
'Every selected row and row locator is retained, including missing or anonymous author data.',
|
||||
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
|
||||
'Rows are not independent scientific works or a complete venue/year public negative census.',
|
||||
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
|
||||
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||
return report
|
||||
except BaseException as error:
|
||||
# No source-specific body or signed redirect URLs in failure reports.
|
||||
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
|
||||
'completed_files': reports, 'error_type': type(error).__name__,
|
||||
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
|
||||
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--split', required=True, choices=SPLITS)
|
||||
parser.add_argument('--out', required=True)
|
||||
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
|
||||
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
|
||||
args = parser.parse_args(argv)
|
||||
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
|
||||
progress=lambda row: print(json.dumps(row), flush=True))
|
||||
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
|
||||
@@ -1,195 +0,0 @@
|
||||
"""Read only selected Parquet column chunks; never fetch other payload pages."""
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
import struct
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
MAX_FOOTER = 4 * 1024**2
|
||||
MAX_SELECTED_BYTES = 64 * 1024**2
|
||||
MAX_REQUESTS = 512
|
||||
|
||||
|
||||
class RangeSource:
|
||||
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
|
||||
def __init__(self, url, size, *, opener=None):
|
||||
self.url, self.size = url, size
|
||||
self.opener = opener or urllib.request.build_opener()
|
||||
self.etag = None
|
||||
self.receipts = []
|
||||
self.bytes_read = 0
|
||||
|
||||
def read(self, start, length):
|
||||
if (type(start) is not int or type(length) is not int or length <= 0
|
||||
or start < 0 or start + length > self.size
|
||||
or self.bytes_read + length > MAX_SELECTED_BYTES
|
||||
or len(self.receipts) >= MAX_REQUESTS):
|
||||
raise ValueError('Bounded metadata range budget exceeded')
|
||||
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
|
||||
if self.etag and not self.etag.startswith('W/'):
|
||||
headers['If-Match'] = self.etag
|
||||
request = urllib.request.Request(self.url, headers=headers)
|
||||
try:
|
||||
with self.opener.open(request, timeout=45) as response:
|
||||
if response.status != 206:
|
||||
raise ValueError('Server did not honor metadata range; full download forbidden')
|
||||
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
|
||||
raise ValueError('Object size or range differs from pinned inventory')
|
||||
encoding = response.headers.get('Content-Encoding', 'identity')
|
||||
if encoding != 'identity':
|
||||
raise ValueError('Encoded HTTP ranges are unsupported')
|
||||
etag = response.headers.get('ETag')
|
||||
if self.receipts and etag != self.etag:
|
||||
raise ValueError('Object ETag changed between metadata ranges')
|
||||
raw = response.read(length + 1)
|
||||
if len(raw) != length:
|
||||
raise ValueError('Truncated or oversized metadata range')
|
||||
if not self.receipts:
|
||||
self.etag = etag
|
||||
except urllib.error.HTTPError as error:
|
||||
# Do not expose signed CDN redirect URLs or retry denied resources.
|
||||
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
|
||||
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
self.bytes_read += length
|
||||
return raw
|
||||
|
||||
|
||||
class SparseFile(io.RawIOBase):
|
||||
"""A logical file backed only by approved bytes; all other positions are zeros.
|
||||
|
||||
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
|
||||
fetching the preceding OCR/review page. Data pages are explicitly selected
|
||||
from footer offsets before they can be downloaded.
|
||||
"""
|
||||
def __init__(self, size, segments):
|
||||
self.size, self.segments, self.position = size, segments, 0
|
||||
|
||||
def readable(self):
|
||||
return True
|
||||
|
||||
def seekable(self):
|
||||
return True
|
||||
|
||||
def tell(self):
|
||||
return self.position
|
||||
|
||||
def seek(self, offset, whence=0):
|
||||
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
|
||||
if base is None or base + offset < 0:
|
||||
raise ValueError('Invalid sparse metadata seek')
|
||||
self.position = base + offset
|
||||
return self.position
|
||||
|
||||
def read(self, length=-1):
|
||||
if length < 0:
|
||||
length = self.size - self.position
|
||||
length = min(length, max(0, self.size - self.position))
|
||||
if length > MAX_SELECTED_BYTES:
|
||||
raise ValueError('Unbounded sparse read forbidden')
|
||||
start, end = self.position, self.position + length
|
||||
raw = bytearray(length)
|
||||
for offset, data in self.segments:
|
||||
left, right = max(offset, start), min(offset + len(data), end)
|
||||
if left < right:
|
||||
raw[left-start:right-start] = data[left-offset:right-offset]
|
||||
self.position = end
|
||||
return bytes(raw)
|
||||
|
||||
def readinto(self, buffer):
|
||||
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
|
||||
return len(raw)
|
||||
|
||||
|
||||
def arrow():
|
||||
try:
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
except ImportError:
|
||||
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
|
||||
return pa, pq
|
||||
|
||||
|
||||
def projection(source, columns, *, progress=None):
|
||||
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
|
||||
pa, pq = arrow()
|
||||
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
|
||||
raise ValueError('Bounded pinned Parquet size required')
|
||||
if (not isinstance(columns, dict) or not columns
|
||||
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
|
||||
raise ValueError('Explicit top-level metadata columns required')
|
||||
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
|
||||
if header != b'PAR1' or trailer[4:] != b'PAR1':
|
||||
raise ValueError('Expected unencrypted Parquet framing')
|
||||
length = struct.unpack('<I', trailer[:4])[0]
|
||||
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
|
||||
raise ValueError('Invalid or oversized Parquet footer')
|
||||
footer_start = source.size - 8 - length
|
||||
footer = source.read(footer_start, length)
|
||||
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
|
||||
sparse = SparseFile(source.size, segments)
|
||||
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
|
||||
try:
|
||||
schema = parquet.schema_arrow
|
||||
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
|
||||
raise ValueError('Missing or repeated metadata columns')
|
||||
for key, kind in columns.items():
|
||||
value = schema.field(key).type
|
||||
if kind == 'string':
|
||||
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
|
||||
elif kind == 'integer':
|
||||
okay = pa.types.is_integer(value)
|
||||
elif kind == 'strings':
|
||||
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
|
||||
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
|
||||
else:
|
||||
raise ValueError('Unsupported safe metadata type')
|
||||
if not okay:
|
||||
raise ValueError('Metadata column type differs: ' + key)
|
||||
metadata = parquet.metadata
|
||||
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
|
||||
raise ValueError('Bounded metadata row count required')
|
||||
wanted, excluded = [], []
|
||||
for group in range(metadata.num_row_groups):
|
||||
rg = metadata.row_group(group)
|
||||
for i in range(rg.num_columns):
|
||||
col = rg.column(i)
|
||||
if col.file_path:
|
||||
raise ValueError('External Parquet column files forbidden')
|
||||
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
|
||||
if not offsets:
|
||||
raise ValueError('Missing column chunk offset')
|
||||
start = min(offsets); end = start + col.total_compressed_size
|
||||
if not 4 <= start < end <= footer_start:
|
||||
raise ValueError('Column chunk lies outside the pinned payload')
|
||||
item = (start, end, group, col.path_in_schema)
|
||||
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
|
||||
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
|
||||
raise ValueError('Selected range overlaps an excluded payload column')
|
||||
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
|
||||
raise ValueError('Metadata projection exceeds byte budget')
|
||||
if len(wanted) + 3 > MAX_REQUESTS:
|
||||
raise ValueError('Metadata projection exceeds request budget')
|
||||
for start, end, group, path in sorted(wanted):
|
||||
segments.append((start, source.read(start, end-start)))
|
||||
if progress:
|
||||
progress({'phase': 'metadata_column_range', 'row_group': group,
|
||||
'column': path, 'metadata_bytes_read': source.bytes_read})
|
||||
rows = []
|
||||
for group in range(metadata.num_row_groups):
|
||||
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
|
||||
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
|
||||
raise ValueError('Unexpected projected metadata fields')
|
||||
rows.extend(table.select(list(columns)).to_pylist())
|
||||
if len(rows) != metadata.num_rows:
|
||||
raise ValueError('Incomplete metadata projection')
|
||||
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
|
||||
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
|
||||
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
|
||||
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
|
||||
'source_bytes': source.size, 'full_file_checksum_verified': False,
|
||||
'excluded_payload_pages_downloaded': False,
|
||||
'footer_saved': False, 'raw_parquet_saved': False}
|
||||
return rows, report
|
||||
finally:
|
||||
parquet.close()
|
||||
@@ -1 +0,0 @@
|
||||
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
|
||||
+1
-2
@@ -12,7 +12,6 @@ dependencies = ["PyYAML==6.0.3"]
|
||||
|
||||
[project.optional-dependencies]
|
||||
postgres = ["psycopg[binary]==3.3.8"]
|
||||
mirrors = ["pyarrow==26.0.1"]
|
||||
|
||||
[project.scripts]
|
||||
paper-atlas = "paper_atlas.cli:main"
|
||||
@@ -23,4 +22,4 @@ exclude = ["tests*"]
|
||||
|
||||
[tool.setuptools.package-data]
|
||||
"paper_atlas.db" = ["schema.sql", "annotations.sql"]
|
||||
"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"]
|
||||
"paper_atlas.resources" = ["venues.json"]
|
||||
|
||||
+706
@@ -0,0 +1,706 @@
|
||||
{
|
||||
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
|
||||
"gold": [
|
||||
{
|
||||
"doi": "10.1145/3767308.3834907",
|
||||
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
|
||||
"authors": [
|
||||
"Jia Wang",
|
||||
"Jiaming Cai",
|
||||
"Zunying Hu",
|
||||
"Zhanjie Wu",
|
||||
"Jinyuan Liu",
|
||||
"Hua Cheng",
|
||||
"Yun Peng",
|
||||
"Xin Fan"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-004.json",
|
||||
"source_position": 246,
|
||||
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
|
||||
"program_id": "mfp0053",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
|
||||
"program_context_pointers": [
|
||||
"/entries/129/papers/19",
|
||||
"/entries/129/subsessions/1/items/7"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3834936",
|
||||
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
|
||||
"authors": [
|
||||
"Jiacheng Lu",
|
||||
"Weijian Wang",
|
||||
"Mingyuan Xiao",
|
||||
"Yang Hua",
|
||||
"Tao Song",
|
||||
"Bo Peng",
|
||||
"Cheng Hua",
|
||||
"Haibing Guan"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-005.json",
|
||||
"source_position": 25,
|
||||
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
|
||||
"program_id": "mfp0229",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
|
||||
"program_context_pointers": [
|
||||
"/entries/109/papers/27",
|
||||
"/entries/109/subsessions/2/items/3"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3834946",
|
||||
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
|
||||
"authors": [
|
||||
"Renye Yan",
|
||||
"Jikang Cheng",
|
||||
"You Wu",
|
||||
"Wei Peng",
|
||||
"Zongwei Wang",
|
||||
"Ling Liang",
|
||||
"Yimao Cai"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-005.json",
|
||||
"source_position": 35,
|
||||
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
|
||||
"program_id": "mfp0297",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
|
||||
"program_context_pointers": [
|
||||
"/entries/61/papers/40",
|
||||
"/entries/61/subsessions/3/items/4"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3834956",
|
||||
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
|
||||
"authors": [
|
||||
"Renye Yan",
|
||||
"Jikang Cheng",
|
||||
"You Wu",
|
||||
"Baojin Huang",
|
||||
"Wei Peng",
|
||||
"Zongwei Wang",
|
||||
"Ling Liang",
|
||||
"Yimao Cai"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-006.json",
|
||||
"source_position": 235,
|
||||
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
|
||||
"program_id": "mfp0339",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/48",
|
||||
"/entries/88/subsessions/4/items/0"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835035",
|
||||
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
|
||||
"authors": [
|
||||
"Haoqian Kang",
|
||||
"Liupeng Li",
|
||||
"Kuofeng Gao",
|
||||
"Jinpeng Wang",
|
||||
"Zhenyu Lu",
|
||||
"Bin Chen",
|
||||
"Ke Chen",
|
||||
"Yaowei Wang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-004.json",
|
||||
"source_position": 8,
|
||||
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
|
||||
"program_id": "mfp0798",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/39",
|
||||
"/entries/88/subsessions/3/items/3"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835055",
|
||||
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
|
||||
"authors": [
|
||||
"Qiang Qi",
|
||||
"Wenqi Shang",
|
||||
"Xiao Wang",
|
||||
"Shuyuan Lin"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-004.json",
|
||||
"source_position": 28,
|
||||
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
|
||||
"program_id": "mfp0903",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
|
||||
"program_context_pointers": [
|
||||
"/entries/129/papers/55",
|
||||
"/entries/129/subsessions/4/items/7"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835101",
|
||||
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
|
||||
"authors": [
|
||||
"Zifan Zheng",
|
||||
"Yu Luo",
|
||||
"Yuxin Huang",
|
||||
"Chuancheng Shi",
|
||||
"Mohammad Masbaul Alam Polash",
|
||||
"Zhiyong Wang",
|
||||
"Kun Hu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-008.json",
|
||||
"source_position": 82,
|
||||
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
|
||||
"program_id": "mfp1175",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
|
||||
"program_context_pointers": [
|
||||
"/entries/109/papers/86",
|
||||
"/entries/109/subsessions/7/items/2"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835374",
|
||||
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
|
||||
"authors": [
|
||||
"Qianpeng Chong",
|
||||
"Jiakun Wen",
|
||||
"Guangyi Wei",
|
||||
"Wenyi Zeng"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-006.json",
|
||||
"source_position": 59,
|
||||
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
|
||||
"program_id": "mfp2617",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
|
||||
"program_context_pointers": [
|
||||
"/entries/61/papers/80",
|
||||
"/entries/61/subsessions/6/items/8"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835424",
|
||||
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
|
||||
"authors": [
|
||||
"Evan Ram",
|
||||
"Tuan Tran",
|
||||
"Youngwook Son",
|
||||
"Sangtae Ha"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-005.json",
|
||||
"source_position": 108,
|
||||
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
|
||||
"program_id": "mfp2909",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
|
||||
"program_context_pointers": [
|
||||
"/entries/129/papers/130",
|
||||
"/entries/129/subsessions/8/items/22"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835452",
|
||||
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
|
||||
"authors": [
|
||||
"He Peize",
|
||||
"Yaodi Luo",
|
||||
"Xiaoqian Liu",
|
||||
"Xuyang Liu",
|
||||
"Jiahang Deng",
|
||||
"Yaosong Du",
|
||||
"Li Bangyu",
|
||||
"Xiyan Gui",
|
||||
"Yuxuan Chen",
|
||||
"Linfeng Zhang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-004.json",
|
||||
"source_position": 167,
|
||||
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
|
||||
"program_id": "mfp3048",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
|
||||
"program_context_pointers": [
|
||||
"/entries/46/papers/78",
|
||||
"/entries/46/subsessions/6/items/6"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835460",
|
||||
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
|
||||
"authors": [
|
||||
"Chi Li",
|
||||
"Rui Lin",
|
||||
"Aobo Ji",
|
||||
"Dongzhu Xu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-004.json",
|
||||
"source_position": 175,
|
||||
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
|
||||
"program_id": "mfp3109",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
|
||||
"program_context_pointers": [
|
||||
"/entries/109/papers/121",
|
||||
"/entries/109/subsessions/8/items/13"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835544",
|
||||
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
|
||||
"authors": [
|
||||
"Boyan Li",
|
||||
"Ruijun Mao",
|
||||
"Zhicheng Liang",
|
||||
"Dayou Zhang",
|
||||
"Fangxin Wang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-006.json",
|
||||
"source_position": 129,
|
||||
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
|
||||
"program_id": "mfp3554",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
|
||||
"program_context_pointers": [
|
||||
"/entries/46/papers/153",
|
||||
"/entries/46/subsessions/9/items/21"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835591",
|
||||
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
|
||||
"authors": [
|
||||
"Jiafeng Huang",
|
||||
"Wenzhen Jia",
|
||||
"Shilong Dong",
|
||||
"Shengjie Zhao"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-007.json",
|
||||
"source_position": 97,
|
||||
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
|
||||
"program_id": "mfp3783",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
|
||||
"program_context_pointers": [
|
||||
"/entries/33/papers/158",
|
||||
"/entries/33/subsessions/10/items/2"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835667",
|
||||
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
|
||||
"authors": [
|
||||
"Yongqi Shan",
|
||||
"Yifan Wang",
|
||||
"Yunzhi Zhuge",
|
||||
"Lijun Wang",
|
||||
"Huchuan Lu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-005.json",
|
||||
"source_position": 209,
|
||||
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
|
||||
"program_id": "mfp4173",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/114",
|
||||
"/entries/88/subsessions/8/items/6"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835763",
|
||||
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
|
||||
"authors": [
|
||||
"Yuxuan Wang",
|
||||
"He Peize",
|
||||
"Xiyan Gui",
|
||||
"Xiaoqian Liu",
|
||||
"Junhao He",
|
||||
"Xuyang Liu",
|
||||
"Xuming Hu",
|
||||
"Linfeng Zhang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-006.json",
|
||||
"source_position": 192,
|
||||
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
|
||||
"program_id": "mfp4778",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
|
||||
"program_context_pointers": [
|
||||
"/entries/129/papers/100",
|
||||
"/entries/129/subsessions/7/items/16"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835895",
|
||||
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
|
||||
"authors": [
|
||||
"Zan Gao",
|
||||
"Xin Duan",
|
||||
"Nian Hu",
|
||||
"Hao Jia",
|
||||
"Yibo Zhao",
|
||||
"Cong Liu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-001.json",
|
||||
"source_position": 39,
|
||||
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
|
||||
"program_id": "mfp5537",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/184",
|
||||
"/entries/88/subsessions/11/items/6"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3835959",
|
||||
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
|
||||
"authors": [
|
||||
"Weiliang Huang",
|
||||
"Weichao Cai",
|
||||
"Bob Zhang",
|
||||
"Chao Huang",
|
||||
"Biao Xue",
|
||||
"Rongxin Zhang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-007.json",
|
||||
"source_position": 21,
|
||||
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
|
||||
"program_id": "mfp5823",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/66",
|
||||
"/entries/88/subsessions/5/items/6"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3836215",
|
||||
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
|
||||
"authors": [
|
||||
"Wentao Tan",
|
||||
"Bowen Wang",
|
||||
"Heng Zhi",
|
||||
"Chenyv Liu",
|
||||
"Zhe Li",
|
||||
"Jian Liu",
|
||||
"Zengrong Lin",
|
||||
"Yukun Dai",
|
||||
"Yipeng Chen",
|
||||
"Wenjie Yang",
|
||||
"Enci Xie",
|
||||
"Hao Xue",
|
||||
"Baixu Ji",
|
||||
"Chen Xu",
|
||||
"Zhibin Wang",
|
||||
"Tianshi Wang",
|
||||
"Lei Zhu",
|
||||
"Heng Tao Shen"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-003.json",
|
||||
"source_position": 46,
|
||||
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
|
||||
"program_id": "mfp7332",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
|
||||
"program_context_pointers": [
|
||||
"/entries/33/papers/13",
|
||||
"/entries/33/subsessions/1/items/1",
|
||||
"/entries/47/papers/0"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3836272",
|
||||
"title": "SEE: Structure-aware Exploring & Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
|
||||
"authors": [
|
||||
"Zhuohang Fan",
|
||||
"Beichen Zhang",
|
||||
"Yuanfa Li",
|
||||
"Changqiao Wu",
|
||||
"Wei Liu",
|
||||
"Jian Luan",
|
||||
"Weigang Zhang"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-002.json",
|
||||
"source_position": 127,
|
||||
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
|
||||
"program_id": "mfp7635",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
|
||||
"program_context_pointers": [
|
||||
"/entries/88/papers/131",
|
||||
"/entries/88/subsessions/8/items/23"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3836293",
|
||||
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
|
||||
"authors": [
|
||||
"Ruizhe Chen",
|
||||
"Zhiting Fan",
|
||||
"Yang Shi",
|
||||
"Enxin Song",
|
||||
"Wenhao Chai",
|
||||
"Tongkun Guan",
|
||||
"Songtao Jiang",
|
||||
"Ruilin Luo",
|
||||
"Yuanxing Zhang",
|
||||
"Shuai Bai",
|
||||
"Sibo Song",
|
||||
"Zhibo Yang",
|
||||
"Zuozhu Liu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-002.json",
|
||||
"source_position": 148,
|
||||
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
|
||||
"program_id": "mfp7732",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
|
||||
"program_context_pointers": [
|
||||
"/entries/46/papers/198",
|
||||
"/entries/46/subsessions/11/items/20"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3836447",
|
||||
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
|
||||
"authors": [
|
||||
"Raghu Vamsi Chittersu",
|
||||
"Yuvraj Singh Rathore",
|
||||
"Pranav Adlinge",
|
||||
"Eswar Sai Pashaula",
|
||||
"Kunal Swami"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-002.json",
|
||||
"source_position": 183,
|
||||
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
|
||||
"program_id": "mfp8648",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
|
||||
"program_context_pointers": [
|
||||
"/entries/107/papers/4",
|
||||
"/entries/117/papers/30",
|
||||
"/entries/117/subsessions/2/items/6"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3836600",
|
||||
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
|
||||
"authors": [
|
||||
"Mansi Sharma",
|
||||
"Dinar Orazgaliyev",
|
||||
"Antonio Krüger",
|
||||
"Philipp Müller"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-003.json",
|
||||
"source_position": 140,
|
||||
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
|
||||
"program_id": "mfp9815",
|
||||
"track_literal": "Main Track",
|
||||
"track": "main",
|
||||
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
|
||||
"program_context_pointers": [
|
||||
"/entries/46/papers/219",
|
||||
"/entries/46/subsessions/12/items/17"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3838656",
|
||||
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
|
||||
"authors": [
|
||||
"Xiaochen Wang",
|
||||
"Yuan Zhong",
|
||||
"Lingwei Zhang",
|
||||
"Lisong Dai",
|
||||
"Ting Wang",
|
||||
"Fenglong Ma"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-003.json",
|
||||
"source_position": 225,
|
||||
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
|
||||
"program_id": "mds177",
|
||||
"track_literal": "Datasets",
|
||||
"track": "datasets",
|
||||
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
|
||||
"program_context_pointers": [
|
||||
"/entries/46/papers/23",
|
||||
"/entries/46/subsessions/1/items/11"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3839012",
|
||||
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
|
||||
"authors": [
|
||||
"Abhijeet Narang",
|
||||
"Hagar Elbatanouny",
|
||||
"Vanishri Shekhawat",
|
||||
"Hasan Al-Nashash",
|
||||
"Abhinav Dhall",
|
||||
"Usman Tariq"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-007.json",
|
||||
"source_position": 238,
|
||||
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
|
||||
"program_id": "mde41",
|
||||
"track_literal": "Demo and Video",
|
||||
"track": "demo_and_video",
|
||||
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
|
||||
"program_context_pointers": [
|
||||
"/entries/94/papers/7"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3839024",
|
||||
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
|
||||
"authors": [
|
||||
"Jiangfan Liu",
|
||||
"Zexuan Cui",
|
||||
"Tianyuan Zhang",
|
||||
"Zonglei Jing",
|
||||
"Yaoyuan Zhang",
|
||||
"Jiakai Wang",
|
||||
"Jinyang Guo",
|
||||
"Yuqing Ma",
|
||||
"Lu Sheng",
|
||||
"Aishan Liu",
|
||||
"Xianglong Liu"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-008.json",
|
||||
"source_position": 0,
|
||||
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
|
||||
"program_id": "mde15",
|
||||
"track_literal": "Demo and Video",
|
||||
"track": "demo_and_video",
|
||||
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
|
||||
"program_context_pointers": [
|
||||
"/entries/49/papers/4"
|
||||
],
|
||||
"single_name_positions": []
|
||||
},
|
||||
{
|
||||
"doi": "10.1145/3767308.3839028",
|
||||
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
|
||||
"authors": [
|
||||
"Xiaoyan Yu",
|
||||
"Ziya Zhao",
|
||||
"Yunshan Ma",
|
||||
"Yifan Wei",
|
||||
"Dianbo Sui",
|
||||
"Tat-Seng Chua"
|
||||
],
|
||||
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||
"source_page": "page-008.json",
|
||||
"source_position": 4,
|
||||
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
|
||||
"program_id": "mde02",
|
||||
"track_literal": "Demo and Video",
|
||||
"track": "demo_and_video",
|
||||
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
|
||||
"program_context_pointers": [
|
||||
"/entries/49/papers/0"
|
||||
],
|
||||
"single_name_positions": []
|
||||
}
|
||||
],
|
||||
"author_disagreement_hold": {
|
||||
"doi": "10.1145/3767308.3836308",
|
||||
"program_id": "mfp7801"
|
||||
},
|
||||
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
|
||||
"held_gold": {
|
||||
"doi": "10.1145/3767308.3836308",
|
||||
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
|
||||
"authors": [
|
||||
"Zhirong Tian",
|
||||
"Yi Zhang",
|
||||
"Weixuan Liang",
|
||||
"Miaomiao Li",
|
||||
"Xinwang Liu"
|
||||
],
|
||||
"kind": "program_format_author_version_requires_review",
|
||||
"source_page": "page-001.json",
|
||||
"source_position": 82,
|
||||
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
|
||||
"program_id": null,
|
||||
"track_literal": null,
|
||||
"track": "proceedings_unclassified",
|
||||
"abstract_sha256": null,
|
||||
"program_context_pointers": [],
|
||||
"single_name_positions": []
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,9 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
fixtures=Path(__file__).parent/'fixtures'
|
||||
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
|
||||
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
|
||||
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
|
||||
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||
@@ -38,8 +41,8 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
|
||||
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
|
||||
for g in self.fixture['independent_gold']:
|
||||
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
|
||||
for g in self.gold:
|
||||
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||
@@ -51,22 +54,22 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
|
||||
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
for g in self.gold:
|
||||
r=self.by_doi[g['doi']];m=r['metadata']
|
||||
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||
if g['abstract_sha256'] is not None:
|
||||
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
|
||||
self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
|
||||
else:
|
||||
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
|
||||
self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
|
||||
|
||||
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||
for p in self.report['unmatched_program']:
|
||||
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
|
||||
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
|
||||
gold=self.program_fixture['independent_inventory']['inventory']
|
||||
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||
for g in gold:
|
||||
@@ -91,7 +94,7 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||
self.assertEqual(len(variants),192)
|
||||
self.assertEqual(len(variants),166)
|
||||
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||
|
||||
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||
@@ -113,7 +116,7 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
|
||||
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||
n=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
for g in self.gold:
|
||||
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||
@@ -123,13 +126,62 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
|
||||
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||
count=0
|
||||
for g in self.fixture['independent_gold']:
|
||||
for g in self.gold:
|
||||
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||
count+=1;r=self.by_doi[g['doi']]
|
||||
self.assertEqual(r['title'],g['title'])
|
||||
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||
|
||||
|
||||
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
|
||||
self.assertEqual(len(self.format_gold['gold']),26)
|
||||
for gold in self.format_gold['gold']:
|
||||
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
|
||||
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
|
||||
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
|
||||
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||
comparison=m['program_title_formatting_comparison']
|
||||
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
|
||||
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
|
||||
self.assertFalse(m['scientific_work_identity_approved'])
|
||||
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
|
||||
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
|
||||
self.assertEqual(held['metadata']['program_exact_candidates'],[])
|
||||
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
|
||||
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
|
||||
|
||||
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
|
||||
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> & C'),'A H2 B1 & C')
|
||||
self.assertEqual(c.formatting_text('A &amp; B'),'A & B')
|
||||
|
||||
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
|
||||
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
|
||||
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A <i>B</i>',
|
||||
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
|
||||
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
|
||||
|
||||
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
|
||||
def observation(pid,authors):
|
||||
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
|
||||
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
|
||||
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
|
||||
self.assertNotIn('program',rows[0])
|
||||
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
|
||||
c.match_program(rows,[observation('p1',['A','B'])])
|
||||
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
|
||||
self.assertNotIn('program',rows[0])
|
||||
|
||||
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
|
||||
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
|
||||
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
|
||||
c.match_program(rows,observations)
|
||||
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
|
||||
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
|
||||
|
||||
|
||||
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
@@ -150,9 +202,12 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||
|
||||
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
|
||||
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
|
||||
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
|
||||
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
|
||||
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
|
||||
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
|
||||
self.assertEqual(self.report['unmatched_program_count'],391)
|
||||
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||
@@ -196,7 +251,7 @@ class MMPublications2026Tests(unittest.TestCase):
|
||||
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
|
||||
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
|
||||
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
|
||||
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726)
|
||||
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
|
||||
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
|
||||
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
|
||||
|
||||
@@ -1,203 +0,0 @@
|
||||
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
|
||||
from collections import Counter
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import urllib.error
|
||||
|
||||
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
|
||||
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
|
||||
|
||||
|
||||
class MemorySource:
|
||||
def __init__(self, raw):
|
||||
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
|
||||
|
||||
def read(self, start, length):
|
||||
raw = self.raw[start:start+length]
|
||||
self.bytes_read += len(raw)
|
||||
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||
return raw
|
||||
|
||||
|
||||
def fixture():
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
rows = []
|
||||
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
|
||||
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
|
||||
'track': 'main', 'venue_id': 'source-venue-claim',
|
||||
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
|
||||
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
|
||||
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
|
||||
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
|
||||
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
|
||||
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
|
||||
rows.append(row)
|
||||
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
|
||||
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
|
||||
|
||||
|
||||
class RangeHTTPTests(unittest.TestCase):
|
||||
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
|
||||
response = io.BytesIO(data); response.status = status
|
||||
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
|
||||
return response
|
||||
|
||||
def test_strict_range_and_changed_object_detection(self):
|
||||
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
|
||||
source = RangeSource('https://example.org/public', 10, opener=opener)
|
||||
self.assertEqual(source.read(2, 2), b'ab')
|
||||
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
|
||||
with self.assertRaisesRegex(ValueError, 'ETag'):
|
||||
source.read(4, 2)
|
||||
self.assertEqual(source.bytes_read, 2)
|
||||
|
||||
def test_no_full_file_fallback_or_denied_source_retry(self):
|
||||
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
|
||||
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
|
||||
opener.open.return_value = response
|
||||
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
|
||||
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
||||
response.read.assert_not_called()
|
||||
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
|
||||
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
|
||||
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
||||
self.assertEqual(opener.open.call_count, 1)
|
||||
|
||||
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
|
||||
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
|
||||
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
|
||||
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
|
||||
with self.assertRaises(ValueError):
|
||||
source.seek(-101, 2)
|
||||
|
||||
|
||||
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
|
||||
class ParquetProjectionTests(unittest.TestCase):
|
||||
def test_complete_selected_rows_and_no_excluded_page_requests(self):
|
||||
import pyarrow.parquet as pq
|
||||
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
|
||||
self.assertEqual(rows, expected)
|
||||
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
|
||||
excluded = []
|
||||
for i in range(metadata.num_row_groups):
|
||||
group = metadata.row_group(i)
|
||||
for j in range(group.num_columns):
|
||||
col = group.column(j)
|
||||
if col.path_in_schema.split('.')[0] not in COLUMNS:
|
||||
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
|
||||
excluded.append((start, start+col.total_compressed_size))
|
||||
for request in source.receipts:
|
||||
a, b = request['start'], request['start']+request['length']
|
||||
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
|
||||
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
|
||||
self.assertLess(source.bytes_read, len(raw))
|
||||
encoded = json.dumps(rows)+json.dumps(proof)
|
||||
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
|
||||
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
|
||||
|
||||
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
|
||||
raw, _ = fixture(); source = MemorySource(raw)
|
||||
with self.assertRaisesRegex(ValueError, 'Missing'):
|
||||
projection(source, {'missing_column': 'string'})
|
||||
self.assertEqual(len(source.receipts), 3)
|
||||
source = MemorySource(raw)
|
||||
with self.assertRaisesRegex(ValueError, 'type differs'):
|
||||
projection(source, {**COLUMNS, 'year': 'string'})
|
||||
self.assertEqual(len(source.receipts), 3)
|
||||
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
|
||||
source = MemorySource(bytes(bad))
|
||||
with self.assertRaisesRegex(ValueError, 'footer'):
|
||||
projection(source, COLUMNS)
|
||||
self.assertEqual(len(source.receipts), 2)
|
||||
|
||||
def test_collector_output_preserves_claims_without_original_identity(self):
|
||||
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
out = Path(temp)/'projected'
|
||||
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
||||
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
||||
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
|
||||
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
|
||||
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
|
||||
for row in projected:
|
||||
self.assertEqual(row['identifiers'], {})
|
||||
self.assertFalse(row['metadata']['original_forum_id_verified'])
|
||||
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||
self.assertNotIn('status', row)
|
||||
self.assertIn(COMMIT, row['external_id'])
|
||||
self.assertEqual(projected[2]['authors'], ['Anonymous'])
|
||||
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
|
||||
self.assertEqual(projected[0]['track'], 'main')
|
||||
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
|
||||
all_saved=''.join(p.read_text() for p in out.iterdir())
|
||||
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
|
||||
with self.assertRaisesRegex(ValueError, 'New output'):
|
||||
collect('emnlp', out)
|
||||
|
||||
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
|
||||
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
||||
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
||||
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
||||
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
|
||||
# Mimic the first reviewed source version: new literal spellings were unknown.
|
||||
for row in before[:2]:
|
||||
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
|
||||
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
|
||||
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
|
||||
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
|
||||
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
|
||||
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
|
||||
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
|
||||
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
|
||||
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
|
||||
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
|
||||
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
|
||||
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
|
||||
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
|
||||
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
|
||||
with self.assertRaisesRegex(ValueError, 'hash differs'):
|
||||
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
|
||||
no_network.assert_not_called()
|
||||
|
||||
def test_failure_is_retained_without_source_payload_or_retries(self):
|
||||
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
|
||||
with tempfile.TemporaryDirectory() as temp:
|
||||
out = Path(temp)/'failed'
|
||||
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
|
||||
with self.assertRaises(RuntimeError):
|
||||
collect('emnlp', out, source_factory=lambda url, size: None)
|
||||
report = json.loads((out/'failure.json').read_text())
|
||||
self.assertEqual(report['status'], 'incomplete_do_not_import')
|
||||
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
|
||||
|
||||
|
||||
class MirrorSemanticsTests(unittest.TestCase):
|
||||
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
|
||||
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
|
||||
result = {s: inventory(s) for s in splits}
|
||||
self.assertEqual(sum(len(v) for v in result.values()), 15)
|
||||
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
|
||||
self.assertEqual(result['emnlp'][0]['size'], 90794691)
|
||||
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
|
||||
with self.assertRaises(ValueError):
|
||||
inventory('uai')
|
||||
|
||||
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
|
||||
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
|
||||
self.assertEqual(decision_claim(literal), 'unknown')
|
||||
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
|
||||
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
|
||||
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
|
||||
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
|
||||
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
|
||||
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
|
||||
self.assertIsNone(decision_track_claim('Accept (Poster)'))
|
||||
Reference in New Issue
Block a user