Compare commits

..

3 Commits

Author SHA1 Message Date
codex-bot 8e548287e9 Preserve existing ACL acceptance for EZ-STANCE title variant
Metadata integrity checks / metadata (push) Successful in 7m7s
2026-11-18 12:10:40 +08:00
codex-bot e85902bf18 Preserve EZ-STANCE later-publication lead without anonymous identity inference
Metadata integrity checks / metadata (push) Successful in 6m44s
2026-11-18 12:07:24 +08:00
codex-bot e82abec4c7 Record current negative-source overlap and unresolved note identities
Metadata integrity checks / metadata (push) Successful in 7m20s
2026-11-18 12:05:05 +08:00
14 changed files with 381 additions and 762 deletions
+1 -1
View File
@@ -35,7 +35,7 @@ jobs:
venv = root / 'venv' venv = root / 'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python = str(venv / 'bin/python') python = str(venv / 'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True) subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
testenv = os.environ.copy() testenv = os.environ.copy()
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
if service_file.is_file(): if service_file.is_file():
-88
View File
@@ -1,88 +0,0 @@
name: ReviewArena metadata projection
on:
workflow_dispatch:
inputs:
split:
description: 'Pinned public mirror family; metadata columns only'
required: true
default: emnlp
projection_name:
description: 'Optional existing metadata output directory name for offline replay'
required: false
default: ''
projection_sha256:
description: 'Exact cached collection-report.json SHA256, required for offline replay'
required: false
default: ''
jobs:
project:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
MIRROR_SPLIT: ${{ inputs.split }}
MIRROR_PROJECTION: ${{ inputs.projection_name }}
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Project permitted metadata on the approved remote runner
run: |
python3 -B - <<'PY'
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
split = os.environ['MIRROR_SPLIT']
cached = os.environ.get('MIRROR_PROJECTION','')
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
raise SystemExit('Explicit approved source commit/split required')
base = pathlib.Path('/data/paper-index')
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
if out.exists():
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
extra=[]
if cached:
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
raise SystemExit('Explicit permitted cached projection name and report hash required')
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
elif cached_sha:
raise SystemExit('Cached projection name required')
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
with urllib.request.urlopen(url,timeout=60) as response:
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Oversized source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
tar.extractall(root,filter='data')
candidates=list(root.glob('*/pyproject.toml'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive')
src=candidates[0].parent
venv=root/'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python=str(venv/'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
wheels=root/'wheels'
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
wheel,=wheels.glob('paper_atlas-*.whl')
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside=root/'outside';outside.mkdir()
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
report=json.loads((out/'collection-report.json').read_bytes())
assert report['database_written'] is False and report['approved_for_import']==0
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
backup=base/'validation'/(out.name+'-recovery.tar.gz')
if backup.exists():
raise SystemExit('Recovery archive already exists; inspect')
with tarfile.open(backup,'w:gz') as tar:
tar.add(out,arcname=str(out.relative_to(base)))
backup.chmod(0o444)
for path in out.iterdir():
if path.is_file():path.chmod(0o444)
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
PY
-2
View File
@@ -2,8 +2,6 @@
**长期维护、按需运行的论文元数据与公开投稿历史工具。** **长期维护、按需运行的论文元数据与公开投稿历史工具。**
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
+2
View File
@@ -202,3 +202,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。 [早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。 - [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
- [2026-11-18 公开负面来源核查](public-negative-source-audit-2026-11-18.md):ResearchArcade 930 条非 ICLR 声明的当前库比对、5 条未绑定声明及原始接口访问证据;不把镜像差集算作新增拒稿。
@@ -1493,7 +1493,7 @@
}, },
"TMLR_current20261108": { "TMLR_current20261108": {
"url": "https://www.jmlr.org/tmlr/faq.html", "url": "https://www.jmlr.org/tmlr/faq.html",
"finding": "Additionalnonconferencevenuealreadyinregistry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance." "finding": "Additionalnonconferencepolicyreference; TMLR is not in the current 43-conference registry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance."
} }
}, },
"public_negative_recovery_notes": { "public_negative_recovery_notes": {
@@ -1507,6 +1507,12 @@
"original_rejection_notes_obtained": false, "original_rejection_notes_obtained": false,
"prior_decisions_inserted": 0, "prior_decisions_inserted": 0,
"requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note." "requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note."
},
"researcharcade_20261118": {
"audit": "public-negative-source-audit-2026-11-18.json",
"summary": "930 non-ICLR 2020+ mirror claims:925 scoped ID matches all have rejected status;5 held,3 with equal title+source abstract but distinct unverified note IDs. No writes or new rejections approved.",
"access": "NeurIPS2021 and EMNLP2023 original note queries403; stopped, no bypass.",
"next": "Seek independent primary evidence for note.id/forum relationships and final decisions; no title-only merge or duplicate-rejection import. No full-public-set coverage claim."
} }
}, },
"restore_rules": [ "restore_rules": [
@@ -0,0 +1,320 @@
{
"version": 1,
"as_of": "2026-11-18",
"scope": "930 cached ResearchArcade non-ICLR negative claims, NeurIPS 2021–2025 and EMNLP 2023; not a conference-completeness audit",
"database_snapshot": {
"database": "foundry_data",
"schema": "research",
"at": "2026-11-18 03:48:34.65659+00",
"snapshot": "1048885:1048885:"
},
"database_written": false,
"approved_for_import": 0,
"claims": 930,
"known_same_scope_attempt_claims": 925,
"all_925_have_rejected_status": true,
"without_same_scope_attempt_claims": 5,
"by_venue_year": [
{
"venue": "EMNLP",
"year": 2023,
"claims": 10
},
{
"venue": "NeurIPS",
"year": 2021,
"claims": 136
},
{
"venue": "NeurIPS",
"year": 2022,
"claims": 153
},
{
"venue": "NeurIPS",
"year": 2023,
"claims": 176
},
{
"venue": "NeurIPS",
"year": 2024,
"claims": 201
},
{
"venue": "NeurIPS",
"year": 2025,
"claims": 254
}
],
"evidence": {
"diff_report_sha256": "d49361a8622f3b3f59e824690ceb8bf1c878be35c3e3758366d3b46a407673f5",
"candidate_packets_sha256": "3b21df27e0f30e56b81086d16d3943c4563bbe382200e866b15de14510e07472",
"dataset_revision": "c61bb99b0452429c32f85b1e392f3afdb65731c4",
"dataset_parquet_sha256": "bb6e165ee152a95d91a7ffa16abc9d4a6effbdfcf59183b3bb819d09171689d1",
"candidate_jsonl_sha256": "848e646596e8c747e0c636c2381a139978aff8f2c19f01ad49c98e7f4270c5e6",
"collector_repository": "https://github.com/ulab-uiuc/research-arcade",
"collector_revision": "ecf9c7c2490840b029bd2add4f02d04aca1d014e",
"collector_openreview_crawler_sha256": "4539d951ca05b20785169638eab3884148aa6c354a34c5ecc04e15c84a5910fe",
"collector_revision_proven_to_generate_dataset": false
},
"collector_semantics": {
"legacy_2020_2022": "Reads Blind_Submission note.id; first reply with content.decision, no invitation/signature verification. Not a verified final-decision note.",
"legacy_2019": "Reads first recommendation, which must not be promoted to a final decision. Outside active 2020+ backfill.",
"modern_branch": "Reads Submission note.id and last venueid component; Conference replaced by venue literal. Withdrawn_Submission skipped.",
"identity": "The exported field paper_openreview_id is note.id; forum equality or equivalence to another mirror ID is not independently proved.",
"coverage": "Six-field table has no authors or decision signatures; no completeness inference for public withdrawn/rejected records."
},
"held_claims": [
{
"source_row": 33219,
"note_id_claim": "FSf9h6t152",
"venue": "NeurIPS",
"year": 2025,
"raw_decision": "Rejected_Submission",
"title": "Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations",
"same_scope_identifier_match": false,
"exact_title_candidates": [],
"approved_for_import": false,
"disposition": "hold_no_exact_title_candidate_not_proof_of_novelty"
},
{
"source_row": 44669,
"note_id_claim": "Pk972F7J8Um",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "Differential Privacy of Dirichlet Posterior Sampling",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 3527429,
"existing_identifiers": [
{
"scheme": "openreview",
"value": "fgBxcKcz9ea",
"paper_id": 3527429
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/fgBxcKcz9ea",
"paper_id": 3527429
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444666,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/fgBxcKcz9ea",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 44696,
"note_id_claim": "ki4eJ1fSJNq",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "Fitting large mixture models using stochastic component selection",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 1545058,
"existing_identifiers": [
{
"scheme": "arxiv",
"value": "2110.04776",
"paper_id": 1545058
},
{
"scheme": "openreview",
"value": "ZS394D3djsg",
"paper_id": 1545058
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/ZS394D3djsg",
"paper_id": 1545058
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444652,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/ZS394D3djsg",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 45757,
"note_id_claim": "bDP7I2fYwLj",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "WeaveNet for Approximating Assignment Problems",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 3527390,
"existing_identifiers": [
{
"scheme": "openreview",
"value": "2rAiDBJgR_",
"paper_id": 3527390
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/2rAiDBJgR_",
"paper_id": 3527390
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444580,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/2rAiDBJgR_",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 49327,
"note_id_claim": "yB8cQIICqe",
"venue": "EMNLP",
"year": 2023,
"raw_decision": "Desk_Rejected_Submission",
"title": "EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection",
"same_scope_identifier_match": false,
"exact_title_candidates": [],
"approved_for_import": false,
"disposition": "hold_possible_later_publication_with_changed_title_and_dataset_size_no_identity_or_author_transfer",
"later_publication_retrieval_lead": {
"url": "https://aclanthology.org/2024.acl-long.838/",
"http_status": 200,
"bytes": 40557,
"response_sha256": "b50dd55c48e31e4c10e97927ce01679369735834aa2c0f8d9f41f8e8e02fc776",
"checked_at": "2026-11-18T04:07:03.699767+00:00",
"anthology_id": "2024.acl-long.838",
"doi": "10.18653/v1/2024.acl-long.838",
"title": "EZ-STANCE: A Large Dataset for English Zero-Shot Stance Detection",
"venue": "ACL",
"year": 2024,
"publication_verified": true,
"link_to_anonymous_rejection_verified": false,
"authors_transferred_to_anonymous_claim": false,
"dataset_pairs_published_version": 47316,
"dataset_pairs_rejection_mirror_version": 30606,
"database_written": false,
"current_database_evidence": {
"snapshot": {
"at": "2026-11-18 04:09:23.720956+00",
"snapshot": "1051887:1051887:"
},
"paper_id": 2131805,
"verified_identifiers": [
{
"scheme": "acl",
"value": "2024.acl-long.838",
"paper_id": 2131805
},
{
"scheme": "acl_anthology",
"value": "2024.acl-long.838",
"paper_id": 2131805
},
{
"scheme": "doi",
"value": "10.18653/v1/2024.acl-long.838",
"paper_id": 2131805
}
],
"existing_attempts": [
{
"venue": "ACL",
"year": 2024,
"track": "main",
"statuses": [
"accepted"
]
}
],
"packet_file_sha256": "de5dfa658177adef87134cf7723afc086bd7395405a31166ff9597b03b3c8af3",
"original_rejection_link_verified": false
}
}
}
],
"access_probes": [
{
"local_evidence": "researcharcade-neurips2021-original-note-check-v1/report.json",
"sha256": "746d752535b516abba43b0e8e6ad901bf39c07da532188dfa97ff911869084ae",
"receipt_summary": {},
"results": [
{
"requested_note_id": "Pk972F7J8Um",
"status": 403,
"url": "https://api.openreview.net/notes?id=Pk972F7J8Um&select=id%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccontent.title%2Ccontent.abstract%2Ccontent.venue%2Ccontent.venueid",
"sha256": "b0113384cebc9d724284062f776b45044d22450219112342721d8f9683571c08",
"checked_at": "2026-11-18T04:01:58.643552+00:00"
}
]
},
{
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/notes-probe.json",
"sha256": "47799dc4520e113d6a2e7c9ea2f362577bb21e3ad583ee02149a36c21cf6ea9d",
"receipt_summary": {
"url": "https://api.openreview.net/notes?invitation=EMNLP%2F2023%2FConference%2F-%2FSubmission&limit=1&select=id%2Cnumber%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccdate%2Ctcdate%2Cmdate%2Ctmdate%2Ccontent.title%2Ccontent.abstract%2Ccontent.authors%2Ccontent.venue%2Ccontent.venueid%2Ccontent.keywords",
"http_status": 403,
"response_sha256": "798e964735eb1eff1e4954d5c9e79aad2afd40c3f88dcb82f8ef5bbc55cc0a61",
"response_bytes": 546,
"at": "2026-11-18T03:38:50.112624+00:00"
}
},
{
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/group-access.json",
"sha256": "8fcb9ccd5615a737c17ef0eeedc9121e85a84ebd63667e83df0bda9ffb83d792",
"receipt_summary": {
"url": "https://api.openreview.net/groups?id=EMNLP%2F2023%2FConference",
"http_status": 200,
"response_sha256": "7c0cc75e2e8be1c18dffc96a9ceabea34f0f7810139c0a990a6bb117d66f6456",
"response_bytes": 13143,
"at": "2026-11-18T03:38:49.929085+00:00"
}
}
],
"access_limit": "NeurIPS 2021 first original note request and EMNLP 2023 source-declared Submission query returned 403. Requests stopped. No alternate API version, endpoint, authentication, or challenge bypass. 403 does not prove zero public records."
}
@@ -0,0 +1,50 @@
# 公开负面来源核查:2026-11-18
本次检查 ResearchArcade 缓存中 NeurIPS 2021–2025、EMNLP 2023 的 930 条负面声明,不包括 ICLR 或 2020 年以前的补漏。原始候选文件未重新下载,PostgreSQL 查询为只读。逐条待办、来源哈希与数据库快照见[机器可读记录](public-negative-source-audit-2026-11-18.json)。没有新增论文、决定、ID 别名或合并。
## 当前库比对
925 条声明通过同会议、同年份的 OpenReview 标识匹配到投稿,且每条都有 rejected 决定。这里只证明对应拒稿已记录,未核验这些论文的所有后续录用历史,也未证明会议公开拒稿全集完整。
另 5 条没有同届标识匹配:
|来源 note.id 声明|会议|标题或识别信息|处置|
|---|---|---|---|
|FSf9h6t152|NeurIPS 2025|Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations|无精确标题候选;不是已证明的新论文|
|Pk972F7J8Um|NeurIPS 2021|Differential Privacy of Dirichlet Posterior Sampling|候选 paper 3527429,已有同届 rejected,原 ID fgBxcKcz9ea|
|ki4eJ1fSJNq|NeurIPS 2021|Fitting large mixture models using stochastic component selection|候选 paper 1545058,已有同届 rejected,原 ID ZS394D3djsg;另保留 arXiv 2110.04776|
|bDP7I2fYwLj|NeurIPS 2021|WeaveNet for Approximating Assignment Problems|候选 paper 3527390,已有同届 rejected,原 ID 2rAiDBJgR_|
|yB8cQIICqe|EMNLP 2023|EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection|镜像称 Desk_Rejected_Submission;原始公开范围及身份未核验|
三个标题候选的镜像摘要都与已有 PaperCopilot 来源摘要逐字相等,但来源 ID 不同。相等文本不能独立证明两个 note.id 指向同一个 forum,也不能据此创建第二次投稿或批准 ID 别名。现有实体包没有截断投稿列表,均只有已记录的 NeurIPS 2021 拒稿;这不代表从未在其他地方录用。
## 采集器证据与使用限制
作者仓库 [research-arcade](https://github.com/ulab-uiuc/research-arcade) 固定版本 `ecf9c7c2490840b029bd2add4f02d04aca1d014e` 的 `openreview_crawler.py` 显示:
- 2020–2022 的旧接口路径读取 Blind_Submission 的 `note.id`,从 replies 中取第一个带 `decision` 字段的值,未验证决定 invitation 或签名。
- 2019 路径取第一个 `recommendation`,不能将其升级为正式决定。该年度属于保留线索的范围,本轮不补导。
- 新接口路径读取 Submission 的 `note.id`,用 `venueid` 尾段作为标签;遇 Conference 再取 `venue`,并跳过 Withdrawn_Submission。
三个源码文件均核对 Git blob SHA-1 和完整文件 SHA-256;没有执行第三方采集器。源码版本早于数据集版本,但未证明该提交实际生成了此数据集。数据集仅有六个元数据字段,缺少作者和决定签名,因此既不能证明完整 withdrawn 覆盖,也不能将导出的 ID 无条件认作已验证 forum。
## 原始来源访问结果
EMNLP 2023 官方分组元数据返回 200,其声明的 Submission invitation 与既有配置一致;对该 invitation 的匿名元数据查询返回 403。NeurIPS 2021 对第一条候选 note `Pk972F7J8Um` 的匿名元数据查询同样返回 403,余下五个 ID 未发请求。均已停止对应来源查询,没有换接口版本、登录或绕过访问限制。403 不能说明没有公开投稿;未保留完整错误正文,也不将具体拒绝原因猜测为已知。
[EMNLP 2023 官方政策](https://2023.emnlp.org/blog/EMNLP-2023-openreview/)说明公开录用稿件与选择公开的拒稿,现有 9 条 rejected 或本镜像的 1 条 desk-rejected 均不能证明完整公开集合。
## EZ-STANCE 后续发表线索
独立的 [ACL Anthology 官方条目](https://aclanthology.org/2024.acl-long.838/)确认相近标题的 EZ-STANCE 在 ACL 2024 Long Papers 发表,DOI `10.18653/v1/2024.acl-long.838`。正式标题多了 English,数据集由拒稿镜像中的 30,606 对变为 47,316 对。已保存官方 HTML 的完整哈希,登记为版本关系核查线索;尚未证明与匿名 EMNLP 2023 声明属于同一投稿或同一科学工作,没有把已发表作者转填到匿名记录,也没有写入关联或第二份论文。后续必须核验版本关系并保留各自的投稿决定证据。
当前 PostgreSQL 的正式标题变体查询命中已有 paper `2131805`,其 ACL/DOI 标识与官方条目一致,已有 `ACL / 2024 / main / accepted` 投稿历史。该实体无需重复新建。匿名 EMNLP 声明与该实体的关系仍未批准,不能把这条线索表述为已经确认的“先拒后录”。快照、标识及完整实体包哈希已记入机器记录。
## 下次从哪里继续
1. 保留五条候选及旧 ID,等待可验证的原始公开元数据或独立可靠来源;不要反复重放已拒绝的原始接口请求。
2. 对三对 ID 分歧先取得 note/forum 关系证据,再独立审核科学工作身份与逐次投稿关系;保留已有拒稿及以后发现的录用。
3. 两条没有精确标题候选的记录仍需标题变体、作者及摘要交叉核查;匿名元数据不能用于推断作者身份。
4. 继续其余会议与年度的公开负面可用性核查,不将这 930 条当作全会、全年份覆盖,也不将 accepted-only 镜像算作拒稿补全。
另修正交接记录:TMLR 是额外的期刊政策线索,当前 43 个会议的 registry 中没有 TMLR;没有因此更改会议分母。2000–2019 已发现线索仍保留。
-52
View File
@@ -1,52 +0,0 @@
# ReviewArena 镜像的索引字段投影
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
## 运行
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
包内命令如下,`--out` 必须是既有父目录内的新目录:
```sh
paper-atlas collect reviewarena-metadata \
--split emnlp \
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
```
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
```sh
paper-atlas collect reviewarena-metadata --split emnlp \
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
--projection-sha256 <已验证的原报告SHA256> \
--out /data/paper-index/collected/<新解释版本>
```
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
## 实际读取和保存范围
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
## 决定和身份边界
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
-1
View File
@@ -20,7 +20,6 @@ COMMANDS = {
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'), ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'), ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
-216
View File
@@ -1,216 +0,0 @@
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
from importlib.resources import files
import json
from pathlib import Path
import re
from paper_atlas.io.parquet_metadata import RangeSource, projection
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
SOURCE = 'reviewarena_mirror'
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
'title': 'string', 'abstract': 'string', 'authors': 'strings',
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
def inventory(split):
if split not in SPLITS:
raise ValueError('Explicit supported mirror split required')
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
raise ValueError('Pinned mirror file inventory changed')
items = json.loads(raw)
if len(items) != 15:
raise ValueError('Pinned mirror shard inventory changed')
selected = []
for item in items:
path = item['path']
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
raise ValueError('Unexpected mirror shard path')
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
raise ValueError('Missing source object size/checksum claim')
if path.startswith('data/'+split+'-'):
selected.append(item)
parts = sorted(selected, key=lambda x: x['path'])
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
raise ValueError('Incomplete selected mirror split inventory')
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
raise ValueError('Mirror shard sequence changed')
return parts
def decision_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'reject', 'rejected'}:
return 'rejected'
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
return 'desk_rejected'
if value in {'withdraw', 'withdrawn'}:
return 'withdrawn'
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
return 'accepted'
return 'unknown'
def decision_track_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'accept-findings', 'accept (findings)'}:
return 'findings'
if value in {'accept-main', 'accept (main)'}:
return 'main'
return None
def record(row, split, item, row_index):
if set(row) != set(COLUMNS):
raise ValueError('Unexpected projected metadata fields')
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
raise ValueError('Mirror conference/year outside pinned source scope')
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
'mirror_decision_category': decision_claim(row['decision']),
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
'source_locator': locator, 'identifiers': {},
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
'scientific_work_identity_approved': False, 'research_type_classified': False,
'approved_for_import': False,
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
def cached_projections(path, expected_sha256, split, items):
path = Path(path)
raw = path.read_bytes()
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
or hashlib.sha256(raw).hexdigest() != expected_sha256):
raise ValueError('Cached projection report hash differs')
report = json.loads(raw)
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
or report.get('approved_for_import') != 0):
raise ValueError('Cached report scope differs from permitted mirror projection')
declared = report.get('files', [])
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
raise ValueError('Incomplete cached mirror shard set')
result = {}
for item in items:
fm = next((f for f in declared if f['source_file'] == item['path']), None)
filename = Path(item['path']).stem+'.metadata.jsonl'
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
raise ValueError('Cached mirror source differs')
file = path.parent/filename
if file.is_symlink():
raise ValueError('Symlink metadata cache forbidden')
content = file.read_bytes()
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
raise ValueError('Cached metadata file hash differs')
rows = []
for index, line in enumerate(content.splitlines()):
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
if previous.get(key) != expected[key]:
raise ValueError('Cached raw metadata/identity differs: '+key)
rows.append(row)
proof = fm['projection']
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
or proof['full_file_checksum_verified'] is not False):
raise ValueError('Cached projection proof differs')
result[item['path']] = (rows, proof)
if sum(len(rows) for rows, _ in result.values()) != report['records']:
raise ValueError('Cached projection row count differs')
return result
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
items = inventory(split); out = Path(out)
if bool(from_projection) != bool(projection_sha256):
raise ValueError('Cached projection and exact report hash required together')
if out.exists() or not out.parent.is_dir() or out.is_symlink():
raise ValueError('New output directory in existing parent required')
out.mkdir()
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
observed = datetime.now(timezone.utc).isoformat()
try:
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
for item in items:
if cached is None:
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
source = source_factory(source_url, item['size'])
rows, proof = projection(source, COLUMNS, progress=progress)
else:
rows, proof = cached[item['path']]
if progress:
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
path = out/(Path(item['path']).stem+'.metadata.jsonl')
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
with part.open('wb') as stream:
for index, row in enumerate(rows):
item_record = record(row, split, item, index)
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
stream.write(raw); h.update(raw); records += 1
counts[item_record['mirror_decision_category']] += 1
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
years[(row['year'], row['track'])] += 1
part.rename(path)
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
'projection': proof})
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
'mirror_decision_track_claim_counts': dict(track_claims),
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
'cached_projection_report_sha256': projection_sha256,
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
'approved_for_import': 0,
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
'Every selected row and row locator is retained, including missing or anonymous author data.',
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
'Rows are not independent scientific works or a complete venue/year public negative census.',
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except BaseException as error:
# No source-specific body or signed redirect URLs in failure reports.
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
'completed_files': reports, 'error_type': type(error).__name__,
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--split', required=True, choices=SPLITS)
parser.add_argument('--out', required=True)
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
args = parser.parse_args(argv)
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
progress=lambda row: print(json.dumps(row), flush=True))
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
-195
View File
@@ -1,195 +0,0 @@
"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()
@@ -1 +0,0 @@
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
+1 -2
View File
@@ -12,7 +12,6 @@ dependencies = ["PyYAML==6.0.3"]
[project.optional-dependencies] [project.optional-dependencies]
postgres = ["psycopg[binary]==3.3.8"] postgres = ["psycopg[binary]==3.3.8"]
mirrors = ["pyarrow==26.0.1"]
[project.scripts] [project.scripts]
paper-atlas = "paper_atlas.cli:main" paper-atlas = "paper_atlas.cli:main"
@@ -23,4 +22,4 @@ exclude = ["tests*"]
[tool.setuptools.package-data] [tool.setuptools.package-data]
"paper_atlas.db" = ["schema.sql", "annotations.sql"] "paper_atlas.db" = ["schema.sql", "annotations.sql"]
"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"] "paper_atlas.resources" = ["venues.json"]
-203
View File
@@ -1,203 +0,0 @@
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
from collections import Counter
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import re
import struct
import tempfile
import unittest
from unittest.mock import patch
import urllib.error
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
class MemorySource:
def __init__(self, raw):
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
def read(self, start, length):
raw = self.raw[start:start+length]
self.bytes_read += len(raw)
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
return raw
def fixture():
import pyarrow as pa
import pyarrow.parquet as pq
rows = []
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
'track': 'main', 'venue_id': 'source-venue-claim',
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
rows.append(row)
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
class RangeHTTPTests(unittest.TestCase):
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
response = io.BytesIO(data); response.status = status
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
return response
def test_strict_range_and_changed_object_detection(self):
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
source = RangeSource('https://example.org/public', 10, opener=opener)
self.assertEqual(source.read(2, 2), b'ab')
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
with self.assertRaisesRegex(ValueError, 'ETag'):
source.read(4, 2)
self.assertEqual(source.bytes_read, 2)
def test_no_full_file_fallback_or_denied_source_retry(self):
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
opener.open.return_value = response
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
response.read.assert_not_called()
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
self.assertEqual(opener.open.call_count, 1)
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
with self.assertRaises(ValueError):
source.seek(-101, 2)
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
class ParquetProjectionTests(unittest.TestCase):
def test_complete_selected_rows_and_no_excluded_page_requests(self):
import pyarrow.parquet as pq
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
self.assertEqual(rows, expected)
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
excluded = []
for i in range(metadata.num_row_groups):
group = metadata.row_group(i)
for j in range(group.num_columns):
col = group.column(j)
if col.path_in_schema.split('.')[0] not in COLUMNS:
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
excluded.append((start, start+col.total_compressed_size))
for request in source.receipts:
a, b = request['start'], request['start']+request['length']
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
self.assertLess(source.bytes_read, len(raw))
encoded = json.dumps(rows)+json.dumps(proof)
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
raw, _ = fixture(); source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'Missing'):
projection(source, {'missing_column': 'string'})
self.assertEqual(len(source.receipts), 3)
source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'type differs'):
projection(source, {**COLUMNS, 'year': 'string'})
self.assertEqual(len(source.receipts), 3)
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
source = MemorySource(bytes(bad))
with self.assertRaisesRegex(ValueError, 'footer'):
projection(source, COLUMNS)
self.assertEqual(len(source.receipts), 2)
def test_collector_output_preserves_claims_without_original_identity(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'projected'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
for row in projected:
self.assertEqual(row['identifiers'], {})
self.assertFalse(row['metadata']['original_forum_id_verified'])
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertNotIn('status', row)
self.assertIn(COMMIT, row['external_id'])
self.assertEqual(projected[2]['authors'], ['Anonymous'])
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
self.assertEqual(projected[0]['track'], 'main')
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
all_saved=''.join(p.read_text() for p in out.iterdir())
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
with self.assertRaisesRegex(ValueError, 'New output'):
collect('emnlp', out)
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
# Mimic the first reviewed source version: new literal spellings were unknown.
for row in before[:2]:
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
with self.assertRaisesRegex(ValueError, 'hash differs'):
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
no_network.assert_not_called()
def test_failure_is_retained_without_source_payload_or_retries(self):
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'failed'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
with self.assertRaises(RuntimeError):
collect('emnlp', out, source_factory=lambda url, size: None)
report = json.loads((out/'failure.json').read_text())
self.assertEqual(report['status'], 'incomplete_do_not_import')
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
class MirrorSemanticsTests(unittest.TestCase):
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
result = {s: inventory(s) for s in splits}
self.assertEqual(sum(len(v) for v in result.values()), 15)
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
self.assertEqual(result['emnlp'][0]['size'], 90794691)
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
with self.assertRaises(ValueError):
inventory('uai')
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
self.assertEqual(decision_claim(literal), 'unknown')
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
self.assertIsNone(decision_track_claim('Accept (Poster)'))