From 90fd789a171ff444126a98f3d8032801feddade9 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 18 Nov 2026 09:10:29 +0800 Subject: [PATCH 1/3] Project pinned ReviewArena metadata without full-text payload pages --- .gitea/workflows/check.yml | 2 +- .gitea/workflows/reviewarena.yml | 69 +++++++ README.md | 2 + docs/reviewarena-metadata.md | 39 ++++ paper_atlas/cli.py | 1 + paper_atlas/collectors/reviewarena.py | 144 ++++++++++++++ paper_atlas/io/parquet_metadata.py | 195 +++++++++++++++++++ paper_atlas/resources/reviewarena-files.json | 1 + pyproject.toml | 3 +- tests/test_reviewarena.py | 172 ++++++++++++++++ 10 files changed, 626 insertions(+), 2 deletions(-) create mode 100644 .gitea/workflows/reviewarena.yml create mode 100644 docs/reviewarena-metadata.md create mode 100644 paper_atlas/collectors/reviewarena.py create mode 100644 paper_atlas/io/parquet_metadata.py create mode 100644 paper_atlas/resources/reviewarena-files.json create mode 100644 tests/test_reviewarena.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 34490c0..e38ad6f 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -35,7 +35,7 @@ jobs: venv = root / 'venv' subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) python = str(venv / 'bin/python') - subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True) + subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==21.0.0'],check=True) testenv = os.environ.copy() service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') if service_file.is_file(): diff --git a/.gitea/workflows/reviewarena.yml b/.gitea/workflows/reviewarena.yml new file mode 100644 index 0000000..ee0a207 --- /dev/null +++ b/.gitea/workflows/reviewarena.yml @@ -0,0 +1,69 @@ +name: ReviewArena metadata projection +on: + workflow_dispatch: + inputs: + split: + description: 'Pinned public mirror family; metadata columns only' + required: true + default: emnlp +jobs: + project: + runs-on: ces-control + env: + ARCHIVE_SHA: ${{ github.sha }} + MIRROR_SPLIT: ${{ inputs.split }} + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Project permitted metadata on the approved remote runner + run: | + python3 -B - <<'PY' + import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request + sha = os.environ['ARCHIVE_SHA'] + split = os.environ['MIRROR_SPLIT'] + if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'): + raise SystemExit('Explicit approved source commit/split required') + base = pathlib.Path('/data/paper-index') + if not (base/'collected').is_dir() or not (base/'validation').is_dir(): + raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host') + out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8]) + if out.exists(): + raise SystemExit('Inspect existing output; no overwrite or automatic retry') + url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz' + with urllib.request.urlopen(url,timeout=60) as response: + archive = response.read(24*1024*1024+1) + if len(archive)>24*1024*1024: + raise SystemExit('Oversized source archive') + with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp: + root = pathlib.Path(tmp) + with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar: + tar.extractall(root,filter='data') + candidates=list(root.glob('*/pyproject.toml')) + if len(candidates)!=1: + raise SystemExit('Unexpected repository archive') + src=candidates[0].parent + venv=root/'venv' + subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) + python=str(venv/'bin/python') + subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==21.0.0'],check=True) + wheels=root/'wheels' + subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True) + wheel,=wheels.glob('paper_atlas-*.whl') + subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) + outside=root/'outside';outside.mkdir() + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out)],cwd=outside,check=True) + report=json.loads((out/'collection-report.json').read_bytes()) + assert report['database_written'] is False and report['approved_for_import']==0 + receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'raw_parquet_saved':False,'database_written':False} + (out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') + backup=base/'validation'/(out.name+'-recovery.tar.gz') + if backup.exists(): + raise SystemExit('Recovery archive already exists; inspect') + with tarfile.open(backup,'w:gz') as tar: + tar.add(out,arcname=str(out.relative_to(base))) + backup.chmod(0o444) + for path in out.iterdir(): + if path.is_file():path.chmod(0o444) + result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt} + (base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n') + print(json.dumps(result),flush=True) + PY diff --git a/README.md b/README.md index 205294f..7ba9b5f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ **长期维护、按需运行的论文元数据与公开投稿历史工具。** +其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。 + Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 diff --git a/docs/reviewarena-metadata.md b/docs/reviewarena-metadata.md new file mode 100644 index 0000000..4874613 --- /dev/null +++ b/docs/reviewarena-metadata.md @@ -0,0 +1,39 @@ +# ReviewArena 镜像的索引字段投影 + +ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。 + +- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md) +- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false) +- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/21.0/python/generated/pyarrow.parquet.ParquetFile.html) + +## 运行 + +仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 21.0.0 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。 + +包内命令如下,`--out` 必须是既有父目录内的新目录: + +```sh +paper-atlas collect reviewarena-metadata \ + --split emnlp \ + --out /data/paper-index/collected/reviewarena-emnlp-reviewed-version +``` + +输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。 + +## 实际读取和保存范围 + +仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。 + +程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。 + +不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。 + +上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。 + +## 决定和身份边界 + +`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。 + +上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。 + +输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index e4f8590..e2e6a48 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -20,6 +20,7 @@ COMMANDS = { ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'), ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'), + ('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'), ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), diff --git a/paper_atlas/collectors/reviewarena.py b/paper_atlas/collectors/reviewarena.py new file mode 100644 index 0000000..66a8209 --- /dev/null +++ b/paper_atlas/collectors/reviewarena.py @@ -0,0 +1,144 @@ +"""Source-attributed mirror metadata; no paper/review text or identity promotion.""" +import argparse +from collections import Counter +from datetime import datetime, timezone +import hashlib +from importlib.resources import files +import json +from pathlib import Path +import re + +from paper_atlas.io.parquet_metadata import RangeSource, projection + +DATASET = 'anonymousNeurIPS2026submission4281/reviewarena' +COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b' +TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20' +SOURCE = 'reviewarena_mirror' +SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr') +COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer', + 'track': 'string', 'venue_id': 'string', 'paper_number': 'integer', + 'title': 'string', 'abstract': 'string', 'authors': 'strings', + 'keywords': 'strings', 'venue': 'string', 'decision': 'string'} + + +def inventory(split): + if split not in SPLITS: + raise ValueError('Explicit supported mirror split required') + raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes() + if hashlib.sha256(raw).hexdigest() != TREE_SHA256: + raise ValueError('Pinned mirror file inventory changed') + items = json.loads(raw) + if len(items) != 15: + raise ValueError('Pinned mirror shard inventory changed') + selected = [] + for item in items: + path = item['path'] + if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path): + raise ValueError('Unexpected mirror shard path') + if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12 + or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])): + raise ValueError('Missing source object size/checksum claim') + if path.startswith('data/'+split+'-'): + selected.append(item) + parts = sorted(selected, key=lambda x: x['path']) + if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts): + raise ValueError('Incomplete selected mirror split inventory') + if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))): + raise ValueError('Mirror shard sequence changed') + return parts + + +def decision_claim(raw): + value = raw.strip().casefold() if isinstance(raw, str) else '' + if value in {'reject', 'rejected'}: + return 'rejected' + if value in {'desk reject', 'desk rejected', 'desk_rejected'}: + return 'desk_rejected' + if value in {'withdraw', 'withdrawn'}: + return 'withdrawn' + if value in {'accept', 'accepted', 'accept (main)', 'accept (findings)', + 'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}: + return 'accepted' + return 'unknown' + + +def record(row, split, item, row_index): + if set(row) != set(COLUMNS): + raise ValueError('Unexpected projected metadata fields') + if (not isinstance(row['conference'], str) or row['conference'].casefold() != split + or type(row['year']) is not int or not 2020 <= row['year'] <= 2026): + raise ValueError('Mirror conference/year outside pinned source scope') + locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index, + 'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']} + return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index), + 'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'], + 'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'], + 'conference': row['conference'], 'year': row['year'], 'track': row['track'], + 'mirror_decision_category': decision_claim(row['decision']), 'raw_fields': row, + 'source_locator': locator, 'identifiers': {}, + 'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False, + 'original_decision_signature_verified': False, 'source_visibility': 'public_mirror', + 'scientific_work_identity_approved': False, 'research_type_classified': False, + 'approved_for_import': False, + 'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}} + + +def collect(split, out, *, source_factory=RangeSource, progress=None): + items = inventory(split); out = Path(out) + if out.exists() or not out.parent.is_dir() or out.is_symlink(): + raise ValueError('New output directory in existing parent required') + out.mkdir() + counts = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = [] + observed = datetime.now(timezone.utc).isoformat() + try: + for item in items: + source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path'] + source = source_factory(source_url, item['size']) + rows, proof = projection(source, COLUMNS, progress=progress) + path = out/(Path(item['path']).stem+'.metadata.jsonl') + part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256() + with part.open('wb') as stream: + for index, row in enumerate(rows): + item_record = record(row, split, item, index) + raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode() + stream.write(raw); h.update(raw); records += 1 + counts[item_record['mirror_decision_category']] += 1 + raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1 + years[(row['year'], row['track'])] += 1 + part.rename(path) + reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows), + 'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'], + 'projection': proof}) + report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity', + 'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256, + 'split': split, 'observed_at': observed, 'records': records, 'files': reports, + 'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts), + 'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))], + 'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports), + 'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False, + 'paper_body_saved': False, 'review_text_saved': False, 'database_written': False, + 'approved_for_import': 0, + 'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.', + 'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.', + 'Every selected row and row locator is retained, including missing or anonymous author data.', + 'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.', + 'Rows are not independent scientific works or a complete venue/year public negative census.', + 'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']} + (out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') + return report + except BaseException as error: + # No source-specific body or signed redirect URLs in failure reports. + failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT, + 'completed_files': reports, 'error_type': type(error).__name__, + 'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True} + (out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n') + raise + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog) + parser.add_argument('--split', required=True, choices=SPLITS) + parser.add_argument('--out', required=True) + args = parser.parse_args(argv) + report = collect(args.split, args.out, progress=lambda row: print(json.dumps(row), flush=True)) + print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True) diff --git a/paper_atlas/io/parquet_metadata.py b/paper_atlas/io/parquet_metadata.py new file mode 100644 index 0000000..108ad33 --- /dev/null +++ b/paper_atlas/io/parquet_metadata.py @@ -0,0 +1,195 @@ +"""Read only selected Parquet column chunks; never fetch other payload pages.""" +import hashlib +import io +import re +import struct +import urllib.error +import urllib.request + +MAX_FOOTER = 4 * 1024**2 +MAX_SELECTED_BYTES = 64 * 1024**2 +MAX_REQUESTS = 512 + + +class RangeSource: + """Strict bounded HTTP ranges, no full-file fallback, redirects kept private.""" + def __init__(self, url, size, *, opener=None): + self.url, self.size = url, size + self.opener = opener or urllib.request.build_opener() + self.etag = None + self.receipts = [] + self.bytes_read = 0 + + def read(self, start, length): + if (type(start) is not int or type(length) is not int or length <= 0 + or start < 0 or start + length > self.size + or self.bytes_read + length > MAX_SELECTED_BYTES + or len(self.receipts) >= MAX_REQUESTS): + raise ValueError('Bounded metadata range budget exceeded') + headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'} + if self.etag and not self.etag.startswith('W/'): + headers['If-Match'] = self.etag + request = urllib.request.Request(self.url, headers=headers) + try: + with self.opener.open(request, timeout=45) as response: + if response.status != 206: + raise ValueError('Server did not honor metadata range; full download forbidden') + if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}': + raise ValueError('Object size or range differs from pinned inventory') + encoding = response.headers.get('Content-Encoding', 'identity') + if encoding != 'identity': + raise ValueError('Encoded HTTP ranges are unsupported') + etag = response.headers.get('ETag') + if self.receipts and etag != self.etag: + raise ValueError('Object ETag changed between metadata ranges') + raw = response.read(length + 1) + if len(raw) != length: + raise ValueError('Truncated or oversized metadata range') + if not self.receipts: + self.etag = etag + except urllib.error.HTTPError as error: + # Do not expose signed CDN redirect URLs or retry denied resources. + raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None + self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()}) + self.bytes_read += length + return raw + + +class SparseFile(io.RawIOBase): + """A logical file backed only by approved bytes; all other positions are zeros. + + Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids + fetching the preceding OCR/review page. Data pages are explicitly selected + from footer offsets before they can be downloaded. + """ + def __init__(self, size, segments): + self.size, self.segments, self.position = size, segments, 0 + + def readable(self): + return True + + def seekable(self): + return True + + def tell(self): + return self.position + + def seek(self, offset, whence=0): + base = {0: 0, 1: self.position, 2: self.size}.get(whence) + if base is None or base + offset < 0: + raise ValueError('Invalid sparse metadata seek') + self.position = base + offset + return self.position + + def read(self, length=-1): + if length < 0: + length = self.size - self.position + length = min(length, max(0, self.size - self.position)) + if length > MAX_SELECTED_BYTES: + raise ValueError('Unbounded sparse read forbidden') + start, end = self.position, self.position + length + raw = bytearray(length) + for offset, data in self.segments: + left, right = max(offset, start), min(offset + len(data), end) + if left < right: + raw[left-start:right-start] = data[left-offset:right-offset] + self.position = end + return bytes(raw) + + def readinto(self, buffer): + raw = self.read(len(buffer)); buffer[:len(raw)] = raw + return len(raw) + + +def arrow(): + try: + import pyarrow as pa + import pyarrow.parquet as pq + except ImportError: + raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None + return pa, pq + + +def projection(source, columns, *, progress=None): + """Return complete metadata rows and range receipts, no full Parquet snapshot.""" + pa, pq = arrow() + if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3: + raise ValueError('Bounded pinned Parquet size required') + if (not isinstance(columns, dict) or not columns + or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)): + raise ValueError('Explicit top-level metadata columns required') + header = source.read(0, 4); trailer = source.read(source.size - 8, 8) + if header != b'PAR1' or trailer[4:] != b'PAR1': + raise ValueError('Expected unencrypted Parquet framing') + length = struct.unpack(' 1000: + raise ValueError('Bounded metadata row count required') + wanted, excluded = [], [] + for group in range(metadata.num_row_groups): + rg = metadata.row_group(group) + for i in range(rg.num_columns): + col = rg.column(i) + if col.file_path: + raise ValueError('External Parquet column files forbidden') + offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0] + if not offsets: + raise ValueError('Missing column chunk offset') + start = min(offsets); end = start + col.total_compressed_size + if not 4 <= start < end <= footer_start: + raise ValueError('Column chunk lies outside the pinned payload') + item = (start, end, group, col.path_in_schema) + (wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item) + if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded): + raise ValueError('Selected range overlaps an excluded payload column') + if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES: + raise ValueError('Metadata projection exceeds byte budget') + if len(wanted) + 3 > MAX_REQUESTS: + raise ValueError('Metadata projection exceeds request budget') + for start, end, group, path in sorted(wanted): + segments.append((start, source.read(start, end-start))) + if progress: + progress({'phase': 'metadata_column_range', 'row_group': group, + 'column': path, 'metadata_bytes_read': source.bytes_read}) + rows = [] + for group in range(metadata.num_row_groups): + table = parquet.read_row_group(group, columns=list(columns), use_threads=False) + if set(table.column_names) != set(columns) or len(table.column_names) != len(columns): + raise ValueError('Unexpected projected metadata fields') + rows.extend(table.select(list(columns)).to_pylist()) + if len(rows) != metadata.num_rows: + raise ValueError('Incomplete metadata projection') + report = {'rows': len(rows), 'row_groups': metadata.num_row_groups, + 'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(), + 'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted], + 'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read, + 'source_bytes': source.size, 'full_file_checksum_verified': False, + 'excluded_payload_pages_downloaded': False, + 'footer_saved': False, 'raw_parquet_saved': False} + return rows, report + finally: + parquet.close() diff --git a/paper_atlas/resources/reviewarena-files.json b/paper_atlas/resources/reviewarena-files.json new file mode 100644 index 0000000..a9b8c08 --- /dev/null +++ b/paper_atlas/resources/reviewarena-files.json @@ -0,0 +1 @@ +[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}] \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9655ba8..4972229 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ dependencies = ["PyYAML==6.0.3"] [project.optional-dependencies] postgres = ["psycopg[binary]==3.3.8"] +mirrors = ["pyarrow==21.0.0"] [project.scripts] paper-atlas = "paper_atlas.cli:main" @@ -22,4 +23,4 @@ exclude = ["tests*"] [tool.setuptools.package-data] "paper_atlas.db" = ["schema.sql", "annotations.sql"] -"paper_atlas.resources" = ["venues.json"] +"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"] diff --git a/tests/test_reviewarena.py b/tests/test_reviewarena.py new file mode 100644 index 0000000..3d64d6f --- /dev/null +++ b/tests/test_reviewarena.py @@ -0,0 +1,172 @@ +"""Metadata projection never requests OCR/review column pages or promotes mirror IDs.""" +from collections import Counter +import hashlib +import importlib.util +import io +import json +from pathlib import Path +import re +import struct +import tempfile +import unittest +from unittest.mock import patch +import urllib.error + +from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, inventory, record +from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection + + +class MemorySource: + def __init__(self, raw): + self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, [] + + def read(self, start, length): + raw = self.raw[start:start+length] + self.bytes_read += len(raw) + self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()}) + return raw + + +def fixture(): + import pyarrow as pa + import pyarrow.parquet as pq + rows = [] + for i, decision in enumerate(['Accept (Findings)', 'Reject', 'Withdrawn', None]): + row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023, + 'track': 'findings' if i == 0 else 'main', 'venue_id': 'source-venue-claim', + 'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i), + 'authors': ['Anonymous'] if i == 1 else ['Ada Author', 'Yi Researcher'], + 'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision, + 'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000), + 'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000), + 'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)} + rows.append(row) + sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy') + return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows] + + +class RangeHTTPTests(unittest.TestCase): + def response(self, data, status=206, start=2, size=10, etag='"stable"'): + response = io.BytesIO(data); response.status = status + response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag} + return response + + def test_strict_range_and_changed_object_detection(self): + opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')] + source = RangeSource('https://example.org/public', 10, opener=opener) + self.assertEqual(source.read(2, 2), b'ab') + self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3') + with self.assertRaisesRegex(ValueError, 'ETag'): + source.read(4, 2) + self.assertEqual(source.bytes_read, 2) + + def test_no_full_file_fallback_or_denied_source_retry(self): + opener = unittest.mock.Mock(); response = self.response(b'ab', status=200) + response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body')) + opener.open.return_value = response + with self.assertRaisesRegex(ValueError, 'full download forbidden'): + RangeSource('https://example.org/public', 10, opener=opener).read(2, 2) + response.read.assert_not_called() + opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None) + with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'): + RangeSource('https://example.org/public', 10, opener=opener).read(2, 2) + self.assertEqual(opener.open.call_count, 1) + + def test_sparse_footer_windows_are_synthetic_not_network_reads(self): + source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')]) + source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote') + source.seek(-4, 2); self.assertEqual(source.tell(), 96) + with self.assertRaises(ValueError): + source.seek(-101, 2) + + +@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner') +class ParquetProjectionTests(unittest.TestCase): + def test_complete_selected_rows_and_no_excluded_page_requests(self): + import pyarrow.parquet as pq + raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS) + self.assertEqual(rows, expected) + metadata = pq.ParquetFile(io.BytesIO(raw)).metadata + excluded = [] + for i in range(metadata.num_row_groups): + group = metadata.row_group(i) + for j in range(group.num_columns): + col = group.column(j) + if col.path_in_schema.split('.')[0] not in COLUMNS: + start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0) + excluded.append((start, start+col.total_compressed_size)) + for request in source.receipts: + a, b = request['start'], request['start']+request['length'] + self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded)) + self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified']) + self.assertLess(source.bytes_read, len(raw)) + encoded = json.dumps(rows)+json.dumps(proof) + self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded) + self.assertNotIn('META_REVIEW_SENTINEL', encoded) + + def test_wrong_schema_and_footer_fail_before_any_payload_download(self): + raw, _ = fixture(); source = MemorySource(raw) + with self.assertRaisesRegex(ValueError, 'Missing'): + projection(source, {'missing_column': 'string'}) + self.assertEqual(len(source.receipts), 3) + source = MemorySource(raw) + with self.assertRaisesRegex(ValueError, 'type differs'): + projection(source, {**COLUMNS, 'year': 'string'}) + self.assertEqual(len(source.receipts), 3) + bad = bytearray(raw); bad[-8:-4] = struct.pack(' Date: Wed, 18 Nov 2026 09:14:27 +0800 Subject: [PATCH 2/3] Use Arrow binary wheels compatible with runner Python 3.14 --- .gitea/workflows/check.yml | 2 +- .gitea/workflows/reviewarena.yml | 2 +- docs/reviewarena-metadata.md | 4 ++-- pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index e38ad6f..d59cf09 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -35,7 +35,7 @@ jobs: venv = root / 'venv' subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) python = str(venv / 'bin/python') - subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==21.0.0'],check=True) + subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True) testenv = os.environ.copy() service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') if service_file.is_file(): diff --git a/.gitea/workflows/reviewarena.yml b/.gitea/workflows/reviewarena.yml index ee0a207..abf7f79 100644 --- a/.gitea/workflows/reviewarena.yml +++ b/.gitea/workflows/reviewarena.yml @@ -44,7 +44,7 @@ jobs: venv=root/'venv' subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) python=str(venv/'bin/python') - subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==21.0.0'],check=True) + subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True) wheels=root/'wheels' subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True) wheel,=wheels.glob('paper_atlas-*.whl') diff --git a/docs/reviewarena-metadata.md b/docs/reviewarena-metadata.md index 4874613..72e5e98 100644 --- a/docs/reviewarena-metadata.md +++ b/docs/reviewarena-metadata.md @@ -4,11 +4,11 @@ ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc259 - [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md) - [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false) -- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/21.0/python/generated/pyarrow.parquet.ParquetFile.html) +- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html) ## 运行 -仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 21.0.0 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。 +仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。 包内命令如下,`--out` 必须是既有父目录内的新目录: diff --git a/pyproject.toml b/pyproject.toml index 4972229..f9213ae 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ dependencies = ["PyYAML==6.0.3"] [project.optional-dependencies] postgres = ["psycopg[binary]==3.3.8"] -mirrors = ["pyarrow==21.0.0"] +mirrors = ["pyarrow==26.0.1"] [project.scripts] paper-atlas = "paper_atlas.cli:main" -- 2.52.0 From 7d9c693761c8330970bc8fd6d892745848c21131 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 18 Nov 2026 09:34:10 +0800 Subject: [PATCH 3/3] Review mirror publication literals and replay frozen metadata offline --- .gitea/workflows/reviewarena.yml | 23 ++++++- docs/reviewarena-metadata.md | 13 ++++ paper_atlas/collectors/reviewarena.py | 90 ++++++++++++++++++++++++--- tests/test_reviewarena.py | 47 +++++++++++--- 4 files changed, 154 insertions(+), 19 deletions(-) diff --git a/.gitea/workflows/reviewarena.yml b/.gitea/workflows/reviewarena.yml index abf7f79..e85ea3b 100644 --- a/.gitea/workflows/reviewarena.yml +++ b/.gitea/workflows/reviewarena.yml @@ -6,12 +6,22 @@ on: description: 'Pinned public mirror family; metadata columns only' required: true default: emnlp + projection_name: + description: 'Optional existing metadata output directory name for offline replay' + required: false + default: '' + projection_sha256: + description: 'Exact cached collection-report.json SHA256, required for offline replay' + required: false + default: '' jobs: project: runs-on: ces-control env: ARCHIVE_SHA: ${{ github.sha }} MIRROR_SPLIT: ${{ inputs.split }} + MIRROR_PROJECTION: ${{ inputs.projection_name }} + MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }} PYTHONDONTWRITEBYTECODE: '1' steps: - name: Project permitted metadata on the approved remote runner @@ -20,6 +30,8 @@ jobs: import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request sha = os.environ['ARCHIVE_SHA'] split = os.environ['MIRROR_SPLIT'] + cached = os.environ.get('MIRROR_PROJECTION','') + cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','') if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'): raise SystemExit('Explicit approved source commit/split required') base = pathlib.Path('/data/paper-index') @@ -28,6 +40,13 @@ jobs: out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8]) if out.exists(): raise SystemExit('Inspect existing output; no overwrite or automatic retry') + extra=[] + if cached: + if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha): + raise SystemExit('Explicit permitted cached projection name and report hash required') + extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha] + elif cached_sha: + raise SystemExit('Cached projection name required') url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz' with urllib.request.urlopen(url,timeout=60) as response: archive = response.read(24*1024*1024+1) @@ -50,10 +69,10 @@ jobs: wheel,=wheels.glob('paper_atlas-*.whl') subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) outside=root/'outside';outside.mkdir() - subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out)],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True) report=json.loads((out/'collection-report.json').read_bytes()) assert report['database_written'] is False and report['approved_for_import']==0 - receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'raw_parquet_saved':False,'database_written':False} + receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False} (out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') backup=base/'validation'/(out.name+'-recovery.tar.gz') if backup.exists(): diff --git a/docs/reviewarena-metadata.md b/docs/reviewarena-metadata.md index 72e5e98..9cb6769 100644 --- a/docs/reviewarena-metadata.md +++ b/docs/reviewarena-metadata.md @@ -20,6 +20,17 @@ paper-atlas collect reviewarena-metadata \ 输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。 +已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet: + +```sh +paper-atlas collect reviewarena-metadata --split emnlp \ + --from-projection /data/paper-index/collected/<既有投影>/collection-report.json \ + --projection-sha256 <已验证的原报告SHA256> \ + --out /data/paper-index/collected/<新解释版本> +``` + +手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。 + ## 实际读取和保存范围 仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。 @@ -34,6 +45,8 @@ paper-atlas collect reviewarena-metadata \ `mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。 +真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。 + 上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。 输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。 diff --git a/paper_atlas/collectors/reviewarena.py b/paper_atlas/collectors/reviewarena.py index 66a8209..4fb7189 100644 --- a/paper_atlas/collectors/reviewarena.py +++ b/paper_atlas/collectors/reviewarena.py @@ -56,12 +56,21 @@ def decision_claim(raw): return 'desk_rejected' if value in {'withdraw', 'withdrawn'}: return 'withdrawn' - if value in {'accept', 'accepted', 'accept (main)', 'accept (findings)', + if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)', 'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}: return 'accepted' return 'unknown' +def decision_track_claim(raw): + value = raw.strip().casefold() if isinstance(raw, str) else '' + if value in {'accept-findings', 'accept (findings)'}: + return 'findings' + if value in {'accept-main', 'accept (main)'}: + return 'main' + return None + + def record(row, split, item, row_index): if set(row) != set(COLUMNS): raise ValueError('Unexpected projected metadata fields') @@ -74,7 +83,8 @@ def record(row, split, item, row_index): 'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'], 'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'], 'conference': row['conference'], 'year': row['year'], 'track': row['track'], - 'mirror_decision_category': decision_claim(row['decision']), 'raw_fields': row, + 'mirror_decision_category': decision_claim(row['decision']), + 'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row, 'source_locator': locator, 'identifiers': {}, 'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False, 'original_decision_signature_verified': False, 'source_visibility': 'public_mirror', @@ -83,18 +93,73 @@ def record(row, split, item, row_index): 'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}} -def collect(split, out, *, source_factory=RangeSource, progress=None): +def cached_projections(path, expected_sha256, split, items): + path = Path(path) + raw = path.read_bytes() + if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256) + or hashlib.sha256(raw).hexdigest() != expected_sha256): + raise ValueError('Cached projection report hash differs') + report = json.loads(raw) + if (report.get('dataset') != DATASET or report.get('commit') != COMMIT + or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256 + or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False + or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False + or report.get('approved_for_import') != 0): + raise ValueError('Cached report scope differs from permitted mirror projection') + declared = report.get('files', []) + if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items): + raise ValueError('Incomplete cached mirror shard set') + result = {} + for item in items: + fm = next((f for f in declared if f['source_file'] == item['path']), None) + filename = Path(item['path']).stem+'.metadata.jsonl' + if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']: + raise ValueError('Cached mirror source differs') + file = path.parent/filename + if file.is_symlink(): + raise ValueError('Symlink metadata cache forbidden') + content = file.read_bytes() + if hashlib.sha256(content).hexdigest() != fm['file_sha256']: + raise ValueError('Cached metadata file hash differs') + rows = [] + for index, line in enumerate(content.splitlines()): + previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index) + for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'): + if previous.get(key) != expected[key]: + raise ValueError('Cached raw metadata/identity differs: '+key) + rows.append(row) + proof = fm['projection'] + if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS + or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False + or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False + or proof['full_file_checksum_verified'] is not False): + raise ValueError('Cached projection proof differs') + result[item['path']] = (rows, proof) + if sum(len(rows) for rows, _ in result.values()) != report['records']: + raise ValueError('Cached projection row count differs') + return result + + +def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None): items = inventory(split); out = Path(out) + if bool(from_projection) != bool(projection_sha256): + raise ValueError('Cached projection and exact report hash required together') if out.exists() or not out.parent.is_dir() or out.is_symlink(): raise ValueError('New output directory in existing parent required') out.mkdir() - counts = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = [] + counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = [] observed = datetime.now(timezone.utc).isoformat() try: + cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None for item in items: - source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path'] - source = source_factory(source_url, item['size']) - rows, proof = projection(source, COLUMNS, progress=progress) + if cached is None: + source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path'] + source = source_factory(source_url, item['size']) + rows, proof = projection(source, COLUMNS, progress=progress) + else: + rows, proof = cached[item['path']] + if progress: + progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)}) path = out/(Path(item['path']).stem+'.metadata.jsonl') part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256() with part.open('wb') as stream: @@ -103,6 +168,7 @@ def collect(split, out, *, source_factory=RangeSource, progress=None): raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode() stream.write(raw); h.update(raw); records += 1 counts[item_record['mirror_decision_category']] += 1 + track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1 raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1 years[(row['year'], row['track'])] += 1 part.rename(path) @@ -113,8 +179,11 @@ def collect(split, out, *, source_factory=RangeSource, progress=None): 'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256, 'split': split, 'observed_at': observed, 'records': records, 'files': reports, 'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts), + 'mirror_decision_track_claim_counts': dict(track_claims), 'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))], - 'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports), + 'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0, + 'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports), + 'cached_projection_report_sha256': projection_sha256, 'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False, 'paper_body_saved': False, 'review_text_saved': False, 'database_written': False, 'approved_for_import': 0, @@ -139,6 +208,9 @@ def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--split', required=True, choices=SPLITS) parser.add_argument('--out', required=True) + parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection') + parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256') args = parser.parse_args(argv) - report = collect(args.split, args.out, progress=lambda row: print(json.dumps(row), flush=True)) + report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256, + progress=lambda row: print(json.dumps(row), flush=True)) print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True) diff --git a/tests/test_reviewarena.py b/tests/test_reviewarena.py index 3d64d6f..0944731 100644 --- a/tests/test_reviewarena.py +++ b/tests/test_reviewarena.py @@ -12,7 +12,7 @@ import unittest from unittest.mock import patch import urllib.error -from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, inventory, record +from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection @@ -31,11 +31,11 @@ def fixture(): import pyarrow as pa import pyarrow.parquet as pq rows = [] - for i, decision in enumerate(['Accept (Findings)', 'Reject', 'Withdrawn', None]): + for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]): row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023, - 'track': 'findings' if i == 0 else 'main', 'venue_id': 'source-venue-claim', + 'track': 'main', 'venue_id': 'source-venue-claim', 'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i), - 'authors': ['Anonymous'] if i == 1 else ['Ada Author', 'Yi Researcher'], + 'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'], 'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision, 'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000), 'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000), @@ -125,8 +125,8 @@ class ParquetProjectionTests(unittest.TestCase): out = Path(temp)/'projected' with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]): report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw)) - self.assertEqual(report['mirror_decision_counts'], {'accepted': 1, 'rejected': 1, 'withdrawn': 1, 'unknown': 1}) - self.assertEqual(report['records'], 4); self.assertFalse(report['database_written']) + self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1}) + self.assertEqual(report['records'], 5); self.assertFalse(report['database_written']) projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()] for row in projected: self.assertEqual(row['identifiers'], {}) @@ -134,13 +134,40 @@ class ParquetProjectionTests(unittest.TestCase): self.assertFalse(row['metadata']['original_decision_signature_verified']) self.assertNotIn('status', row) self.assertIn(COMMIT, row['external_id']) - self.assertEqual(projected[1]['authors'], ['Anonymous']) - self.assertEqual(projected[1]['metadata']['forum_id_claim'], 'unverified-source-1.xml') + self.assertEqual(projected[2]['authors'], ['Anonymous']) + self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml') + self.assertEqual(projected[0]['track'], 'main') + self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings') all_saved=''.join(p.read_text() for p in out.iterdir()) self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved) with self.assertRaisesRegex(ValueError, 'New output'): collect('emnlp', out) + def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self): + raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}} + with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]): + out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw)) + file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()] + # Mimic the first reviewed source version: new literal spellings were unknown. + for row in before[:2]: + row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim') + file.write_text(''.join(json.dumps(x)+'\n' for x in before)) + report_path = out/'collection-report.json'; report = json.loads(report_path.read_text()) + report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest() + report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest() + no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source')) + result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network) + no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0) + self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read']) + self.assertEqual(result['mirror_decision_counts']['accepted'], 2) + self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3}) + new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()] + self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before]) + self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before]) + with self.assertRaisesRegex(ValueError, 'hash differs'): + collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network) + no_network.assert_not_called() + def test_failure_is_retained_without_source_payload_or_retries(self): item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}} with tempfile.TemporaryDirectory() as temp: @@ -170,3 +197,7 @@ class MirrorSemanticsTests(unittest.TestCase): self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected') self.assertEqual(decision_claim('Withdrawn'), 'withdrawn') self.assertEqual(decision_claim('Accept (Findings)'), 'accepted') + self.assertEqual(decision_claim('Accept-Main'), 'accepted') + self.assertEqual(decision_claim('Accept-Findings'), 'accepted') + self.assertEqual(decision_track_claim('Accept-Findings'), 'findings') + self.assertIsNone(decision_track_claim('Accept (Poster)')) -- 2.52.0