From 7d9c693761c8330970bc8fd6d892745848c21131 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 18 Nov 2026 09:34:10 +0800 Subject: [PATCH] Review mirror publication literals and replay frozen metadata offline --- .gitea/workflows/reviewarena.yml | 23 ++++++- docs/reviewarena-metadata.md | 13 ++++ paper_atlas/collectors/reviewarena.py | 90 ++++++++++++++++++++++++--- tests/test_reviewarena.py | 47 +++++++++++--- 4 files changed, 154 insertions(+), 19 deletions(-) diff --git a/.gitea/workflows/reviewarena.yml b/.gitea/workflows/reviewarena.yml index abf7f79..e85ea3b 100644 --- a/.gitea/workflows/reviewarena.yml +++ b/.gitea/workflows/reviewarena.yml @@ -6,12 +6,22 @@ on: description: 'Pinned public mirror family; metadata columns only' required: true default: emnlp + projection_name: + description: 'Optional existing metadata output directory name for offline replay' + required: false + default: '' + projection_sha256: + description: 'Exact cached collection-report.json SHA256, required for offline replay' + required: false + default: '' jobs: project: runs-on: ces-control env: ARCHIVE_SHA: ${{ github.sha }} MIRROR_SPLIT: ${{ inputs.split }} + MIRROR_PROJECTION: ${{ inputs.projection_name }} + MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }} PYTHONDONTWRITEBYTECODE: '1' steps: - name: Project permitted metadata on the approved remote runner @@ -20,6 +30,8 @@ jobs: import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request sha = os.environ['ARCHIVE_SHA'] split = os.environ['MIRROR_SPLIT'] + cached = os.environ.get('MIRROR_PROJECTION','') + cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','') if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'): raise SystemExit('Explicit approved source commit/split required') base = pathlib.Path('/data/paper-index') @@ -28,6 +40,13 @@ jobs: out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8]) if out.exists(): raise SystemExit('Inspect existing output; no overwrite or automatic retry') + extra=[] + if cached: + if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha): + raise SystemExit('Explicit permitted cached projection name and report hash required') + extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha] + elif cached_sha: + raise SystemExit('Cached projection name required') url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz' with urllib.request.urlopen(url,timeout=60) as response: archive = response.read(24*1024*1024+1) @@ -50,10 +69,10 @@ jobs: wheel,=wheels.glob('paper_atlas-*.whl') subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) outside=root/'outside';outside.mkdir() - subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out)],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True) report=json.loads((out/'collection-report.json').read_bytes()) assert report['database_written'] is False and report['approved_for_import']==0 - receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'raw_parquet_saved':False,'database_written':False} + receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False} (out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n') backup=base/'validation'/(out.name+'-recovery.tar.gz') if backup.exists(): diff --git a/docs/reviewarena-metadata.md b/docs/reviewarena-metadata.md index 72e5e98..9cb6769 100644 --- a/docs/reviewarena-metadata.md +++ b/docs/reviewarena-metadata.md @@ -20,6 +20,17 @@ paper-atlas collect reviewarena-metadata \ 输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。 +已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet: + +```sh +paper-atlas collect reviewarena-metadata --split emnlp \ + --from-projection /data/paper-index/collected/<既有投影>/collection-report.json \ + --projection-sha256 <已验证的原报告SHA256> \ + --out /data/paper-index/collected/<新解释版本> +``` + +手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。 + ## 实际读取和保存范围 仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。 @@ -34,6 +45,8 @@ paper-atlas collect reviewarena-metadata \ `mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。 +真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。 + 上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。 输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。 diff --git a/paper_atlas/collectors/reviewarena.py b/paper_atlas/collectors/reviewarena.py index 66a8209..4fb7189 100644 --- a/paper_atlas/collectors/reviewarena.py +++ b/paper_atlas/collectors/reviewarena.py @@ -56,12 +56,21 @@ def decision_claim(raw): return 'desk_rejected' if value in {'withdraw', 'withdrawn'}: return 'withdrawn' - if value in {'accept', 'accepted', 'accept (main)', 'accept (findings)', + if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)', 'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}: return 'accepted' return 'unknown' +def decision_track_claim(raw): + value = raw.strip().casefold() if isinstance(raw, str) else '' + if value in {'accept-findings', 'accept (findings)'}: + return 'findings' + if value in {'accept-main', 'accept (main)'}: + return 'main' + return None + + def record(row, split, item, row_index): if set(row) != set(COLUMNS): raise ValueError('Unexpected projected metadata fields') @@ -74,7 +83,8 @@ def record(row, split, item, row_index): 'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'], 'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'], 'conference': row['conference'], 'year': row['year'], 'track': row['track'], - 'mirror_decision_category': decision_claim(row['decision']), 'raw_fields': row, + 'mirror_decision_category': decision_claim(row['decision']), + 'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row, 'source_locator': locator, 'identifiers': {}, 'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False, 'original_decision_signature_verified': False, 'source_visibility': 'public_mirror', @@ -83,18 +93,73 @@ def record(row, split, item, row_index): 'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}} -def collect(split, out, *, source_factory=RangeSource, progress=None): +def cached_projections(path, expected_sha256, split, items): + path = Path(path) + raw = path.read_bytes() + if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256) + or hashlib.sha256(raw).hexdigest() != expected_sha256): + raise ValueError('Cached projection report hash differs') + report = json.loads(raw) + if (report.get('dataset') != DATASET or report.get('commit') != COMMIT + or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256 + or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False + or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False + or report.get('approved_for_import') != 0): + raise ValueError('Cached report scope differs from permitted mirror projection') + declared = report.get('files', []) + if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items): + raise ValueError('Incomplete cached mirror shard set') + result = {} + for item in items: + fm = next((f for f in declared if f['source_file'] == item['path']), None) + filename = Path(item['path']).stem+'.metadata.jsonl' + if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']: + raise ValueError('Cached mirror source differs') + file = path.parent/filename + if file.is_symlink(): + raise ValueError('Symlink metadata cache forbidden') + content = file.read_bytes() + if hashlib.sha256(content).hexdigest() != fm['file_sha256']: + raise ValueError('Cached metadata file hash differs') + rows = [] + for index, line in enumerate(content.splitlines()): + previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index) + for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'): + if previous.get(key) != expected[key]: + raise ValueError('Cached raw metadata/identity differs: '+key) + rows.append(row) + proof = fm['projection'] + if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS + or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False + or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False + or proof['full_file_checksum_verified'] is not False): + raise ValueError('Cached projection proof differs') + result[item['path']] = (rows, proof) + if sum(len(rows) for rows, _ in result.values()) != report['records']: + raise ValueError('Cached projection row count differs') + return result + + +def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None): items = inventory(split); out = Path(out) + if bool(from_projection) != bool(projection_sha256): + raise ValueError('Cached projection and exact report hash required together') if out.exists() or not out.parent.is_dir() or out.is_symlink(): raise ValueError('New output directory in existing parent required') out.mkdir() - counts = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = [] + counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = [] observed = datetime.now(timezone.utc).isoformat() try: + cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None for item in items: - source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path'] - source = source_factory(source_url, item['size']) - rows, proof = projection(source, COLUMNS, progress=progress) + if cached is None: + source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path'] + source = source_factory(source_url, item['size']) + rows, proof = projection(source, COLUMNS, progress=progress) + else: + rows, proof = cached[item['path']] + if progress: + progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)}) path = out/(Path(item['path']).stem+'.metadata.jsonl') part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256() with part.open('wb') as stream: @@ -103,6 +168,7 @@ def collect(split, out, *, source_factory=RangeSource, progress=None): raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode() stream.write(raw); h.update(raw); records += 1 counts[item_record['mirror_decision_category']] += 1 + track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1 raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1 years[(row['year'], row['track'])] += 1 part.rename(path) @@ -113,8 +179,11 @@ def collect(split, out, *, source_factory=RangeSource, progress=None): 'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256, 'split': split, 'observed_at': observed, 'records': records, 'files': reports, 'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts), + 'mirror_decision_track_claim_counts': dict(track_claims), 'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))], - 'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports), + 'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0, + 'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports), + 'cached_projection_report_sha256': projection_sha256, 'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False, 'paper_body_saved': False, 'review_text_saved': False, 'database_written': False, 'approved_for_import': 0, @@ -139,6 +208,9 @@ def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog) parser.add_argument('--split', required=True, choices=SPLITS) parser.add_argument('--out', required=True) + parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection') + parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256') args = parser.parse_args(argv) - report = collect(args.split, args.out, progress=lambda row: print(json.dumps(row), flush=True)) + report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256, + progress=lambda row: print(json.dumps(row), flush=True)) print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True) diff --git a/tests/test_reviewarena.py b/tests/test_reviewarena.py index 3d64d6f..0944731 100644 --- a/tests/test_reviewarena.py +++ b/tests/test_reviewarena.py @@ -12,7 +12,7 @@ import unittest from unittest.mock import patch import urllib.error -from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, inventory, record +from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection @@ -31,11 +31,11 @@ def fixture(): import pyarrow as pa import pyarrow.parquet as pq rows = [] - for i, decision in enumerate(['Accept (Findings)', 'Reject', 'Withdrawn', None]): + for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]): row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023, - 'track': 'findings' if i == 0 else 'main', 'venue_id': 'source-venue-claim', + 'track': 'main', 'venue_id': 'source-venue-claim', 'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i), - 'authors': ['Anonymous'] if i == 1 else ['Ada Author', 'Yi Researcher'], + 'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'], 'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision, 'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000), 'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000), @@ -125,8 +125,8 @@ class ParquetProjectionTests(unittest.TestCase): out = Path(temp)/'projected' with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]): report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw)) - self.assertEqual(report['mirror_decision_counts'], {'accepted': 1, 'rejected': 1, 'withdrawn': 1, 'unknown': 1}) - self.assertEqual(report['records'], 4); self.assertFalse(report['database_written']) + self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1}) + self.assertEqual(report['records'], 5); self.assertFalse(report['database_written']) projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()] for row in projected: self.assertEqual(row['identifiers'], {}) @@ -134,13 +134,40 @@ class ParquetProjectionTests(unittest.TestCase): self.assertFalse(row['metadata']['original_decision_signature_verified']) self.assertNotIn('status', row) self.assertIn(COMMIT, row['external_id']) - self.assertEqual(projected[1]['authors'], ['Anonymous']) - self.assertEqual(projected[1]['metadata']['forum_id_claim'], 'unverified-source-1.xml') + self.assertEqual(projected[2]['authors'], ['Anonymous']) + self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml') + self.assertEqual(projected[0]['track'], 'main') + self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings') all_saved=''.join(p.read_text() for p in out.iterdir()) self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved) with self.assertRaisesRegex(ValueError, 'New output'): collect('emnlp', out) + def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self): + raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}} + with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]): + out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw)) + file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()] + # Mimic the first reviewed source version: new literal spellings were unknown. + for row in before[:2]: + row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim') + file.write_text(''.join(json.dumps(x)+'\n' for x in before)) + report_path = out/'collection-report.json'; report = json.loads(report_path.read_text()) + report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest() + report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest() + no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source')) + result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network) + no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0) + self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read']) + self.assertEqual(result['mirror_decision_counts']['accepted'], 2) + self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3}) + new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()] + self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before]) + self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before]) + with self.assertRaisesRegex(ValueError, 'hash differs'): + collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network) + no_network.assert_not_called() + def test_failure_is_retained_without_source_payload_or_retries(self): item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}} with tempfile.TemporaryDirectory() as temp: @@ -170,3 +197,7 @@ class MirrorSemanticsTests(unittest.TestCase): self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected') self.assertEqual(decision_claim('Withdrawn'), 'withdrawn') self.assertEqual(decision_claim('Accept (Findings)'), 'accepted') + self.assertEqual(decision_claim('Accept-Main'), 'accepted') + self.assertEqual(decision_claim('Accept-Findings'), 'accepted') + self.assertEqual(decision_track_claim('Accept-Findings'), 'findings') + self.assertIsNone(decision_track_claim('Accept (Poster)'))