"""Metadata projection never requests OCR/review column pages or promotes mirror IDs.""" from collections import Counter import hashlib import importlib.util import io import json from pathlib import Path import re import struct import tempfile import unittest from unittest.mock import patch import urllib.error from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection class MemorySource: def __init__(self, raw): self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, [] def read(self, start, length): raw = self.raw[start:start+length] self.bytes_read += len(raw) self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()}) return raw def fixture(): import pyarrow as pa import pyarrow.parquet as pq rows = [] for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]): row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023, 'track': 'main', 'venue_id': 'source-venue-claim', 'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i), 'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'], 'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision, 'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000), 'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000), 'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)} rows.append(row) sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy') return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows] class RangeHTTPTests(unittest.TestCase): def response(self, data, status=206, start=2, size=10, etag='"stable"'): response = io.BytesIO(data); response.status = status response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag} return response def test_strict_range_and_changed_object_detection(self): opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')] source = RangeSource('https://example.org/public', 10, opener=opener) self.assertEqual(source.read(2, 2), b'ab') self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3') with self.assertRaisesRegex(ValueError, 'ETag'): source.read(4, 2) self.assertEqual(source.bytes_read, 2) def test_no_full_file_fallback_or_denied_source_retry(self): opener = unittest.mock.Mock(); response = self.response(b'ab', status=200) response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body')) opener.open.return_value = response with self.assertRaisesRegex(ValueError, 'full download forbidden'): RangeSource('https://example.org/public', 10, opener=opener).read(2, 2) response.read.assert_not_called() opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None) with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'): RangeSource('https://example.org/public', 10, opener=opener).read(2, 2) self.assertEqual(opener.open.call_count, 1) def test_sparse_footer_windows_are_synthetic_not_network_reads(self): source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')]) source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote') source.seek(-4, 2); self.assertEqual(source.tell(), 96) with self.assertRaises(ValueError): source.seek(-101, 2) @unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner') class ParquetProjectionTests(unittest.TestCase): def test_complete_selected_rows_and_no_excluded_page_requests(self): import pyarrow.parquet as pq raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS) self.assertEqual(rows, expected) metadata = pq.ParquetFile(io.BytesIO(raw)).metadata excluded = [] for i in range(metadata.num_row_groups): group = metadata.row_group(i) for j in range(group.num_columns): col = group.column(j) if col.path_in_schema.split('.')[0] not in COLUMNS: start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0) excluded.append((start, start+col.total_compressed_size)) for request in source.receipts: a, b = request['start'], request['start']+request['length'] self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded)) self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified']) self.assertLess(source.bytes_read, len(raw)) encoded = json.dumps(rows)+json.dumps(proof) self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded) self.assertNotIn('META_REVIEW_SENTINEL', encoded) def test_wrong_schema_and_footer_fail_before_any_payload_download(self): raw, _ = fixture(); source = MemorySource(raw) with self.assertRaisesRegex(ValueError, 'Missing'): projection(source, {'missing_column': 'string'}) self.assertEqual(len(source.receipts), 3) source = MemorySource(raw) with self.assertRaisesRegex(ValueError, 'type differs'): projection(source, {**COLUMNS, 'year': 'string'}) self.assertEqual(len(source.receipts), 3) bad = bytearray(raw); bad[-8:-4] = struct.pack('