204 lines
13 KiB
Python
204 lines
13 KiB
Python
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
|
|
from collections import Counter
|
|
import hashlib
|
|
import importlib.util
|
|
import io
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import struct
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
import urllib.error
|
|
|
|
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
|
|
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
|
|
|
|
|
|
class MemorySource:
|
|
def __init__(self, raw):
|
|
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
|
|
|
|
def read(self, start, length):
|
|
raw = self.raw[start:start+length]
|
|
self.bytes_read += len(raw)
|
|
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
|
return raw
|
|
|
|
|
|
def fixture():
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
rows = []
|
|
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
|
|
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
|
|
'track': 'main', 'venue_id': 'source-venue-claim',
|
|
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
|
|
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
|
|
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
|
|
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
|
|
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
|
|
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
|
|
rows.append(row)
|
|
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
|
|
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
|
|
|
|
|
|
class RangeHTTPTests(unittest.TestCase):
|
|
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
|
|
response = io.BytesIO(data); response.status = status
|
|
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
|
|
return response
|
|
|
|
def test_strict_range_and_changed_object_detection(self):
|
|
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
|
|
source = RangeSource('https://example.org/public', 10, opener=opener)
|
|
self.assertEqual(source.read(2, 2), b'ab')
|
|
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
|
|
with self.assertRaisesRegex(ValueError, 'ETag'):
|
|
source.read(4, 2)
|
|
self.assertEqual(source.bytes_read, 2)
|
|
|
|
def test_no_full_file_fallback_or_denied_source_retry(self):
|
|
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
|
|
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
|
|
opener.open.return_value = response
|
|
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
|
|
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
|
response.read.assert_not_called()
|
|
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
|
|
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
|
|
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
|
self.assertEqual(opener.open.call_count, 1)
|
|
|
|
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
|
|
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
|
|
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
|
|
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
|
|
with self.assertRaises(ValueError):
|
|
source.seek(-101, 2)
|
|
|
|
|
|
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
|
|
class ParquetProjectionTests(unittest.TestCase):
|
|
def test_complete_selected_rows_and_no_excluded_page_requests(self):
|
|
import pyarrow.parquet as pq
|
|
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
|
|
self.assertEqual(rows, expected)
|
|
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
|
|
excluded = []
|
|
for i in range(metadata.num_row_groups):
|
|
group = metadata.row_group(i)
|
|
for j in range(group.num_columns):
|
|
col = group.column(j)
|
|
if col.path_in_schema.split('.')[0] not in COLUMNS:
|
|
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
|
|
excluded.append((start, start+col.total_compressed_size))
|
|
for request in source.receipts:
|
|
a, b = request['start'], request['start']+request['length']
|
|
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
|
|
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
|
|
self.assertLess(source.bytes_read, len(raw))
|
|
encoded = json.dumps(rows)+json.dumps(proof)
|
|
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
|
|
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
|
|
|
|
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
|
|
raw, _ = fixture(); source = MemorySource(raw)
|
|
with self.assertRaisesRegex(ValueError, 'Missing'):
|
|
projection(source, {'missing_column': 'string'})
|
|
self.assertEqual(len(source.receipts), 3)
|
|
source = MemorySource(raw)
|
|
with self.assertRaisesRegex(ValueError, 'type differs'):
|
|
projection(source, {**COLUMNS, 'year': 'string'})
|
|
self.assertEqual(len(source.receipts), 3)
|
|
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
|
|
source = MemorySource(bytes(bad))
|
|
with self.assertRaisesRegex(ValueError, 'footer'):
|
|
projection(source, COLUMNS)
|
|
self.assertEqual(len(source.receipts), 2)
|
|
|
|
def test_collector_output_preserves_claims_without_original_identity(self):
|
|
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
out = Path(temp)/'projected'
|
|
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
|
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
|
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
|
|
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
|
|
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
|
|
for row in projected:
|
|
self.assertEqual(row['identifiers'], {})
|
|
self.assertFalse(row['metadata']['original_forum_id_verified'])
|
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
|
self.assertNotIn('status', row)
|
|
self.assertIn(COMMIT, row['external_id'])
|
|
self.assertEqual(projected[2]['authors'], ['Anonymous'])
|
|
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
|
|
self.assertEqual(projected[0]['track'], 'main')
|
|
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
|
|
all_saved=''.join(p.read_text() for p in out.iterdir())
|
|
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
|
|
with self.assertRaisesRegex(ValueError, 'New output'):
|
|
collect('emnlp', out)
|
|
|
|
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
|
|
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
|
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
|
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
|
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
|
|
# Mimic the first reviewed source version: new literal spellings were unknown.
|
|
for row in before[:2]:
|
|
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
|
|
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
|
|
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
|
|
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
|
|
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
|
|
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
|
|
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
|
|
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
|
|
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
|
|
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
|
|
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
|
|
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
|
|
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
|
|
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
|
|
with self.assertRaisesRegex(ValueError, 'hash differs'):
|
|
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
|
|
no_network.assert_not_called()
|
|
|
|
def test_failure_is_retained_without_source_payload_or_retries(self):
|
|
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
|
|
with tempfile.TemporaryDirectory() as temp:
|
|
out = Path(temp)/'failed'
|
|
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
|
|
with self.assertRaises(RuntimeError):
|
|
collect('emnlp', out, source_factory=lambda url, size: None)
|
|
report = json.loads((out/'failure.json').read_text())
|
|
self.assertEqual(report['status'], 'incomplete_do_not_import')
|
|
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
|
|
|
|
|
|
class MirrorSemanticsTests(unittest.TestCase):
|
|
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
|
|
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
|
|
result = {s: inventory(s) for s in splits}
|
|
self.assertEqual(sum(len(v) for v in result.values()), 15)
|
|
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
|
|
self.assertEqual(result['emnlp'][0]['size'], 90794691)
|
|
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
|
|
with self.assertRaises(ValueError):
|
|
inventory('uai')
|
|
|
|
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
|
|
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
|
|
self.assertEqual(decision_claim(literal), 'unknown')
|
|
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
|
|
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
|
|
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
|
|
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
|
|
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
|
|
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
|
|
self.assertIsNone(decision_track_claim('Accept (Poster)'))
|