Files
paper-atlas/paper_atlas/io/parquet_metadata.py
T
codex-bot 90fd789a17
Metadata integrity checks / metadata (push) Failing after 47s
Project pinned ReviewArena metadata without full-text payload pages
2026-11-18 09:10:29 +08:00

196 lines
9.5 KiB
Python

"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()