196 lines
9.5 KiB
Python
196 lines
9.5 KiB
Python
"""Read only selected Parquet column chunks; never fetch other payload pages."""
|
|
import hashlib
|
|
import io
|
|
import re
|
|
import struct
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
MAX_FOOTER = 4 * 1024**2
|
|
MAX_SELECTED_BYTES = 64 * 1024**2
|
|
MAX_REQUESTS = 512
|
|
|
|
|
|
class RangeSource:
|
|
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
|
|
def __init__(self, url, size, *, opener=None):
|
|
self.url, self.size = url, size
|
|
self.opener = opener or urllib.request.build_opener()
|
|
self.etag = None
|
|
self.receipts = []
|
|
self.bytes_read = 0
|
|
|
|
def read(self, start, length):
|
|
if (type(start) is not int or type(length) is not int or length <= 0
|
|
or start < 0 or start + length > self.size
|
|
or self.bytes_read + length > MAX_SELECTED_BYTES
|
|
or len(self.receipts) >= MAX_REQUESTS):
|
|
raise ValueError('Bounded metadata range budget exceeded')
|
|
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
|
|
if self.etag and not self.etag.startswith('W/'):
|
|
headers['If-Match'] = self.etag
|
|
request = urllib.request.Request(self.url, headers=headers)
|
|
try:
|
|
with self.opener.open(request, timeout=45) as response:
|
|
if response.status != 206:
|
|
raise ValueError('Server did not honor metadata range; full download forbidden')
|
|
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
|
|
raise ValueError('Object size or range differs from pinned inventory')
|
|
encoding = response.headers.get('Content-Encoding', 'identity')
|
|
if encoding != 'identity':
|
|
raise ValueError('Encoded HTTP ranges are unsupported')
|
|
etag = response.headers.get('ETag')
|
|
if self.receipts and etag != self.etag:
|
|
raise ValueError('Object ETag changed between metadata ranges')
|
|
raw = response.read(length + 1)
|
|
if len(raw) != length:
|
|
raise ValueError('Truncated or oversized metadata range')
|
|
if not self.receipts:
|
|
self.etag = etag
|
|
except urllib.error.HTTPError as error:
|
|
# Do not expose signed CDN redirect URLs or retry denied resources.
|
|
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
|
|
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
|
self.bytes_read += length
|
|
return raw
|
|
|
|
|
|
class SparseFile(io.RawIOBase):
|
|
"""A logical file backed only by approved bytes; all other positions are zeros.
|
|
|
|
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
|
|
fetching the preceding OCR/review page. Data pages are explicitly selected
|
|
from footer offsets before they can be downloaded.
|
|
"""
|
|
def __init__(self, size, segments):
|
|
self.size, self.segments, self.position = size, segments, 0
|
|
|
|
def readable(self):
|
|
return True
|
|
|
|
def seekable(self):
|
|
return True
|
|
|
|
def tell(self):
|
|
return self.position
|
|
|
|
def seek(self, offset, whence=0):
|
|
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
|
|
if base is None or base + offset < 0:
|
|
raise ValueError('Invalid sparse metadata seek')
|
|
self.position = base + offset
|
|
return self.position
|
|
|
|
def read(self, length=-1):
|
|
if length < 0:
|
|
length = self.size - self.position
|
|
length = min(length, max(0, self.size - self.position))
|
|
if length > MAX_SELECTED_BYTES:
|
|
raise ValueError('Unbounded sparse read forbidden')
|
|
start, end = self.position, self.position + length
|
|
raw = bytearray(length)
|
|
for offset, data in self.segments:
|
|
left, right = max(offset, start), min(offset + len(data), end)
|
|
if left < right:
|
|
raw[left-start:right-start] = data[left-offset:right-offset]
|
|
self.position = end
|
|
return bytes(raw)
|
|
|
|
def readinto(self, buffer):
|
|
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
|
|
return len(raw)
|
|
|
|
|
|
def arrow():
|
|
try:
|
|
import pyarrow as pa
|
|
import pyarrow.parquet as pq
|
|
except ImportError:
|
|
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
|
|
return pa, pq
|
|
|
|
|
|
def projection(source, columns, *, progress=None):
|
|
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
|
|
pa, pq = arrow()
|
|
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
|
|
raise ValueError('Bounded pinned Parquet size required')
|
|
if (not isinstance(columns, dict) or not columns
|
|
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
|
|
raise ValueError('Explicit top-level metadata columns required')
|
|
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
|
|
if header != b'PAR1' or trailer[4:] != b'PAR1':
|
|
raise ValueError('Expected unencrypted Parquet framing')
|
|
length = struct.unpack('<I', trailer[:4])[0]
|
|
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
|
|
raise ValueError('Invalid or oversized Parquet footer')
|
|
footer_start = source.size - 8 - length
|
|
footer = source.read(footer_start, length)
|
|
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
|
|
sparse = SparseFile(source.size, segments)
|
|
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
|
|
try:
|
|
schema = parquet.schema_arrow
|
|
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
|
|
raise ValueError('Missing or repeated metadata columns')
|
|
for key, kind in columns.items():
|
|
value = schema.field(key).type
|
|
if kind == 'string':
|
|
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
|
|
elif kind == 'integer':
|
|
okay = pa.types.is_integer(value)
|
|
elif kind == 'strings':
|
|
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
|
|
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
|
|
else:
|
|
raise ValueError('Unsupported safe metadata type')
|
|
if not okay:
|
|
raise ValueError('Metadata column type differs: ' + key)
|
|
metadata = parquet.metadata
|
|
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
|
|
raise ValueError('Bounded metadata row count required')
|
|
wanted, excluded = [], []
|
|
for group in range(metadata.num_row_groups):
|
|
rg = metadata.row_group(group)
|
|
for i in range(rg.num_columns):
|
|
col = rg.column(i)
|
|
if col.file_path:
|
|
raise ValueError('External Parquet column files forbidden')
|
|
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
|
|
if not offsets:
|
|
raise ValueError('Missing column chunk offset')
|
|
start = min(offsets); end = start + col.total_compressed_size
|
|
if not 4 <= start < end <= footer_start:
|
|
raise ValueError('Column chunk lies outside the pinned payload')
|
|
item = (start, end, group, col.path_in_schema)
|
|
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
|
|
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
|
|
raise ValueError('Selected range overlaps an excluded payload column')
|
|
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
|
|
raise ValueError('Metadata projection exceeds byte budget')
|
|
if len(wanted) + 3 > MAX_REQUESTS:
|
|
raise ValueError('Metadata projection exceeds request budget')
|
|
for start, end, group, path in sorted(wanted):
|
|
segments.append((start, source.read(start, end-start)))
|
|
if progress:
|
|
progress({'phase': 'metadata_column_range', 'row_group': group,
|
|
'column': path, 'metadata_bytes_read': source.bytes_read})
|
|
rows = []
|
|
for group in range(metadata.num_row_groups):
|
|
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
|
|
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
|
|
raise ValueError('Unexpected projected metadata fields')
|
|
rows.extend(table.select(list(columns)).to_pylist())
|
|
if len(rows) != metadata.num_rows:
|
|
raise ValueError('Incomplete metadata projection')
|
|
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
|
|
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
|
|
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
|
|
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
|
|
'source_bytes': source.size, 'full_file_checksum_verified': False,
|
|
'excluded_payload_pages_downloaded': False,
|
|
'footer_saved': False, 'raw_parquet_saved': False}
|
|
return rows, report
|
|
finally:
|
|
parquet.close()
|