107 lines
5.3 KiB
Python
107 lines
5.3 KiB
Python
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
|
from datetime import datetime, timezone
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import urllib.request
|
|
import urllib.error
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
def digest(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def checked_html(raw, receipt, url, limit):
|
|
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
|
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
|
or receipt.get('sha256') != digest(raw)
|
|
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
|
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
|
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
|
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
|
raise ValueError('Access challenge; stop without bypass')
|
|
return raw, receipt
|
|
|
|
|
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
def redirect_request(self, *args, **kwargs):
|
|
return None
|
|
|
|
|
|
class PublicHTMLCache:
|
|
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
|
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
|
self.root = Path(directory)
|
|
self.root.mkdir(parents=True, exist_ok=True)
|
|
if self.root.is_symlink():
|
|
raise ValueError('Cache symlinks rejected')
|
|
self.origin = origin
|
|
self.limit = limit
|
|
self.delay = delay
|
|
self.opener = urllib.request.build_opener(NoRedirect())
|
|
|
|
def paths(self, url):
|
|
u = urlsplit(url)
|
|
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
|
raise ValueError('Unapproved public metadata origin')
|
|
key = digest(url.encode())
|
|
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
|
|
|
def cached(self, url):
|
|
body, receipt, failure = self.paths(url)
|
|
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
|
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
|
raise ValueError('Cache symlinks rejected')
|
|
if any(p.exists() for p in parts):
|
|
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
|
if failure.exists():
|
|
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
|
if body.exists() != receipt.exists():
|
|
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
|
if not body.exists():
|
|
return None
|
|
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
|
raise ValueError('Oversized cache entry')
|
|
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
|
|
|
def get(self, url):
|
|
cached = self.cached(url)
|
|
if cached is not None:
|
|
return cached
|
|
body, receipt, failure = self.paths(url)
|
|
try:
|
|
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
|
with self.opener.open(request, timeout=30) as response:
|
|
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
|
'content_type': response.headers.get('Content-Type', '')}
|
|
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
|
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
|
raw = response.read(self.limit+1)
|
|
if len(raw) > self.limit:
|
|
raise ValueError('Metadata response exceeds size limit')
|
|
expected = response.headers.get('Content-Length')
|
|
if expected is not None and int(expected) != len(raw):
|
|
raise ConnectionError('Incomplete metadata response')
|
|
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
|
checked_html(raw, rec, url, self.limit)
|
|
except (urllib.error.HTTPError, ValueError) as error:
|
|
if isinstance(error, urllib.error.HTTPError):
|
|
status = error.code
|
|
retry_after = error.headers.get('Retry-After') if error.headers else None
|
|
error.close()
|
|
else:
|
|
status = None
|
|
retry_after = None
|
|
# Access errors remain sticky until the operator verifies availability.
|
|
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
|
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
|
raise
|
|
part = body.with_suffix('.html.part')
|
|
part.write_bytes(raw); part.replace(body)
|
|
part = receipt.with_suffix('.json.part')
|
|
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
|
time.sleep(self.delay)
|
|
return raw, rec
|