"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh.""" from datetime import datetime, timezone import hashlib import json from pathlib import Path import time import urllib.request import urllib.error from urllib.parse import urlsplit def digest(raw): return hashlib.sha256(raw).hexdigest() def checked_html(raw, receipt, url, limit): if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw) or receipt.get('sha256') != digest(raw) or not str(receipt.get('content_type', '')).lower().startswith('text/html') or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None): raise ValueError('Invalid public HTML receipt, response or timestamp') if any(v in raw.lower() for v in (b'just a moment', b'cf-chl-', b'botstopper')): raise ValueError('Access challenge; stop without bypass') return raw, receipt class NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, *args, **kwargs): return None class PublicHTMLCache: """Caller holds a single-writer lock and restricts the allowed URL paths.""" def __init__(self, directory, origin, *, limit=2*1024**2, delay=1): self.root = Path(directory) self.root.mkdir(parents=True, exist_ok=True) if self.root.is_symlink(): raise ValueError('Cache symlinks rejected') self.origin = origin self.limit = limit self.delay = delay self.opener = urllib.request.build_opener(NoRedirect()) def paths(self, url): u = urlsplit(url) if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin: raise ValueError('Unapproved public metadata origin') key = digest(url.encode()) return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json') def cached(self, url): body, receipt, failure = self.paths(url) parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part')) if any(p.is_symlink() for p in (body, receipt, failure, *parts)): raise ValueError('Cache symlinks rejected') if any(p.exists() for p in parts): raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery') if failure.exists(): raise ValueError('Previous access failure requires inspection; no automatic retry') if body.exists() != receipt.exists(): raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery') if not body.exists(): return None if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024: raise ValueError('Oversized cache entry') return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit) def get(self, url): cached = self.cached(url) if cached is not None: return cached body, receipt, failure = self.paths(url) try: request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'}) with self.opener.open(request, timeout=30) as response: rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status, 'content_type': response.headers.get('Content-Type', '')} if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'): raise ValueError('Unexpected metadata redirect, HTTP status or content type') raw = response.read(self.limit+1) if len(raw) > self.limit: raise ValueError('Metadata response exceeds size limit') expected = response.headers.get('Content-Length') if expected is not None and int(expected) != len(raw): raise ConnectionError('Incomplete metadata response') rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat()) checked_html(raw, rec, url, self.limit) except (urllib.error.HTTPError, ValueError) as error: if isinstance(error, urllib.error.HTTPError): status = error.code retry_after = error.headers.get('Retry-After') if error.headers else None error.close() else: status = None retry_after = None # Access errors remain sticky until the operator verifies availability. failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__, 'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n') raise part = body.with_suffix('.html.part') part.write_bytes(raw); part.replace(body) part = receipt.with_suffix('.json.part') part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt) time.sleep(self.delay) return raw, rec