51 lines
2.4 KiB
Python
51 lines
2.4 KiB
Python
"""Bounded public metadata requests and atomic response caching."""
|
|
import hashlib
|
|
from http.client import IncompleteRead
|
|
import json
|
|
from pathlib import Path
|
|
import ssl
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
|
|
def fetch(url,cache,limit=64*1024**2,delay=0.5):
|
|
cache = Path(cache)
|
|
cache.mkdir(parents=True,exist_ok=True)
|
|
path = cache/hashlib.sha256(url.encode()).hexdigest()
|
|
if path.exists() and time.time()-path.stat().st_mtime < 24*3600:
|
|
if path.stat().st_size>limit:raise RuntimeError('Metadata response exceeds byte limit')
|
|
return path.read_bytes()
|
|
for attempt in range(3):
|
|
try:
|
|
request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'})
|
|
with urllib.request.urlopen(request,timeout=90) as response:
|
|
if response.status != 200:
|
|
raise RuntimeError('Expected complete HTTP 200 response')
|
|
expected=int(response.headers.get('Content-Length','0'))
|
|
if expected>limit:
|
|
raise RuntimeError('Metadata response exceeds byte limit')
|
|
data = response.read(limit+1)
|
|
if len(data)>limit:
|
|
raise RuntimeError('Metadata response exceeds byte limit')
|
|
if expected and len(data)!=expected:
|
|
raise IncompleteRead(data,max(0,expected-len(data)))
|
|
part = path.with_suffix('.part')
|
|
part.write_bytes(data)
|
|
part.replace(path)
|
|
path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}))
|
|
time.sleep(delay)
|
|
return data
|
|
except urllib.error.HTTPError as exc:
|
|
if exc.code not in (429,500,502,503,504) or attempt==2:
|
|
exc.close()
|
|
raise
|
|
retry_after=exc.headers.get('Retry-After','1');exc.close()
|
|
try:retry_after=int(retry_after)
|
|
except ValueError:retry_after=5
|
|
time.sleep(min(30,max(2**attempt,retry_after)))
|
|
except (urllib.error.URLError,TimeoutError,ConnectionError,IncompleteRead,ssl.SSLError) as exc:
|
|
reason=getattr(exc,'reason',exc)
|
|
if isinstance(reason,ssl.SSLCertVerificationError) or attempt==2:raise
|
|
time.sleep(min(30,2**(attempt+1)))
|