111 lines
6.2 KiB
Python
111 lines
6.2 KiB
Python
"""Checksum-verified streaming downloads; existing valid snapshots stay intact."""
|
|
import argparse
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import shutil
|
|
import time
|
|
from urllib.parse import urlsplit
|
|
import urllib.request
|
|
|
|
|
|
def digest_file(path):
|
|
sha=hashlib.sha256();md5=hashlib.md5(usedforsecurity=False);size=0
|
|
with path.open('rb') as stream:
|
|
while block:=stream.read(1024**2):
|
|
sha.update(block);md5.update(block);size+=len(block)
|
|
return {'sha256':sha.hexdigest(),'md5':md5.hexdigest(),'bytes':size}
|
|
|
|
|
|
def verify(actual,sha256,md5,expected_bytes):
|
|
if expected_bytes is not None and actual['bytes']!=expected_bytes:
|
|
raise ValueError('Snapshot size differs from trusted expected size')
|
|
if sha256 and actual['sha256']!=sha256:
|
|
raise ValueError('Snapshot SHA256 mismatch; partial file retained')
|
|
if md5 and actual['md5']!=md5:
|
|
raise ValueError('Snapshot MD5 mismatch; partial file retained')
|
|
|
|
|
|
def write_manifest(output,url,actual):
|
|
manifest={'status':'verified','file':str(output),'url':url,**actual,
|
|
'verified_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())}
|
|
metadata=Path(str(output)+'.manifest.json');stage=Path(str(metadata)+'.part')
|
|
stage.write_text(json.dumps(manifest,indent=2));stage.replace(metadata)
|
|
return manifest
|
|
|
|
|
|
def download(url,output,sha256=None,md5=None,expected_bytes=None,resume=False,min_free=8*1024**3):
|
|
parsed=urlsplit(url)
|
|
if parsed.scheme!='https' or not parsed.netloc or parsed.username or parsed.password:
|
|
raise ValueError('A public HTTPS URL without embedded credentials is required')
|
|
if not (sha256 or md5):raise ValueError('A trusted published checksum is required')
|
|
if expected_bytes is not None and expected_bytes<=0:raise ValueError('Expected byte count must be positive')
|
|
if min_free<0:raise ValueError('Disk reserve must not be negative')
|
|
for name,value,length in [('SHA256',sha256,64),('MD5',md5,32)]:
|
|
if value and not re.fullmatch('[0-9a-f]{'+str(length)+'}',value):raise ValueError('Invalid '+name)
|
|
output=Path(output);output.parent.mkdir(parents=True,exist_ok=True)
|
|
if output.is_symlink():raise ValueError('Refusing a symlink output')
|
|
intent={'url':url,'expected_sha256':sha256,'expected_md5':md5,'expected_bytes':expected_bytes}
|
|
with Path(str(output)+'.download.lock').open('a') as lock:
|
|
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
|
if output.exists():
|
|
actual=digest_file(output);verify(actual,sha256,md5,expected_bytes)
|
|
if not Path(str(output)+'.manifest.json').exists():write_manifest(output,url,actual)
|
|
return {'status':'already_verified','file':str(output),**actual}
|
|
part=Path(str(output)+'.part');intent_path=Path(str(part)+'.json')
|
|
offset=0
|
|
if part.exists():
|
|
if not resume or not intent_path.is_file() or json.loads(intent_path.read_text())!=intent:
|
|
raise ValueError('Existing partial requires --resume and the same snapshot identity')
|
|
offset=part.stat().st_size
|
|
else:
|
|
intent_path.write_text(json.dumps(intent,indent=2))
|
|
remaining=max(0,(expected_bytes or 0)-offset)
|
|
if shutil.disk_usage(output.parent).free<min_free+remaining:
|
|
raise RuntimeError('Insufficient free space for snapshot and reserve')
|
|
if expected_bytes is None or offset!=expected_bytes:
|
|
headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'}
|
|
if offset:headers['Range']=f'bytes={offset}-'
|
|
with urllib.request.urlopen(urllib.request.Request(url,headers=headers),timeout=90) as response:
|
|
if offset:
|
|
content_range=response.headers.get('Content-Range','')
|
|
match=re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)',content_range)
|
|
if response.status!=206 or not match or int(match[1])!=offset:
|
|
raise ValueError('Server did not honor resume range; partial retained')
|
|
total=int(match[3])
|
|
else:
|
|
if response.status!=200:raise ValueError('Expected a complete HTTP 200 response')
|
|
total=int(response.headers.get('Content-Length','0')) or expected_bytes
|
|
if expected_bytes is not None and total is not None and total!=expected_bytes:
|
|
raise ValueError('HTTP size differs from expected snapshot')
|
|
if total and shutil.disk_usage(output.parent).free<min_free+total-offset:
|
|
raise RuntimeError('Insufficient disk reserve for response')
|
|
written=offset;reported=offset
|
|
with part.open('ab' if part.exists() else 'xb') as target:
|
|
while block:=response.read(1024**2):
|
|
target.write(block);written+=len(block)
|
|
if total and written>total:raise ValueError('HTTP response exceeds advertised size')
|
|
if written-reported>=64*1024**2:
|
|
print(json.dumps({'status':'downloading','file':str(output),'bytes':written,'expected_bytes':total}),flush=True);reported=written
|
|
if shutil.disk_usage(output.parent).free<min_free:raise RuntimeError('Stopped at disk reserve')
|
|
if total and written!=total:raise ValueError('Incomplete HTTP response; partial retained')
|
|
actual=digest_file(part);verify(actual,sha256,md5,expected_bytes)
|
|
part.replace(output)
|
|
manifest=write_manifest(output,url,actual)
|
|
intent_path.unlink(missing_ok=True)
|
|
return manifest
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--url',required=True);parser.add_argument('--output',required=True)
|
|
parser.add_argument('--sha256');parser.add_argument('--md5');parser.add_argument('--expected-bytes',type=int)
|
|
parser.add_argument('--resume',action='store_true');parser.add_argument('--min-free-gib',type=float,default=8)
|
|
args=parser.parse_args(argv)
|
|
print(json.dumps(download(args.url,args.output,args.sha256,args.md5,args.expected_bytes,args.resume,int(args.min_free_gib*1024**3))),flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|