"""Checksum-verified streaming downloads; existing valid snapshots stay intact.""" import argparse import fcntl import hashlib import json from pathlib import Path import re import shutil import time from urllib.parse import urlsplit import urllib.request def digest_file(path): sha=hashlib.sha256();md5=hashlib.md5(usedforsecurity=False);size=0 with path.open('rb') as stream: while block:=stream.read(1024**2): sha.update(block);md5.update(block);size+=len(block) return {'sha256':sha.hexdigest(),'md5':md5.hexdigest(),'bytes':size} def verify(actual,sha256,md5,expected_bytes): if expected_bytes is not None and actual['bytes']!=expected_bytes: raise ValueError('Snapshot size differs from trusted expected size') if sha256 and actual['sha256']!=sha256: raise ValueError('Snapshot SHA256 mismatch; partial file retained') if md5 and actual['md5']!=md5: raise ValueError('Snapshot MD5 mismatch; partial file retained') def write_manifest(output,url,actual): manifest={'status':'verified','file':str(output),'url':url,**actual, 'verified_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())} metadata=Path(str(output)+'.manifest.json');stage=Path(str(metadata)+'.part') stage.write_text(json.dumps(manifest,indent=2));stage.replace(metadata) return manifest def download(url,output,sha256=None,md5=None,expected_bytes=None,resume=False,min_free=8*1024**3): parsed=urlsplit(url) if parsed.scheme!='https' or not parsed.netloc or parsed.username or parsed.password: raise ValueError('A public HTTPS URL without embedded credentials is required') if not (sha256 or md5):raise ValueError('A trusted published checksum is required') if expected_bytes is not None and expected_bytes<=0:raise ValueError('Expected byte count must be positive') if min_free<0:raise ValueError('Disk reserve must not be negative') for name,value,length in [('SHA256',sha256,64),('MD5',md5,32)]: if value and not re.fullmatch('[0-9a-f]{'+str(length)+'}',value):raise ValueError('Invalid '+name) output=Path(output);output.parent.mkdir(parents=True,exist_ok=True) if output.is_symlink():raise ValueError('Refusing a symlink output') intent={'url':url,'expected_sha256':sha256,'expected_md5':md5,'expected_bytes':expected_bytes} with Path(str(output)+'.download.lock').open('a') as lock: fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) if output.exists(): actual=digest_file(output);verify(actual,sha256,md5,expected_bytes) if not Path(str(output)+'.manifest.json').exists():write_manifest(output,url,actual) return {'status':'already_verified','file':str(output),**actual} part=Path(str(output)+'.part');intent_path=Path(str(part)+'.json') offset=0 if part.exists(): if not resume or not intent_path.is_file() or json.loads(intent_path.read_text())!=intent: raise ValueError('Existing partial requires --resume and the same snapshot identity') offset=part.stat().st_size else: intent_path.write_text(json.dumps(intent,indent=2)) remaining=max(0,(expected_bytes or 0)-offset) if shutil.disk_usage(output.parent).freetotal:raise ValueError('HTTP response exceeds advertised size') if written-reported>=64*1024**2: print(json.dumps({'status':'downloading','file':str(output),'bytes':written,'expected_bytes':total}),flush=True);reported=written if shutil.disk_usage(output.parent).free