"""Download public daily index observations; retain responses and provenance."""
import concurrent.futures
import datetime as dt
import hashlib
import json
from pathlib import Path
import urllib.request

ROOT = Path(__file__).resolve().parent
RAW = ROOT / 'raw'
RAW.mkdir(parents=True, exist_ok=True)
END = int(dt.datetime(2026, 10, 20, tzinfo=dt.timezone.utc).timestamp())
SOURCES = {
    'yahoo_nasdaq.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EIXIC?period1=-2208988800&period2={END}&interval=1d',
    'yahoo_sp500.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC?period1=-2208988800&period2={END}&interval=1d',
    'fred_nasdaq.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=NASDAQCOM&cosd=1971-02-05',
    'fred_sp500.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=SP500',
    'stooq_sp500.csv': 'https://stooq.com/q/d/l/?s=%5Espx&i=d',
}

def fetch(item):
    name, url = item
    try:
        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
        with urllib.request.urlopen(req, timeout=60) as response:
            data = response.read()
        if name.endswith('.json'):
            doc = json.loads(data)
            if not doc.get('chart', {}).get('result'):
                raise ValueError('No Yahoo chart result')
        elif not data.startswith((b'observation_date,', b'DATE,', b'Date,')):
            raise ValueError(f'Unexpected CSV response: {data[:60]!r}')
        (RAW / name).write_bytes(data)
        result = {'file': name, 'url': url, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'retrieved_utc': dt.datetime.now(dt.timezone.utc).isoformat(), 'status': 'ok'}
        if name.endswith('.json'):
            x = doc['chart']['result'][0]
            result.update(rows=len(x['timestamp']), first_timestamp=x['timestamp'][0], last_timestamp=x['timestamp'][-1])
    except Exception as exc:
        result = {'file': name, 'url': url, 'status': 'failed', 'error': str(exc)}
    print(json.dumps(result), flush=True)
    return result

if __name__ == '__main__':
    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
        results = list(executor.map(fetch, SOURCES.items()))
    (ROOT / 'source_manifest.json').write_text(json.dumps(results, ensure_ascii=False, indent=2))