47 lines
2.3 KiB
Python
47 lines
2.3 KiB
Python
"""Download public daily index observations; retain responses and provenance."""
|
|
import concurrent.futures
|
|
import datetime as dt
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import urllib.request
|
|
|
|
ROOT = Path(__file__).resolve().parent
|
|
RAW = ROOT / 'raw'
|
|
RAW.mkdir(parents=True, exist_ok=True)
|
|
END = int(dt.datetime(2026, 10, 20, tzinfo=dt.timezone.utc).timestamp())
|
|
SOURCES = {
|
|
'yahoo_nasdaq.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EIXIC?period1=-2208988800&period2={END}&interval=1d',
|
|
'yahoo_sp500.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC?period1=-2208988800&period2={END}&interval=1d',
|
|
'fred_nasdaq.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=NASDAQCOM&cosd=1971-02-05',
|
|
'fred_sp500.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=SP500',
|
|
'stooq_sp500.csv': 'https://stooq.com/q/d/l/?s=%5Espx&i=d',
|
|
}
|
|
|
|
def fetch(item):
|
|
name, url = item
|
|
try:
|
|
req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
|
|
with urllib.request.urlopen(req, timeout=60) as response:
|
|
data = response.read()
|
|
if name.endswith('.json'):
|
|
doc = json.loads(data)
|
|
if not doc.get('chart', {}).get('result'):
|
|
raise ValueError('No Yahoo chart result')
|
|
elif not data.startswith((b'observation_date,', b'DATE,', b'Date,')):
|
|
raise ValueError(f'Unexpected CSV response: {data[:60]!r}')
|
|
(RAW / name).write_bytes(data)
|
|
result = {'file': name, 'url': url, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'retrieved_utc': dt.datetime.now(dt.timezone.utc).isoformat(), 'status': 'ok'}
|
|
if name.endswith('.json'):
|
|
x = doc['chart']['result'][0]
|
|
result.update(rows=len(x['timestamp']), first_timestamp=x['timestamp'][0], last_timestamp=x['timestamp'][-1])
|
|
except Exception as exc:
|
|
result = {'file': name, 'url': url, 'status': 'failed', 'error': str(exc)}
|
|
print(json.dumps(result), flush=True)
|
|
return result
|
|
|
|
if __name__ == '__main__':
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
|
|
results = list(executor.map(fetch, SOURCES.items()))
|
|
(ROOT / 'source_manifest.json').write_text(json.dumps(results, ensure_ascii=False, indent=2))
|