Import project source and Git ignore rules

fetch_data.py

@@ -0,0 +1,46 @@
+"""Download public daily index observations; retain responses and provenance."""
+import concurrent.futures
+import datetime as dt
+import hashlib
+import json
+from pathlib import Path
+import urllib.request
+
+ROOT = Path(__file__).resolve().parent
+RAW = ROOT / 'raw'
+RAW.mkdir(parents=True, exist_ok=True)
+END = int(dt.datetime(2026, 10, 20, tzinfo=dt.timezone.utc).timestamp())
+SOURCES = {
+    'yahoo_nasdaq.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EIXIC?period1=-2208988800&period2={END}&interval=1d',
+    'yahoo_sp500.json': f'https://query1.finance.yahoo.com/v8/finance/chart/%5EGSPC?period1=-2208988800&period2={END}&interval=1d',
+    'fred_nasdaq.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=NASDAQCOM&cosd=1971-02-05',
+    'fred_sp500.csv': 'https://fred.stlouisfed.org/graph/fredgraph.csv?id=SP500',
+    'stooq_sp500.csv': 'https://stooq.com/q/d/l/?s=%5Espx&i=d',
+}
+
+def fetch(item):
+    name, url = item
+    try:
+        req = urllib.request.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
+        with urllib.request.urlopen(req, timeout=60) as response:
+            data = response.read()
+        if name.endswith('.json'):
+            doc = json.loads(data)
+            if not doc.get('chart', {}).get('result'):
+                raise ValueError('No Yahoo chart result')
+        elif not data.startswith((b'observation_date,', b'DATE,', b'Date,')):
+            raise ValueError(f'Unexpected CSV response: {data[:60]!r}')
+        (RAW / name).write_bytes(data)
+        result = {'file': name, 'url': url, 'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'retrieved_utc': dt.datetime.now(dt.timezone.utc).isoformat(), 'status': 'ok'}
+        if name.endswith('.json'):
+            x = doc['chart']['result'][0]
+            result.update(rows=len(x['timestamp']), first_timestamp=x['timestamp'][0], last_timestamp=x['timestamp'][-1])
+    except Exception as exc:
+        result = {'file': name, 'url': url, 'status': 'failed', 'error': str(exc)}
+    print(json.dumps(result), flush=True)
+    return result
+
+if __name__ == '__main__':
+    with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
+        results = list(executor.map(fetch, SOURCES.items()))
+    (ROOT / 'source_manifest.json').write_text(json.dumps(results, ensure_ascii=False, indent=2))