feat: FRED bootstrap + Calendar page complete rebuild
- backend/services/fred_bootstrap.py: fetch 11 FRED series (PAYEMS, UNRATE, CPI, PCE, FEDFUNDS, ICSA, GDP, HY spread, T10Y2Y, T10Y3M) from public CSV endpoint — no API key needed; computes rolling z-scores and upserts into economic_events table - backend/routers/eco.py: new /api/eco router with bootstrap (POST + status GET), events list with full filtering (date range, category, series, min z-score, direction, sort/pagination), series catalog, and db status endpoints - backend/main.py: register eco router - frontend/src/pages/CalendarPage.tsx: complete rewrite — real data table from /api/eco/events, Bootstrap FRED button with live polling, filter bar (date range, category, series chips, |z| threshold, direction), sort by date/z-score/series, pagination, z-score badges with color coding, sidebar with series inventory + geo alerts + z-score guide Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
@@ -13,6 +13,7 @@ from routers import logs as logs_router
|
||||
from routers import var as var_router
|
||||
from routers import reports as reports_router
|
||||
from routers import institutional as institutional_router
|
||||
from routers import eco as eco_router
|
||||
from services.database import init_db, get_config, cleanup_stale_running_cycles
|
||||
import os
|
||||
import logging
|
||||
@@ -134,6 +135,7 @@ app.include_router(impact_router.router)
|
||||
app.include_router(cycle_actions_router.router)
|
||||
app.include_router(market_events_router.router)
|
||||
app.include_router(ai_desks_router.router)
|
||||
app.include_router(eco_router.router)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
||||
198
backend/routers/eco.py
Normal file
198
backend/routers/eco.py
Normal file
@@ -0,0 +1,198 @@
|
||||
"""
|
||||
Economic calendar router — FRED-backed historical events + bootstrap endpoint.
|
||||
Prefix: /api/eco
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, BackgroundTasks, HTTPException, Query
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/eco", tags=["Economic Calendar"])
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_row(row: Dict) -> Dict:
|
||||
for field in ("assets_impacted",):
|
||||
val = row.get(field)
|
||||
if isinstance(val, str):
|
||||
try:
|
||||
row[field] = json.loads(val) if val else []
|
||||
except Exception:
|
||||
row[field] = []
|
||||
elif val is None:
|
||||
row[field] = []
|
||||
return row
|
||||
|
||||
|
||||
# ── Bootstrap ─────────────────────────────────────────────────────────────────
|
||||
|
||||
_bootstrap_status: Dict[str, Any] = {"running": False, "last_result": None}
|
||||
|
||||
|
||||
def _run_bootstrap(from_date: str, series_ids: Optional[List[str]], force: bool):
|
||||
global _bootstrap_status
|
||||
_bootstrap_status["running"] = True
|
||||
try:
|
||||
from services.fred_bootstrap import bootstrap_fred
|
||||
result = bootstrap_fred(from_date=from_date, series_ids=series_ids, force=force)
|
||||
_bootstrap_status["last_result"] = result
|
||||
total_inserted = sum(v.get("inserted", 0) for v in result.values() if isinstance(v, dict))
|
||||
logger.info(f"[eco/bootstrap] Done — {total_inserted} rows inserted")
|
||||
except Exception as e:
|
||||
logger.error(f"[eco/bootstrap] Failed: {e}")
|
||||
_bootstrap_status["last_result"] = {"error": str(e)}
|
||||
finally:
|
||||
_bootstrap_status["running"] = False
|
||||
|
||||
|
||||
@router.post("/bootstrap")
|
||||
def bootstrap(
|
||||
background_tasks: BackgroundTasks,
|
||||
from_date: str = Query("2020-01-01", description="Start date for historical data"),
|
||||
series: Optional[str] = Query(None, description="Comma-separated series IDs (blank = all)"),
|
||||
force: bool = Query(False, description="Overwrite existing rows"),
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Trigger FRED data bootstrap (runs in background).
|
||||
Fetches public CSV endpoint — no API key required.
|
||||
"""
|
||||
if _bootstrap_status["running"]:
|
||||
raise HTTPException(409, "Bootstrap already running")
|
||||
series_ids = [s.strip() for s in series.split(",") if s.strip()] if series else None
|
||||
background_tasks.add_task(_run_bootstrap, from_date, series_ids, force)
|
||||
return {"status": "started", "from_date": from_date, "series": series_ids or "all"}
|
||||
|
||||
|
||||
@router.get("/bootstrap/status")
|
||||
def bootstrap_status() -> Dict[str, Any]:
|
||||
return _bootstrap_status
|
||||
|
||||
|
||||
# ── Series catalog ────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/series")
|
||||
def list_series() -> List[Dict[str, Any]]:
|
||||
"""Return available FRED series metadata."""
|
||||
from services.fred_bootstrap import FRED_SERIES, CATEGORIES
|
||||
return [
|
||||
{
|
||||
"id": sid,
|
||||
"name": meta["name"],
|
||||
"category": meta["category"],
|
||||
"freq": meta["freq"],
|
||||
"unit": meta["unit"],
|
||||
"assets": meta["assets"],
|
||||
}
|
||||
for sid, meta in FRED_SERIES.items()
|
||||
]
|
||||
|
||||
|
||||
# ── Events list ───────────────────────────────────────────────────────────────
|
||||
|
||||
_SORT_COLS = {
|
||||
"date": "ee.event_date",
|
||||
"zscore": "ABS(COALESCE(ee.surprise_zscore, 0))",
|
||||
"series": "ee.series_id",
|
||||
"name": "ee.event_name",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/events")
|
||||
def list_eco_events(
|
||||
date_from: Optional[str] = Query(None),
|
||||
date_to: Optional[str] = Query(None),
|
||||
series: Optional[str] = Query(None, description="Comma-separated series IDs"),
|
||||
category: Optional[str] = Query(None, description="employment|inflation|growth|monetary|credit|rates"),
|
||||
min_zscore: float = Query(0.0, ge=0.0, description="Minimum |z-score| filter"),
|
||||
direction: Optional[str] = Query(None, description="bullish|bearish|neutral"),
|
||||
sort_by: str = Query("date", description="date|zscore|series|name"),
|
||||
sort_dir: str = Query("desc", description="asc|desc"),
|
||||
limit: int = Query(200, ge=1, le=2000),
|
||||
offset: int = Query(0, ge=0),
|
||||
) -> Dict[str, Any]:
|
||||
from services.database import get_conn
|
||||
from services.fred_bootstrap import FRED_SERIES
|
||||
|
||||
conn = get_conn()
|
||||
where_parts: List[str] = []
|
||||
params: List[Any] = []
|
||||
|
||||
if date_from:
|
||||
where_parts.append("ee.event_date >= ?")
|
||||
params.append(date_from)
|
||||
if date_to:
|
||||
where_parts.append("ee.event_date <= ?")
|
||||
params.append(date_to)
|
||||
|
||||
if series:
|
||||
ids = [s.strip() for s in series.split(",") if s.strip()]
|
||||
if ids:
|
||||
placeholders = ",".join("?" * len(ids))
|
||||
where_parts.append(f"ee.series_id IN ({placeholders})")
|
||||
params.extend(ids)
|
||||
|
||||
if category:
|
||||
# Map category → series IDs
|
||||
matching = [sid for sid, m in FRED_SERIES.items() if m["category"] == category]
|
||||
if matching:
|
||||
placeholders = ",".join("?" * len(matching))
|
||||
where_parts.append(f"ee.series_id IN ({placeholders})")
|
||||
params.extend(matching)
|
||||
else:
|
||||
# No matching series — return empty
|
||||
conn.close()
|
||||
return {"total": 0, "offset": offset, "limit": limit, "events": []}
|
||||
|
||||
if min_zscore > 0:
|
||||
where_parts.append("ABS(COALESCE(ee.surprise_zscore, 0)) >= ?")
|
||||
params.append(min_zscore)
|
||||
if direction:
|
||||
where_parts.append("ee.surprise_direction = ?")
|
||||
params.append(direction)
|
||||
|
||||
where_sql = ("WHERE " + " AND ".join(where_parts)) if where_parts else ""
|
||||
sort_col = _SORT_COLS.get(sort_by, "ee.event_date")
|
||||
order_sql = f"ORDER BY {sort_col} {'DESC' if sort_dir == 'desc' else 'ASC'}"
|
||||
|
||||
count_sql = f"SELECT COUNT(*) FROM economic_events ee {where_sql}"
|
||||
data_sql = f"SELECT ee.* FROM economic_events ee {where_sql} {order_sql} LIMIT ? OFFSET ?"
|
||||
|
||||
try:
|
||||
total = conn.execute(count_sql, params).fetchone()[0]
|
||||
rows = conn.execute(data_sql, params + [limit, offset]).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
events = [_parse_row(dict(r)) for r in rows]
|
||||
return {"total": total, "offset": offset, "limit": limit, "events": events}
|
||||
|
||||
|
||||
# ── DB count summary ──────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/status")
|
||||
def eco_status() -> Dict[str, Any]:
|
||||
from services.database import get_conn
|
||||
conn = get_conn()
|
||||
try:
|
||||
total = conn.execute("SELECT COUNT(*) FROM economic_events").fetchone()[0]
|
||||
latest = conn.execute(
|
||||
"SELECT MAX(event_date) FROM economic_events"
|
||||
).fetchone()[0]
|
||||
earliest = conn.execute(
|
||||
"SELECT MIN(event_date) FROM economic_events"
|
||||
).fetchone()[0]
|
||||
by_series = conn.execute(
|
||||
"SELECT series_id, COUNT(*) as cnt, MAX(event_date) as latest "
|
||||
"FROM economic_events GROUP BY series_id ORDER BY series_id"
|
||||
).fetchall()
|
||||
return {
|
||||
"total": total,
|
||||
"earliest": earliest,
|
||||
"latest": latest,
|
||||
"by_series": [dict(r) for r in by_series],
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
281
backend/services/fred_bootstrap.py
Normal file
281
backend/services/fred_bootstrap.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""
|
||||
FRED historical bootstrap — fetches monthly/quarterly/weekly release data
|
||||
from FRED's public CSV endpoint (no API key required) and stores it in
|
||||
the economic_events table with rolling z-score surprises.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, date
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import httpx
|
||||
import pandas as pd
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ── Series catalog ────────────────────────────────────────────────────────────
|
||||
|
||||
FRED_SERIES: Dict[str, Dict[str, Any]] = {
|
||||
"PAYEMS": {
|
||||
"name": "Non-Farm Payrolls",
|
||||
"unit": "K",
|
||||
"freq": "monthly",
|
||||
"category": "employment",
|
||||
"higher_is_bullish": True,
|
||||
"assets": ["SPY", "QQQ", "EURUSD=X", "TLT"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"UNRATE": {
|
||||
"name": "Unemployment Rate",
|
||||
"unit": "%",
|
||||
"freq": "monthly",
|
||||
"category": "employment",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["SPY", "QQQ", "EURUSD=X"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"CPIAUCSL": {
|
||||
"name": "CPI (All Items YoY)",
|
||||
"unit": "%",
|
||||
"freq": "monthly",
|
||||
"category": "inflation",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["TLT", "GLD", "EURUSD=X", "SPY"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"CPILFESL": {
|
||||
"name": "Core CPI",
|
||||
"unit": "%",
|
||||
"freq": "monthly",
|
||||
"category": "inflation",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["TLT", "GLD", "EURUSD=X"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"PCEPILFE": {
|
||||
"name": "Core PCE",
|
||||
"unit": "%",
|
||||
"freq": "monthly",
|
||||
"category": "inflation",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["TLT", "GLD", "SPY"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"FEDFUNDS": {
|
||||
"name": "Fed Funds Rate (FOMC)",
|
||||
"unit": "%",
|
||||
"freq": "monthly",
|
||||
"category": "monetary",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["TLT", "SPY", "EURUSD=X", "GLD"],
|
||||
"zscore_window": 12,
|
||||
},
|
||||
"ICSA": {
|
||||
"name": "Initial Jobless Claims",
|
||||
"unit": "K",
|
||||
"freq": "weekly",
|
||||
"category": "employment",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["SPY", "QQQ"],
|
||||
"zscore_window": 52,
|
||||
},
|
||||
"A191RL1Q225SBEA": {
|
||||
"name": "GDP Growth (Quarterly)",
|
||||
"unit": "%",
|
||||
"freq": "quarterly",
|
||||
"category": "growth",
|
||||
"higher_is_bullish": True,
|
||||
"assets": ["SPY", "QQQ", "EURUSD=X", "TLT"],
|
||||
"zscore_window": 8,
|
||||
},
|
||||
"BAMLH0A0HYM2": {
|
||||
"name": "HY Credit Spread (OAS)",
|
||||
"unit": "pp",
|
||||
"freq": "weekly",
|
||||
"category": "credit",
|
||||
"higher_is_bullish": False,
|
||||
"assets": ["HYG", "LQD", "SPY"],
|
||||
"zscore_window": 52,
|
||||
},
|
||||
"T10Y2Y": {
|
||||
"name": "Yield Spread 10Y-2Y",
|
||||
"unit": "pp",
|
||||
"freq": "weekly",
|
||||
"category": "rates",
|
||||
"higher_is_bullish": True,
|
||||
"assets": ["TLT", "IEF", "SPY", "HYG"],
|
||||
"zscore_window": 52,
|
||||
},
|
||||
"T10Y3M": {
|
||||
"name": "Yield Spread 10Y-3M",
|
||||
"unit": "pp",
|
||||
"freq": "weekly",
|
||||
"category": "rates",
|
||||
"higher_is_bullish": True,
|
||||
"assets": ["TLT", "IEF", "SPY"],
|
||||
"zscore_window": 52,
|
||||
},
|
||||
}
|
||||
|
||||
CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
|
||||
|
||||
|
||||
# ── FRED fetch ────────────────────────────────────────────────────────────────
|
||||
|
||||
_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
|
||||
|
||||
|
||||
def _fetch_fred_csv(series_id: str, from_date: str = "2019-01-01") -> Optional[pd.DataFrame]:
|
||||
"""
|
||||
Download FRED series as CSV. Returns DataFrame with DATE index and VALUE column.
|
||||
Uses public endpoint — no API key required.
|
||||
"""
|
||||
url = _FRED_CSV_URL.format(series_id=series_id)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
resp.raise_for_status()
|
||||
from io import StringIO
|
||||
df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE")
|
||||
df.columns = ["value"]
|
||||
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
||||
df = df.dropna()
|
||||
df = df[df.index >= pd.Timestamp(from_date)]
|
||||
df = df.sort_index()
|
||||
return df
|
||||
except Exception as e:
|
||||
logger.warning(f"[FRED] Failed to fetch {series_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
# ── Z-score computation ───────────────────────────────────────────────────────
|
||||
|
||||
def _compute_zscore_series(values: pd.Series, window: int) -> Tuple[pd.Series, pd.Series, pd.Series]:
|
||||
"""
|
||||
Returns (z_scores, rolling_mean, rolling_std) for a value series.
|
||||
Z-score = (value - rolling_mean_prev) / rolling_std_prev
|
||||
Uses previous window to avoid look-ahead.
|
||||
"""
|
||||
roll_mean = values.shift(1).rolling(window, min_periods=max(4, window // 3)).mean()
|
||||
roll_std = values.shift(1).rolling(window, min_periods=max(4, window // 3)).std()
|
||||
z = (values - roll_mean) / roll_std.replace(0, float("nan"))
|
||||
return z.round(2), roll_mean.round(4), roll_std.round(4)
|
||||
|
||||
|
||||
def _direction(zscore: float, higher_is_bullish: bool) -> str:
|
||||
if abs(zscore) < 0.5:
|
||||
return "neutral"
|
||||
positive = zscore > 0
|
||||
return "bullish" if positive == higher_is_bullish else "bearish"
|
||||
|
||||
|
||||
# ── Downsample for noisy daily/weekly series ──────────────────────────────────
|
||||
|
||||
def _resample_weekly(df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Resample to weekly (last value of each week, Friday)."""
|
||||
return df.resample("W-FRI").last().dropna()
|
||||
|
||||
|
||||
# ── Main bootstrap ────────────────────────────────────────────────────────────
|
||||
|
||||
def bootstrap_fred(
|
||||
from_date: str = "2020-01-01",
|
||||
series_ids: Optional[List[str]] = None,
|
||||
force: bool = False,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch and store FRED data from from_date to today.
|
||||
Returns summary dict with counts per series.
|
||||
"""
|
||||
from services.database import get_conn
|
||||
import json
|
||||
|
||||
target_series = series_ids or list(FRED_SERIES.keys())
|
||||
results: Dict[str, Any] = {}
|
||||
|
||||
conn = get_conn()
|
||||
|
||||
for sid in target_series:
|
||||
meta = FRED_SERIES.get(sid)
|
||||
if not meta:
|
||||
logger.warning(f"[FRED bootstrap] Unknown series: {sid}")
|
||||
continue
|
||||
|
||||
logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']}) from {from_date}")
|
||||
|
||||
# Fetch one extra year before from_date for z-score warm-up
|
||||
fetch_from = str(date(int(from_date[:4]) - 1, 1, 1))
|
||||
df = _fetch_fred_csv(sid, from_date=fetch_from)
|
||||
if df is None or df.empty:
|
||||
results[sid] = {"status": "fetch_failed", "count": 0}
|
||||
continue
|
||||
|
||||
# For weekly-sampled continuous series, resample
|
||||
if meta["freq"] == "weekly" and len(df) > 200:
|
||||
df = _resample_weekly(df)
|
||||
|
||||
window = meta["zscore_window"]
|
||||
z_series, mean_series, _ = _compute_zscore_series(df["value"], window)
|
||||
|
||||
inserted = 0
|
||||
skipped = 0
|
||||
|
||||
for dt, row in df.iterrows():
|
||||
ev_date = dt.strftime("%Y-%m-%d")
|
||||
# Skip rows before the real from_date (warm-up period)
|
||||
if ev_date < from_date:
|
||||
continue
|
||||
|
||||
val = float(row["value"])
|
||||
z = z_series.get(dt)
|
||||
prev_val = df["value"].shift(1).get(dt)
|
||||
|
||||
if z is None or pd.isna(z):
|
||||
z = 0.0
|
||||
z = float(z)
|
||||
|
||||
prev = float(prev_val) if prev_val is not None and not pd.isna(prev_val) else None
|
||||
surprise_pct = round((val - prev) / abs(prev) * 100, 2) if prev and prev != 0 else None
|
||||
direction = _direction(z, meta["higher_is_bullish"])
|
||||
|
||||
try:
|
||||
conn.execute(
|
||||
"""INSERT INTO economic_events
|
||||
(event_name, series_id, event_date, actual_value, actual_unit,
|
||||
forecast_value, previous_value, surprise_pct, surprise_zscore,
|
||||
surprise_direction, assets_impacted, source)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?)
|
||||
ON CONFLICT(series_id, event_date) DO """ + (
|
||||
"UPDATE SET actual_value=excluded.actual_value, "
|
||||
"previous_value=excluded.previous_value, "
|
||||
"surprise_pct=excluded.surprise_pct, "
|
||||
"surprise_zscore=excluded.surprise_zscore, "
|
||||
"surprise_direction=excluded.surprise_direction, "
|
||||
"fetched_at=datetime('now')"
|
||||
if force else "NOTHING"
|
||||
),
|
||||
(
|
||||
meta["name"], sid, ev_date, val, meta["unit"],
|
||||
None, # no consensus forecast available
|
||||
prev, surprise_pct, z, direction,
|
||||
json.dumps(meta["assets"]),
|
||||
"FRED/bootstrap",
|
||||
),
|
||||
)
|
||||
if conn.execute("SELECT changes()").fetchone()[0]:
|
||||
inserted += 1
|
||||
else:
|
||||
skipped += 1
|
||||
except Exception as e:
|
||||
logger.debug(f"[FRED bootstrap] {sid} {ev_date}: {e}")
|
||||
|
||||
conn.commit()
|
||||
results[sid] = {
|
||||
"status": "ok",
|
||||
"name": meta["name"],
|
||||
"inserted": inserted,
|
||||
"skipped": skipped,
|
||||
"total_fetched": len(df),
|
||||
}
|
||||
logger.info(f"[FRED bootstrap] {sid}: {inserted} inserted, {skipped} skipped")
|
||||
|
||||
conn.close()
|
||||
return results
|
||||
Reference in New Issue
Block a user