diff --git a/backend/services/fred_bootstrap.py b/backend/services/fred_bootstrap.py index 7a01139..abf0143 100644 --- a/backend/services/fred_bootstrap.py +++ b/backend/services/fred_bootstrap.py @@ -4,7 +4,7 @@ from FRED's public CSV endpoint (no API key required) and stores it in the economic_events table with rolling z-score surprises. """ import logging -from datetime import datetime, date +from datetime import date from typing import Any, Dict, List, Optional, Tuple import httpx @@ -13,6 +13,15 @@ import pandas as pd logger = logging.getLogger(__name__) # ── Series catalog ──────────────────────────────────────────────────────────── +# +# transform options: +# None — store raw FRED value as-is +# "yoy_pct" — compute year-over-year %: (val / val_12m_ago - 1) * 100 +# "qoq_annualized"— compute annualized QoQ growth: ((val/val_prev)^4 - 1)*100 +# "div1000" — divide by 1000 (FRED gives raw count, display in K) +# +# delta_absolute: if True, surprise_pct = val - prev (absolute pp change) +# if False, surprise_pct = (val-prev)/|prev| * 100 (% change) FRED_SERIES: Dict[str, Dict[str, Any]] = { "PAYEMS": { @@ -23,6 +32,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": True, "assets": ["SPY", "QQQ", "EURUSD=X", "TLT"], "zscore_window": 12, + "transform": None, + "delta_absolute": False, }, "UNRATE": { "name": "Unemployment Rate", @@ -32,33 +43,41 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": False, "assets": ["SPY", "QQQ", "EURUSD=X"], "zscore_window": 12, + "transform": None, + "delta_absolute": True, # pp change (e.g. 4.1 → 4.0 = -0.1 pp) }, "CPIAUCSL": { - "name": "CPI (All Items YoY)", + "name": "CPI All Items (YoY %)", "unit": "%", "freq": "monthly", "category": "inflation", "higher_is_bullish": False, "assets": ["TLT", "GLD", "EURUSD=X", "SPY"], "zscore_window": 12, + "transform": "yoy_pct", # FRED gives index level → compute YoY + "delta_absolute": True, # pp change between consecutive YoY readings }, "CPILFESL": { - "name": "Core CPI", + "name": "Core CPI (YoY %)", "unit": "%", "freq": "monthly", "category": "inflation", "higher_is_bullish": False, "assets": ["TLT", "GLD", "EURUSD=X"], "zscore_window": 12, + "transform": "yoy_pct", + "delta_absolute": True, }, "PCEPILFE": { - "name": "Core PCE", + "name": "Core PCE (YoY %)", "unit": "%", "freq": "monthly", "category": "inflation", "higher_is_bullish": False, "assets": ["TLT", "GLD", "SPY"], "zscore_window": 12, + "transform": "yoy_pct", + "delta_absolute": True, }, "FEDFUNDS": { "name": "Fed Funds Rate (FOMC)", @@ -68,6 +87,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": False, "assets": ["TLT", "SPY", "EURUSD=X", "GLD"], "zscore_window": 12, + "transform": None, + "delta_absolute": True, # bp/pp move }, "ICSA": { "name": "Initial Jobless Claims", @@ -77,15 +98,20 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": False, "assets": ["SPY", "QQQ"], "zscore_window": 52, + "transform": "div1000", # FRED gives raw count → display in K + "delta_absolute": False, }, - "A191RL1Q225SBEA": { - "name": "GDP Growth (Quarterly)", + "GDPC1": { + # Real GDP level in billions (chained 2017$) → compute QoQ annualized growth + "name": "GDP Growth (QoQ Ann. %)", "unit": "%", "freq": "quarterly", "category": "growth", "higher_is_bullish": True, "assets": ["SPY", "QQQ", "EURUSD=X", "TLT"], "zscore_window": 8, + "transform": "qoq_annualized", + "delta_absolute": True, # pp change between quarterly readings }, "BAMLH0A0HYM2": { "name": "HY Credit Spread (OAS)", @@ -95,6 +121,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": False, "assets": ["HYG", "LQD", "SPY"], "zscore_window": 52, + "transform": None, + "delta_absolute": True, # pp change in spread }, "T10Y2Y": { "name": "Yield Spread 10Y-2Y", @@ -104,6 +132,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": True, "assets": ["TLT", "IEF", "SPY", "HYG"], "zscore_window": 52, + "transform": None, + "delta_absolute": True, }, "T10Y3M": { "name": "Yield Spread 10Y-3M", @@ -113,9 +143,14 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = { "higher_is_bullish": True, "assets": ["TLT", "IEF", "SPY"], "zscore_window": 52, + "transform": None, + "delta_absolute": True, }, } +# Series that were renamed/replaced — will be purged from DB on bootstrap +DEPRECATED_SERIES = ["A191RL1Q225SBEA"] + CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()}) @@ -124,11 +159,7 @@ CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()}) _FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}" -def _fetch_fred_csv(series_id: str, from_date: str = "2019-01-01") -> Optional[pd.DataFrame]: - """ - Download FRED series as CSV. Returns DataFrame with DATE index and VALUE column. - Uses public endpoint — no API key required. - """ +def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]: url = _FRED_CSV_URL.format(series_id=series_id) try: resp = httpx.get(url, timeout=30, follow_redirects=True) @@ -139,20 +170,39 @@ def _fetch_fred_csv(series_id: str, from_date: str = "2019-01-01") -> Optional[p df["value"] = pd.to_numeric(df["value"], errors="coerce") df = df.dropna() df = df[df.index >= pd.Timestamp(from_date)] - df = df.sort_index() - return df + return df.sort_index() except Exception as e: logger.warning(f"[FRED] Failed to fetch {series_id}: {e}") return None -# ── Z-score computation ─────────────────────────────────────────────────────── +# ── Transforms ──────────────────────────────────────────────────────────────── + +def _apply_transform(df: pd.DataFrame, transform: str) -> pd.DataFrame: + """Apply a series-level transform before storing. Drops NaN rows introduced.""" + if transform == "yoy_pct": + # Year-over-year % change from index level + df = df.copy() + df["value"] = (df["value"] / df["value"].shift(12) - 1) * 100 + df = df.dropna() + elif transform == "qoq_annualized": + # Annualized quarter-over-quarter growth rate + df = df.copy() + ratio = df["value"] / df["value"].shift(1) + df["value"] = (ratio ** 4 - 1) * 100 + df = df.dropna() + elif transform == "div1000": + df = df.copy() + df["value"] = df["value"] / 1000.0 + return df + + +# ── Z-score ─────────────────────────────────────────────────────────────────── def _compute_zscore_series(values: pd.Series, window: int) -> Tuple[pd.Series, pd.Series, pd.Series]: """ - Returns (z_scores, rolling_mean, rolling_std) for a value series. - Z-score = (value - rolling_mean_prev) / rolling_std_prev - Uses previous window to avoid look-ahead. + Z-score using previous-window only (no look-ahead). + Z = (value - rolling_mean_prev_N) / rolling_std_prev_N """ roll_mean = values.shift(1).rolling(window, min_periods=max(4, window // 3)).mean() roll_std = values.shift(1).rolling(window, min_periods=max(4, window // 3)).std() @@ -163,14 +213,10 @@ def _compute_zscore_series(values: pd.Series, window: int) -> Tuple[pd.Series, p def _direction(zscore: float, higher_is_bullish: bool) -> str: if abs(zscore) < 0.5: return "neutral" - positive = zscore > 0 - return "bullish" if positive == higher_is_bullish else "bearish" + return "bullish" if (zscore > 0) == higher_is_bullish else "bearish" -# ── Downsample for noisy daily/weekly series ────────────────────────────────── - def _resample_weekly(df: pd.DataFrame) -> pd.DataFrame: - """Resample to weekly (last value of each week, Friday).""" return df.resample("W-FRI").last().dropna() @@ -183,6 +229,7 @@ def bootstrap_fred( ) -> Dict[str, Any]: """ Fetch and store FRED data from from_date to today. + Transforms are applied before storing (YoY%, QoQ Ann., /1000). Returns summary dict with counts per series. """ from services.database import get_conn @@ -190,50 +237,73 @@ def bootstrap_fred( target_series = series_ids or list(FRED_SERIES.keys()) results: Dict[str, Any] = {} - conn = get_conn() + # Purge deprecated series from DB + for dep in DEPRECATED_SERIES: + try: + conn.execute("DELETE FROM economic_events WHERE series_id=?", (dep,)) + except Exception: + pass + conn.commit() + for sid in target_series: meta = FRED_SERIES.get(sid) if not meta: logger.warning(f"[FRED bootstrap] Unknown series: {sid}") continue - logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']}) from {from_date}") + logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']})") + + # Warm-up period: extra years before from_date for transform + z-score + transform = meta.get("transform") + extra_years = 3 if transform == "qoq_annualized" else (2 if transform == "yoy_pct" else 1) + fetch_from = str(date(int(from_date[:4]) - extra_years, 1, 1)) - # Fetch one extra year before from_date for z-score warm-up - fetch_from = str(date(int(from_date[:4]) - 1, 1, 1)) df = _fetch_fred_csv(sid, from_date=fetch_from) if df is None or df.empty: results[sid] = {"status": "fetch_failed", "count": 0} continue - # For weekly-sampled continuous series, resample + # Apply transform on full (warm-up included) dataset + if transform: + df = _apply_transform(df, transform) + if df.empty: + results[sid] = {"status": "transform_empty", "count": 0} + continue + + # Downsample continuous daily/weekly series to weekly if meta["freq"] == "weekly" and len(df) > 200: df = _resample_weekly(df) + # Z-score on transformed values window = meta["zscore_window"] - z_series, mean_series, _ = _compute_zscore_series(df["value"], window) + z_series, _, _ = _compute_zscore_series(df["value"], window) + delta_absolute = meta.get("delta_absolute", False) inserted = 0 skipped = 0 for dt, row in df.iterrows(): ev_date = dt.strftime("%Y-%m-%d") - # Skip rows before the real from_date (warm-up period) if ev_date < from_date: - continue + continue # warm-up only, don't store - val = float(row["value"]) - z = z_series.get(dt) - prev_val = df["value"].shift(1).get(dt) - - if z is None or pd.isna(z): + val = float(row["value"]) + z = float(z_series.get(dt) or 0.0) + if pd.isna(z): z = 0.0 - z = float(z) - prev = float(prev_val) if prev_val is not None and not pd.isna(prev_val) else None - surprise_pct = round((val - prev) / abs(prev) * 100, 2) if prev and prev != 0 else None + prev_raw = df["value"].shift(1).get(dt) + prev = float(prev_raw) if prev_raw is not None and not pd.isna(prev_raw) else None + + if delta_absolute: + # pp / absolute change (for rates, spreads, YoY series) + surprise_pct = round(val - prev, 4) if prev is not None else None + else: + # % change (for levels: NFP, ICSA in K, etc.) + surprise_pct = round((val - prev) / abs(prev) * 100, 2) if prev and prev != 0 else None + direction = _direction(z, meta["higher_is_bullish"]) try: @@ -253,9 +323,8 @@ def bootstrap_fred( if force else "NOTHING" ), ( - meta["name"], sid, ev_date, val, meta["unit"], - None, # no consensus forecast available - prev, surprise_pct, z, direction, + meta["name"], sid, ev_date, round(val, 4), meta["unit"], + None, prev, surprise_pct, z, direction, json.dumps(meta["assets"]), "FRED/bootstrap", ), @@ -270,10 +339,10 @@ def bootstrap_fred( conn.commit() results[sid] = { "status": "ok", - "name": meta["name"], + "name": meta["name"], "inserted": inserted, - "skipped": skipped, - "total_fetched": len(df), + "skipped": skipped, + "total_fetched": len(df[df.index >= pd.Timestamp(from_date)]), } logger.info(f"[FRED bootstrap] {sid}: {inserted} inserted, {skipped} skipped") diff --git a/frontend/src/pages/CalendarPage.tsx b/frontend/src/pages/CalendarPage.tsx index 36804a0..572f62b 100644 --- a/frontend/src/pages/CalendarPage.tsx +++ b/frontend/src/pages/CalendarPage.tsx @@ -118,12 +118,19 @@ function DirBadge({ dir }: { dir: string | null }) { ) } -function SurprisePct({ v }: { v: number | null }) { +// For rate/spread series (unit % or pp): delta_absolute=true → show absolute pp change +// For level series (K, raw): delta_absolute=false → show relative % change +const DELTA_ABSOLUTE_UNITS = new Set(['%', 'pp']) + +function SurprisePct({ v, unit }: { v: number | null; unit: string }) { if (v === null || v === undefined) return — + const isPp = DELTA_ABSOLUTE_UNITS.has(unit) const sign = v > 0 ? '+' : '' + const decimals = isPp ? 2 : 1 + const suffix = isPp ? ' pp' : '%' return ( 0 ? 'text-emerald-400' : v < 0 ? 'text-red-400' : 'text-slate-400')}> - {sign}{v.toFixed(1)}% + {sign}{v.toFixed(decimals)}{suffix} ) } @@ -510,7 +517,7 @@ export default function CalendarPage() { Actuel Précédent - Δ% + Δ vs préc. toggleSort('zscore')}> Z-Score @@ -542,7 +549,7 @@ export default function CalendarPage() { {fmt(ev.previous_value)} - +