admin/OpenFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: FRED data quality — GDP growth %, CPI/PCE YoY%, ICSA in K

Browse Source 
- GDPC1 replaces A191RL1Q225SBEA: compute annualized QoQ growth from
  GDP level ((val/prev)^4 - 1)*100 → displays proper ~2-3% not 31 819
- CPIAUCSL/CPILFESL/PCEPILFE: yoy_pct transform (val/val_12m_ago-1)*100
  → displays 3.x% YoY inflation, not raw index level 334
- ICSA: div1000 transform → displays 226 K claims, not 226 000 K
- delta_absolute flag: pp change for rate/% series, % change for levels
- SurprisePct component: shows 'pp' suffix for %, '%' for K/levels
- Column header renamed from 'Δ%' to 'Δ vs préc.' with tooltip
- Deprecated A191RL1Q225SBEA rows cleaned from DB on next bootstrap
- Warm-up periods: 2yr for yoy_pct, 3yr for qoq_annualized, 1yr others

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-26 15:16:44 +02:00
parent 2469b76769
commit d25ca92694
2 changed files with 124 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

157
backend/services/fred_bootstrap.py
View File

@@ -4,7 +4,7 @@ from FRED's public CSV endpoint (no API key required) and stores it in
the economic_events table with rolling z-score surprises.
"""
import logging
from datetime import datetime, date
from datetime import date
from typing import Any, Dict, List, Optional, Tuple
import httpx
@@ -13,6 +13,15 @@ import pandas as pd
logger = logging.getLogger(__name__)
# ── Series catalog ────────────────────────────────────────────────────────────
#
# transform options:
# None — store raw FRED value as-is
# "yoy_pct" — compute year-over-year %: (val / val_12m_ago - 1) * 100
# "qoq_annualized"— compute annualized QoQ growth: ((val/val_prev)^4 - 1)*100
# "div1000" — divide by 1000 (FRED gives raw count, display in K)
#
# delta_absolute: if True, surprise_pct = val - prev (absolute pp change)
# if False, surprise_pct = (val-prev)/|prev| * 100 (% change)
FRED_SERIES: Dict[str, Dict[str, Any]] = {
"PAYEMS": {
@@ -23,6 +32,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": True,
"assets": ["SPY", "QQQ", "EURUSD=X", "TLT"],
"zscore_window": 12,
"transform": None,
"delta_absolute": False,
},
"UNRATE": {
"name": "Unemployment Rate",
@@ -32,33 +43,41 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": False,
"assets": ["SPY", "QQQ", "EURUSD=X"],
"zscore_window": 12,
"transform": None,
"delta_absolute": True, # pp change (e.g. 4.1 → 4.0 = -0.1 pp)
},
"CPIAUCSL": {
"name": "CPI (All Items YoY)",
"name": "CPI All Items (YoY %)",
"unit": "%",
"freq": "monthly",
"category": "inflation",
"higher_is_bullish": False,
"assets": ["TLT", "GLD", "EURUSD=X", "SPY"],
"zscore_window": 12,
"transform": "yoy_pct", # FRED gives index level → compute YoY
"delta_absolute": True, # pp change between consecutive YoY readings
},
"CPILFESL": {
"name": "Core CPI",
"name": "Core CPI (YoY %)",
"unit": "%",
"freq": "monthly",
"category": "inflation",
"higher_is_bullish": False,
"assets": ["TLT", "GLD", "EURUSD=X"],
"zscore_window": 12,
"transform": "yoy_pct",
"delta_absolute": True,
},
"PCEPILFE": {
"name": "Core PCE",
"name": "Core PCE (YoY %)",
"unit": "%",
"freq": "monthly",
"category": "inflation",
"higher_is_bullish": False,
"assets": ["TLT", "GLD", "SPY"],
"zscore_window": 12,
"transform": "yoy_pct",
"delta_absolute": True,
},
"FEDFUNDS": {
"name": "Fed Funds Rate (FOMC)",
@@ -68,6 +87,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": False,
"assets": ["TLT", "SPY", "EURUSD=X", "GLD"],
"zscore_window": 12,
"transform": None,
"delta_absolute": True, # bp/pp move
},
"ICSA": {
"name": "Initial Jobless Claims",
@@ -77,15 +98,20 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": False,
"assets": ["SPY", "QQQ"],
"zscore_window": 52,
"transform": "div1000", # FRED gives raw count → display in K
"delta_absolute": False,
},
"A191RL1Q225SBEA": {
"name": "GDP Growth (Quarterly)",
"GDPC1": {
# Real GDP level in billions (chained 2017$) → compute QoQ annualized growth
"name": "GDP Growth (QoQ Ann. %)",
"unit": "%",
"freq": "quarterly",
"category": "growth",
"higher_is_bullish": True,
"assets": ["SPY", "QQQ", "EURUSD=X", "TLT"],
"zscore_window": 8,
"transform": "qoq_annualized",
"delta_absolute": True, # pp change between quarterly readings
},
"BAMLH0A0HYM2": {
"name": "HY Credit Spread (OAS)",
@@ -95,6 +121,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": False,
"assets": ["HYG", "LQD", "SPY"],
"zscore_window": 52,
"transform": None,
"delta_absolute": True, # pp change in spread
},
"T10Y2Y": {
"name": "Yield Spread 10Y-2Y",
@@ -104,6 +132,8 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": True,
"assets": ["TLT", "IEF", "SPY", "HYG"],
"zscore_window": 52,
"transform": None,
"delta_absolute": True,
},
"T10Y3M": {
"name": "Yield Spread 10Y-3M",
@@ -113,9 +143,14 @@ FRED_SERIES: Dict[str, Dict[str, Any]] = {
"higher_is_bullish": True,
"assets": ["TLT", "IEF", "SPY"],
"zscore_window": 52,
"transform": None,
"delta_absolute": True,
},
}
# Series that were renamed/replaced — will be purged from DB on bootstrap
DEPRECATED_SERIES = ["A191RL1Q225SBEA"]
CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
@@ -124,11 +159,7 @@ CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
def _fetch_fred_csv(series_id: str, from_date: str = "2019-01-01") -> Optional[pd.DataFrame]:
"""
Download FRED series as CSV. Returns DataFrame with DATE index and VALUE column.
Uses public endpoint — no API key required.
"""
def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
url = _FRED_CSV_URL.format(series_id=series_id)
try:
resp = httpx.get(url, timeout=30, follow_redirects=True)
@@ -139,20 +170,39 @@ def _fetch_fred_csv(series_id: str, from_date: str = "2019-01-01") -> Optional[p
df["value"] = pd.to_numeric(df["value"], errors="coerce")
df = df.dropna()
df = df[df.index >= pd.Timestamp(from_date)]
df = df.sort_index()
return df
return df.sort_index()
except Exception as e:
logger.warning(f"[FRED] Failed to fetch {series_id}: {e}")
return None
# ── Z-score computation ───────────────────────────────────────────────────────
# ── Transforms ────────────────────────────────────────────────────────────────
def _apply_transform(df: pd.DataFrame, transform: str) -> pd.DataFrame:
"""Apply a series-level transform before storing. Drops NaN rows introduced."""
if transform == "yoy_pct":
# Year-over-year % change from index level
df = df.copy()
df["value"] = (df["value"] / df["value"].shift(12) - 1) * 100
df = df.dropna()
elif transform == "qoq_annualized":
# Annualized quarter-over-quarter growth rate
df = df.copy()
ratio = df["value"] / df["value"].shift(1)
df["value"] = (ratio ** 4 - 1) * 100
df = df.dropna()
elif transform == "div1000":
df = df.copy()
df["value"] = df["value"] / 1000.0
return df
# ── Z-score ───────────────────────────────────────────────────────────────────
def _compute_zscore_series(values: pd.Series, window: int) -> Tuple[pd.Series, pd.Series, pd.Series]:
"""
Returns (z_scores, rolling_mean, rolling_std) for a value series.
Z-score = (value - rolling_mean_prev) / rolling_std_prev
Uses previous window to avoid look-ahead.
Z-score using previous-window only (no look-ahead).
Z = (value - rolling_mean_prev_N) / rolling_std_prev_N
"""
roll_mean = values.shift(1).rolling(window, min_periods=max(4, window // 3)).mean()
roll_std = values.shift(1).rolling(window, min_periods=max(4, window // 3)).std()
@@ -163,14 +213,10 @@ def _compute_zscore_series(values: pd.Series, window: int) -> Tuple[pd.Series, p
def _direction(zscore: float, higher_is_bullish: bool) -> str:
if abs(zscore) < 0.5:
return "neutral"
positive = zscore > 0
return "bullish" if positive == higher_is_bullish else "bearish"
return "bullish" if (zscore > 0) == higher_is_bullish else "bearish"
# ── Downsample for noisy daily/weekly series ──────────────────────────────────
def _resample_weekly(df: pd.DataFrame) -> pd.DataFrame:
"""Resample to weekly (last value of each week, Friday)."""
return df.resample("W-FRI").last().dropna()
@@ -183,6 +229,7 @@ def bootstrap_fred(
) -> Dict[str, Any]:
"""
Fetch and store FRED data from from_date to today.
Transforms are applied before storing (YoY%, QoQ Ann., /1000).
Returns summary dict with counts per series.
"""
from services.database import get_conn
@@ -190,50 +237,73 @@ def bootstrap_fred(
target_series = series_ids or list(FRED_SERIES.keys())
results: Dict[str, Any] = {}
conn = get_conn()
# Purge deprecated series from DB
for dep in DEPRECATED_SERIES:
try:
conn.execute("DELETE FROM economic_events WHERE series_id=?", (dep,))
except Exception:
pass
conn.commit()
for sid in target_series:
meta = FRED_SERIES.get(sid)
if not meta:
logger.warning(f"[FRED bootstrap] Unknown series: {sid}")
continue
logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']}) from {from_date}")
logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']})")
# Warm-up period: extra years before from_date for transform + z-score
transform = meta.get("transform")
extra_years = 3 if transform == "qoq_annualized" else (2 if transform == "yoy_pct" else 1)
fetch_from = str(date(int(from_date[:4]) - extra_years, 1, 1))
# Fetch one extra year before from_date for z-score warm-up
fetch_from = str(date(int(from_date[:4]) - 1, 1, 1))
df = _fetch_fred_csv(sid, from_date=fetch_from)
if df is None or df.empty:
results[sid] = {"status": "fetch_failed", "count": 0}
continue
# For weekly-sampled continuous series, resample
# Apply transform on full (warm-up included) dataset
if transform:
df = _apply_transform(df, transform)
if df.empty:
results[sid] = {"status": "transform_empty", "count": 0}
continue
# Downsample continuous daily/weekly series to weekly
if meta["freq"] == "weekly" and len(df) > 200:
df = _resample_weekly(df)
# Z-score on transformed values
window = meta["zscore_window"]
z_series, mean_series, _ = _compute_zscore_series(df["value"], window)
z_series, _, _ = _compute_zscore_series(df["value"], window)
delta_absolute = meta.get("delta_absolute", False)
inserted = 0
skipped = 0
for dt, row in df.iterrows():
ev_date = dt.strftime("%Y-%m-%d")
# Skip rows before the real from_date (warm-up period)
if ev_date < from_date:
continue
continue # warm-up only, don't store
val = float(row["value"])
z = z_series.get(dt)
prev_val = df["value"].shift(1).get(dt)
if z is None or pd.isna(z):
val = float(row["value"])
z = float(z_series.get(dt) or 0.0)
if pd.isna(z):
z = 0.0
z = float(z)
prev = float(prev_val) if prev_val is not None and not pd.isna(prev_val) else None
surprise_pct = round((val - prev) / abs(prev) * 100, 2) if prev and prev != 0 else None
prev_raw = df["value"].shift(1).get(dt)
prev = float(prev_raw) if prev_raw is not None and not pd.isna(prev_raw) else None
if delta_absolute:
# pp / absolute change (for rates, spreads, YoY series)
surprise_pct = round(val - prev, 4) if prev is not None else None
else:
# % change (for levels: NFP, ICSA in K, etc.)
surprise_pct = round((val - prev) / abs(prev) * 100, 2) if prev and prev != 0 else None
direction = _direction(z, meta["higher_is_bullish"])
try:
@@ -253,9 +323,8 @@ def bootstrap_fred(
if force else "NOTHING"
),
(
meta["name"], sid, ev_date, val, meta["unit"],
None, # no consensus forecast available
prev, surprise_pct, z, direction,
meta["name"], sid, ev_date, round(val, 4), meta["unit"],
None, prev, surprise_pct, z, direction,
json.dumps(meta["assets"]),
"FRED/bootstrap",
),
@@ -270,10 +339,10 @@ def bootstrap_fred(
conn.commit()
results[sid] = {
"status": "ok",
"name": meta["name"],
"name": meta["name"],
"inserted": inserted,
"skipped": skipped,
"total_fetched": len(df),
"skipped": skipped,
"total_fetched": len(df[df.index >= pd.Timestamp(from_date)]),
}
logger.info(f"[FRED bootstrap] {sid}: {inserted} inserted, {skipped} skipped")