OpenFin/backend/services/fred_fetcher.py

"""
FRED API fetcher — récupère les dernières releases macro US.

Séries suivies :
  CPIAUCSL   → CPI mensuel (YoY calculé)
  PAYEMS     → Non-Farm Payrolls (variation mensuelle k emplois)
  UNRATE     → Taux de chômage
  FEDFUNDS   → Fed Funds Rate
  GDP        → PIB US trimestriel (croissance %)
  ICSA       → Initial Jobless Claims (hebdo)
  T10Y2Y     → Spread 10Y-2Y (courbe des taux)
  DEXUSEU    → EUR/USD (proxy macro)
"""
from __future__ import annotations

import logging
from datetime import datetime, timedelta
from typing import Any, Dict, List, Optional

logger = logging.getLogger("fred_fetcher")

# Series config : (id, label, unit, asset_impact, direction_interpretation)
FRED_SERIES = [
    ("CPIAUCSL", "US CPI (inflation)",      "index",  ["metals", "rates", "forex"],  "higher_bearish"),
    ("PAYEMS",   "US Non-Farm Payrolls",     "k jobs", ["indices", "forex", "rates"], "higher_bullish"),
    ("UNRATE",   "US Unemployment Rate",     "%",      ["indices", "forex"],          "higher_bearish"),
    ("FEDFUNDS", "Fed Funds Rate",           "%",      ["bonds", "forex", "indices"], "higher_bearish_growth"),
    ("GDP",      "US GDP Growth",            "bn$",    ["indices", "forex"],          "higher_bullish"),
    ("ICSA",     "US Initial Jobless Claims","k claims",["indices", "forex"],         "higher_bearish"),
    ("T10Y2Y",   "10Y-2Y Spread (courbe)",  "%",      ["bonds", "indices"],          "positive_bullish"),
]


def _get_fred_key() -> Optional[str]:
    try:
        from services.database import get_config
        return get_config("fred_api_key") or None
    except Exception:
        return None


def _fetch_series_observations(series_id: str, api_key: str, count: int = 3) -> List[Dict]:
    """Fetch last N observations from FRED for a given series."""
    import urllib.request
    import urllib.parse
    import json

    params = urllib.parse.urlencode({
        "series_id": series_id,
        "api_key": api_key,
        "file_type": "json",
        "sort_order": "desc",
        "limit": count,
        "observation_start": (datetime.utcnow() - timedelta(days=730)).strftime("%Y-%m-%d"),
    })
    url = f"https://api.stlouisfed.org/fred/series/observations?{params}"
    try:
        with urllib.request.urlopen(url, timeout=8) as resp:
            data = json.loads(resp.read())
        obs = data.get("observations", [])
        return [{"date": o["date"], "value": o["value"]} for o in obs if o.get("value") not in (".", None)]
    except Exception as e:
        logger.warning(f"[FRED] {series_id} fetch failed: {e}")
        return []


def _parse_value(v: str) -> Optional[float]:
    try:
        return float(v)
    except (ValueError, TypeError):
        return None


def _surprise_direction(series_id: str, current: float, previous: float, direction_hint: str) -> str:
    """Returns 'bullish', 'bearish', or 'neutral' based on change direction."""
    delta = current - previous
    if abs(delta) < 0.01:
        return "neutral"
    if direction_hint == "higher_bullish":
        return "bullish" if delta > 0 else "bearish"
    elif direction_hint in ("higher_bearish", "higher_bearish_growth"):
        return "bearish" if delta > 0 else "bullish"
    elif direction_hint == "positive_bullish":
        return "bullish" if current > 0 else "bearish"
    return "neutral"


def _yoy_change(obs: List[Dict]) -> Optional[float]:
    """For monthly series, compute rough YoY % if we have ≥12 month history."""
    if len(obs) < 2:
        return None
    v_latest = _parse_value(obs[0]["value"])
    v_prev = _parse_value(obs[-1]["value"])
    if v_latest is None or v_prev is None or v_prev == 0:
        return None
    return round((v_latest - v_prev) / abs(v_prev) * 100, 2)


def get_fred_recent_releases() -> List[Dict[str, Any]]:
    """
    Fetch latest FRED data for key macro series.

    Returns a list of dicts, one per series, with:
      series_id, label, unit, latest_date, latest_value,
      previous_value, change, change_pct, direction, asset_impact
    """
    api_key = _get_fred_key()
    if not api_key:
        return []

    results = []
    for series_id, label, unit, asset_impact, direction_hint in FRED_SERIES:
        obs = _fetch_series_observations(series_id, api_key, count=13)  # 13 for YoY
        if not obs:
            continue
        latest = obs[0]
        previous = obs[1] if len(obs) > 1 else None

        v_latest = _parse_value(latest["value"])
        v_prev = _parse_value(previous["value"]) if previous else None

        if v_latest is None:
            continue

        change = round(v_latest - v_prev, 4) if v_prev is not None else None
        change_pct = round((v_latest - v_prev) / abs(v_prev) * 100, 2) if v_prev and v_prev != 0 else None
        direction = _surprise_direction(series_id, v_latest, v_prev, direction_hint) if v_prev is not None else "neutral"

        # For CPI, compute YoY
        display_value = v_latest
        display_unit = unit
        if series_id == "CPIAUCSL" and len(obs) >= 13:
            yoy = _yoy_change(obs[:13])
            if yoy is not None:
                display_value = yoy
                display_unit = "% YoY"
                # Re-compute change vs previous month's YoY
                if len(obs) >= 14:
                    yoy_prev = _yoy_change(obs[1:14])
                    if yoy_prev is not None:
                        change = round(yoy - yoy_prev, 3)
                        direction = "bearish" if change > 0 else "bullish"  # higher CPI = bearish markets

        entry: Dict[str, Any] = {
            "series_id": series_id,
            "label": label,
            "unit": display_unit,
            "latest_date": latest["date"],
            "latest_value": display_value,
            "previous_value": v_prev,
            "change": change,
            "direction": direction,
            "asset_impact": asset_impact,
        }
        results.append(entry)

    return results


def _compute_zscore_surprise(series_id: str, api_key: str, latest_value: float) -> tuple:
    """Compute z-score of latest value vs 12-period MA. Returns (forecast, surprise_pct, zscore)."""
    import math
    obs = _fetch_series_observations(series_id, api_key, count=14)
    values = [_parse_value(o["value"]) for o in obs if _parse_value(o["value"]) is not None]
    if len(values) < 3:
        return None, None, None
    # Use obs[1:] as history (exclude latest), up to 12 periods
    history = values[1:13]
    if not history:
        return None, None, None
    mean = sum(history) / len(history)
    variance = sum((x - mean) ** 2 for x in history) / len(history)
    std = math.sqrt(variance) if variance > 0 else 0.0
    surprise_pct = round((latest_value - mean) / abs(mean) * 100, 2) if mean != 0 else 0.0
    zscore = round((latest_value - mean) / std, 2) if std > 0 else 0.0
    return round(mean, 4), surprise_pct, zscore


def save_fred_releases_to_db(releases: List[Dict], api_key: str = "") -> int:
    """Persist FRED releases to economic_events table with surprise scores. Returns count saved."""
    from services.database import save_economic_event
    import json as _json

    # Map series_id → assets_impacted from FRED_SERIES config
    _assets_map = {sid: assets for sid, _, _, assets, _ in FRED_SERIES}

    saved = 0
    for r in releases:
        sid = r.get("series_id", "")
        if not sid or r.get("latest_value") is None:
            continue
        # Compute z-score surprise if api_key available
        forecast_val, surprise_pct, zscore = (None, None, None)
        if api_key:
            try:
                forecast_val, surprise_pct, zscore = _compute_zscore_surprise(sid, api_key, r["latest_value"])
            except Exception:
                pass

        try:
            save_economic_event(
                event_name=r.get("label", sid),
                series_id=sid,
                event_date=r.get("latest_date", ""),
                actual_value=r.get("latest_value"),
                actual_unit=r.get("unit", ""),
                forecast_value=forecast_val,
                previous_value=r.get("previous_value"),
                surprise_pct=surprise_pct,
                surprise_zscore=zscore,
                surprise_direction=r.get("direction", "neutral"),
                assets_impacted=_assets_map.get(sid, []),
                source="FRED",
            )
            saved += 1
        except Exception as e:
            logger.warning(f"[FRED] Failed to save {sid} to DB: {e}")

    return saved


def build_fred_context_block(releases: List[Dict]) -> str:
    """Format FRED releases as a prompt-ready string with surprise scoring."""
    if not releases:
        return ""
    lines = ["## RECENT MACRO DATA (FRED — latest releases)"]
    for r in releases:
        val_str = f"{r['latest_value']:.2f} {r['unit']}" if isinstance(r.get("latest_value"), float) else str(r.get("latest_value", "N/A"))
        chg_str = ""
        if r.get("change") is not None:
            arrow = "+" if r["change"] > 0 else ""
            chg_str = f" ({arrow}{r['change']:.3f} vs prior)"
        surprise_str = ""
        if r.get("surprise_zscore") is not None:
            z = r["surprise_zscore"]
            if abs(z) >= 1.5:
                surprise_str = f" ⚡ SURPRISE z={z:+.1f}"
            elif abs(z) >= 0.8:
                surprise_str = f" (notable z={z:+.1f})"
        dir_tag = {"bullish": "[BULLISH]", "bearish": "[BEARISH]", "neutral": "[NEUTRAL]"}.get(r.get("direction", "neutral"), "")
        lines.append(
            f"  {r['label']} [{r['latest_date']}]: {val_str}{chg_str}{surprise_str} → {dir_tag}"
        )
    lines.append("Use these figures to assess whether current market pricing already reflects macro reality.")
    return "\n".join(lines)


def build_economic_surprise_block(days: int = 14) -> str:
    """Build a prompt block from stored economic_events table — recent surprises highlighted."""
    try:
        from services.database import get_recent_economic_surprises
        events = get_recent_economic_surprises(days=days, min_zscore=0.0)
        if not events:
            return ""
        lines = [f"## ECONOMIC SURPRISE TRACKER (last {days} days — FRED actuals vs trend baseline)"]
        for ev in events[:8]:
            z = ev.get("surprise_zscore") or 0.0
            sp = ev.get("surprise_pct") or 0.0
            direction = ev.get("surprise_direction", "neutral")
            actual = ev.get("actual_value")
            unit = ev.get("actual_unit", "")
            forecast = ev.get("forecast_value")
            flag = ""
            if abs(z) >= 1.5:
                flag = " ⚡ SIGNIFICANT SURPRISE"
            elif abs(z) >= 0.8:
                flag = " (notable)"
            actual_str = f"{actual:.2f}{unit}" if actual is not None else "N/A"
            forecast_str = f"{forecast:.2f}{unit}" if forecast is not None else "N/A"
            dir_tag = {"bullish": "BULLISH", "bearish": "BEARISH", "neutral": "NEUTRAL"}.get(direction, "")
            lines.append(
                f"  [{ev['event_date']}] {ev['event_name']}: actual={actual_str} vs baseline={forecast_str}"
                f" (z={z:+.2f}, {sp:+.1f}%) → {dir_tag}{flag}"
            )
        lines.append("→ High z-scores = unexpected move vs recent trend → potential mispricing in related assets")
        return "\n".join(lines)
    except Exception as e:
        logger.warning(f"[FRED] build_economic_surprise_block failed: {e}")
        return ""