From b058b7297a7305e6078e4e9725e576a5a4b1d6b3 Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Fri, 26 Jun 2026 15:47:16 +0200 Subject: [PATCH] fix: use FRED official JSON API (bypasses CloudFlare) + API key UI - fred_bootstrap.py: switch from CSV graph endpoint (CloudFlare-blocked) to api.stlouisfed.org/fred/series/observations JSON API; reads key from DB config 'fred_api_key'; returns clear error if key missing - eco.py: add GET/POST /api/eco/fred-key to check/save the FRED API key - CalendarPage.tsx: BootstrapPanel shows API key section with status, input to paste key + save button, disables Launch if key missing Free key: fred.stlouisfed.org -> My Account -> API Keys Co-Authored-By: Claude Sonnet 4.6 --- backend/routers/eco.py | 22 +++++++ backend/services/fred_bootstrap.py | 94 ++++++++++++++++++----------- frontend/src/pages/CalendarPage.tsx | 82 ++++++++++++++++++++++--- 3 files changed, 156 insertions(+), 42 deletions(-) diff --git a/backend/routers/eco.py b/backend/routers/eco.py index 7e0201b..2b19d64 100644 --- a/backend/routers/eco.py +++ b/backend/routers/eco.py @@ -48,6 +48,28 @@ def _run_bootstrap(from_date: str, series_ids: Optional[List[str]], force: bool) _bootstrap_status["running"] = False +@router.get("/fred-key") +def get_fred_key() -> Dict[str, Any]: + """Check if a FRED API key is configured.""" + from services.database import get_config + key = get_config("fred_api_key") or "" + return { + "configured": bool(key), + "preview": (key[:6] + "…" + key[-4:]) if len(key) > 10 else ("***" if key else ""), + } + + +@router.post("/fred-key") +def save_fred_key(key: str = Query(..., description="FRED API key")) -> Dict[str, Any]: + """Save FRED API key to DB config.""" + from services.database import set_config + key = key.strip() + if not key: + raise HTTPException(400, "Empty key") + set_config("fred_api_key", key) + return {"status": "saved", "preview": key[:6] + "…" + key[-4:]} + + @router.post("/bootstrap") def bootstrap( background_tasks: BackgroundTasks, diff --git a/backend/services/fred_bootstrap.py b/backend/services/fred_bootstrap.py index 2cbb974..5aba140 100644 --- a/backend/services/fred_bootstrap.py +++ b/backend/services/fred_bootstrap.py @@ -154,41 +154,55 @@ DEPRECATED_SERIES = ["A191RL1Q225SBEA"] CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()}) -# ── FRED fetch ──────────────────────────────────────────────────────────────── +# ── FRED fetch (official JSON API — bypasses CloudFlare) ────────────────────── +# +# The public CSV endpoint (fred.stlouisfed.org/graph/fredgraph.csv) is blocked +# by CloudFlare on server IPs. Use the official REST API instead: +# https://api.stlouisfed.org/fred/series/observations +# Free API key: https://fred.stlouisfed.org/docs/api/api_key.html -_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}" +_FRED_API_URL = "https://api.stlouisfed.org/fred/series/observations" -_HEADERS = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/124.0.0.0 Safari/537.36" - ), - "Accept": "text/csv,text/plain,*/*", - "Accept-Language": "en-US,en;q=0.9", -} - - -def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]: - url = _FRED_CSV_URL.format(series_id=series_id) - print(f"[FRED] Fetching {series_id} from {url}", flush=True) +def _fetch_fred_api(series_id: str, from_date: str, api_key: str) -> Optional[pd.DataFrame]: + params = { + "series_id": series_id, + "api_key": api_key, + "observation_start": from_date, + "file_type": "json", + "sort_order": "asc", + } + print(f"[FRED API] Fetching {series_id} from {from_date}", flush=True) try: - with httpx.Client(timeout=httpx.Timeout(10.0, connect=10.0), follow_redirects=True) as client: - resp = client.get(url, headers=_HEADERS) - print(f"[FRED] {series_id} status={resp.status_code} len={len(resp.content)}", flush=True) + resp = httpx.get(_FRED_API_URL, params=params, timeout=30, follow_redirects=True) + print(f"[FRED API] {series_id} HTTP {resp.status_code}", flush=True) resp.raise_for_status() - from io import StringIO - df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE") - df.columns = ["value"] - df["value"] = pd.to_numeric(df["value"], errors="coerce") - df = df.dropna() - df = df[df.index >= pd.Timestamp(from_date)] - print(f"[FRED] {series_id}: {len(df)} rows from {from_date}", flush=True) - return df.sort_index() + data = resp.json() + + if "error_message" in data: + print(f"[FRED API] {series_id} API error: {data['error_message']}", flush=True) + return None + + records = [] + for obs in data.get("observations", []): + val_str = obs.get("value", ".") + if val_str == ".": # FRED uses "." for missing/unreleased values + continue + try: + records.append({"date": pd.Timestamp(obs["date"]), "value": float(val_str)}) + except (ValueError, KeyError): + continue + + if not records: + print(f"[FRED API] {series_id}: no valid observations", flush=True) + return None + + df = pd.DataFrame(records).set_index("date").sort_index() + print(f"[FRED API] {series_id}: {len(df)} rows OK", flush=True) + return df except Exception as e: - print(f"[FRED] ERROR {series_id}: {type(e).__name__}: {e}", flush=True) - logger.warning(f"[FRED] Failed to fetch {series_id}: {e}") + print(f"[FRED API] ERROR {series_id}: {type(e).__name__}: {e}", flush=True) + logger.warning(f"[FRED API] Failed {series_id}: {e}") return None @@ -242,15 +256,27 @@ def bootstrap_fred( from_date: str = "2020-01-01", series_ids: Optional[List[str]] = None, force: bool = False, + api_key: Optional[str] = None, ) -> Dict[str, Any]: """ Fetch and store FRED data from from_date to today. - Transforms are applied before storing (YoY%, QoQ Ann., /1000). - Returns summary dict with counts per series. + Requires a free FRED API key (fred.stlouisfed.org/docs/api/api_key.html). + If api_key is not passed, reads from DB config key 'fred_api_key'. """ - from services.database import get_conn + from services.database import get_conn, get_config import json + # Resolve API key + key = api_key or get_config("fred_api_key") or "" + if not key: + return { + "_error": ( + "Clé API FRED manquante. " + "Inscris-toi gratuitement sur fred.stlouisfed.org/docs/api/api_key.html " + "puis saisis la clé dans le panel Bootstrap." + ) + } + target_series = series_ids or list(FRED_SERIES.keys()) results: Dict[str, Any] = {} conn = get_conn() @@ -269,14 +295,12 @@ def bootstrap_fred( logger.warning(f"[FRED bootstrap] Unknown series: {sid}") continue - logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']})") - # Warm-up period: extra years before from_date for transform + z-score transform = meta.get("transform") extra_years = 3 if transform == "qoq_annualized" else (2 if transform == "yoy_pct" else 1) fetch_from = str(date(int(from_date[:4]) - extra_years, 1, 1)) - df = _fetch_fred_csv(sid, from_date=fetch_from) + df = _fetch_fred_api(sid, from_date=fetch_from, api_key=key) if df is None or df.empty: results[sid] = {"status": "fetch_failed", "count": 0} continue diff --git a/frontend/src/pages/CalendarPage.tsx b/frontend/src/pages/CalendarPage.tsx index 572f62b..e19f0ec 100644 --- a/frontend/src/pages/CalendarPage.tsx +++ b/frontend/src/pages/CalendarPage.tsx @@ -142,11 +142,37 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) { const [force, setForce] = useState(false) const [bsStatus, setBsStatus] = useState({ running: false, last_result: null }) const [polling, setPolling] = useState(false) + const [apiKey, setApiKey] = useState('') + const [keyConfigured, setKeyConfigured] = useState(null) + const [keyPreview, setKeyPreview] = useState('') + const [savingKey, setSavingKey] = useState(false) const onDoneRef = useRef(onDone) useEffect(() => { onDoneRef.current = onDone }, [onDone]) + // Check if FRED key is already saved + useEffect(() => { + fetch('/api/eco/fred-key') + .then(r => r.json()) + .then(d => { setKeyConfigured(d.configured); setKeyPreview(d.preview) }) + .catch(() => setKeyConfigured(false)) + }, []) + + const saveKey = async () => { + if (!apiKey.trim()) return + setSavingKey(true) + try { + const r = await fetch(`/api/eco/fred-key?key=${encodeURIComponent(apiKey.trim())}`, { method: 'POST' }) + const d = await r.json() + setKeyConfigured(true) + setKeyPreview(d.preview) + setApiKey('') + } finally { + setSavingKey(false) + } + } + const startBootstrap = async () => { - const url = `${API_BASE}/api/eco/bootstrap?from_date=${fromDate}&force=${force}` + const url = `/api/eco/bootstrap?from_date=${fromDate}&force=${force}` await fetch(url, { method: 'POST' }) setPolling(true) setBsStatus(s => ({ ...s, running: true })) @@ -155,7 +181,7 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) { useEffect(() => { if (!polling) return const iv = setInterval(async () => { - const r = await fetch(`${API_BASE}/api/eco/bootstrap/status`) + const r = await fetch(`/api/eco/bootstrap/status`) const data: BootstrapStatus = await r.json() setBsStatus(data) if (!data.running) { @@ -168,7 +194,9 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) { const rawResult = bsStatus.last_result as Record | null const bootstrapError: string | null = - rawResult && typeof (rawResult as { error?: unknown }).error === 'string' + rawResult && typeof (rawResult as { _error?: unknown })._error === 'string' + ? (rawResult as { _error: string })._error + : rawResult && typeof (rawResult as { error?: unknown }).error === 'string' ? (rawResult as { error: string }).error : null const bootstrapEntries: [string, BootstrapSeriesResult][] = @@ -181,9 +209,46 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
Bootstrap FRED
-
- Télécharge les données FRED (endpoint CSV public, sans clé API) depuis la date choisie. - Séries : NFP, Chômage, CPI, Core CPI, Core PCE, FOMC, Jobless Claims, GDP, HY Spread, T10Y2Y, T10Y3M. + + {/* API Key section */} +
+
+ Clé API FRED + {keyConfigured === true && ( + {keyPreview} ✓ + )} + {keyConfigured === false && ( + Non configurée + )} +
+ {keyConfigured === false && ( +
+ L'endpoint CSV public est bloqué par CloudFlare sur les IPs serveur. + L'API officielle FRED nécessite une clé gratuite :{' '} + fred.stlouisfed.org → My Account → API Keys +
+ )} +
+ setApiKey(e.target.value)} + placeholder={keyConfigured ? 'Nouvelle clé (optionnel)' : 'Colle ta clé FRED ici…'} + className="flex-1 text-xs bg-dark-700 border border-slate-700 rounded px-2 py-1 text-white font-mono placeholder:text-slate-600" + onKeyDown={e => e.key === 'Enter' && saveKey()} + /> + +
+
+ +
+ Séries : NFP · Chômage · CPI · Core CPI · Core PCE · FOMC · Jobless Claims · GDP · HY Spread · T10Y2Y · T10Y3M
@@ -200,11 +265,14 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {