fix: use FRED official JSON API (bypasses CloudFlare) + API key UI
- fred_bootstrap.py: switch from CSV graph endpoint (CloudFlare-blocked) to api.stlouisfed.org/fred/series/observations JSON API; reads key from DB config 'fred_api_key'; returns clear error if key missing - eco.py: add GET/POST /api/eco/fred-key to check/save the FRED API key - CalendarPage.tsx: BootstrapPanel shows API key section with status, input to paste key + save button, disables Launch if key missing Free key: fred.stlouisfed.org -> My Account -> API Keys Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
@@ -48,6 +48,28 @@ def _run_bootstrap(from_date: str, series_ids: Optional[List[str]], force: bool)
|
||||
_bootstrap_status["running"] = False
|
||||
|
||||
|
||||
@router.get("/fred-key")
|
||||
def get_fred_key() -> Dict[str, Any]:
|
||||
"""Check if a FRED API key is configured."""
|
||||
from services.database import get_config
|
||||
key = get_config("fred_api_key") or ""
|
||||
return {
|
||||
"configured": bool(key),
|
||||
"preview": (key[:6] + "…" + key[-4:]) if len(key) > 10 else ("***" if key else ""),
|
||||
}
|
||||
|
||||
|
||||
@router.post("/fred-key")
|
||||
def save_fred_key(key: str = Query(..., description="FRED API key")) -> Dict[str, Any]:
|
||||
"""Save FRED API key to DB config."""
|
||||
from services.database import set_config
|
||||
key = key.strip()
|
||||
if not key:
|
||||
raise HTTPException(400, "Empty key")
|
||||
set_config("fred_api_key", key)
|
||||
return {"status": "saved", "preview": key[:6] + "…" + key[-4:]}
|
||||
|
||||
|
||||
@router.post("/bootstrap")
|
||||
def bootstrap(
|
||||
background_tasks: BackgroundTasks,
|
||||
|
||||
@@ -154,41 +154,55 @@ DEPRECATED_SERIES = ["A191RL1Q225SBEA"]
|
||||
CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
|
||||
|
||||
|
||||
# ── FRED fetch ────────────────────────────────────────────────────────────────
|
||||
# ── FRED fetch (official JSON API — bypasses CloudFlare) ──────────────────────
|
||||
#
|
||||
# The public CSV endpoint (fred.stlouisfed.org/graph/fredgraph.csv) is blocked
|
||||
# by CloudFlare on server IPs. Use the official REST API instead:
|
||||
# https://api.stlouisfed.org/fred/series/observations
|
||||
# Free API key: https://fred.stlouisfed.org/docs/api/api_key.html
|
||||
|
||||
_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
|
||||
_FRED_API_URL = "https://api.stlouisfed.org/fred/series/observations"
|
||||
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/csv,text/plain,*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
|
||||
url = _FRED_CSV_URL.format(series_id=series_id)
|
||||
print(f"[FRED] Fetching {series_id} from {url}", flush=True)
|
||||
def _fetch_fred_api(series_id: str, from_date: str, api_key: str) -> Optional[pd.DataFrame]:
|
||||
params = {
|
||||
"series_id": series_id,
|
||||
"api_key": api_key,
|
||||
"observation_start": from_date,
|
||||
"file_type": "json",
|
||||
"sort_order": "asc",
|
||||
}
|
||||
print(f"[FRED API] Fetching {series_id} from {from_date}", flush=True)
|
||||
try:
|
||||
with httpx.Client(timeout=httpx.Timeout(10.0, connect=10.0), follow_redirects=True) as client:
|
||||
resp = client.get(url, headers=_HEADERS)
|
||||
print(f"[FRED] {series_id} status={resp.status_code} len={len(resp.content)}", flush=True)
|
||||
resp = httpx.get(_FRED_API_URL, params=params, timeout=30, follow_redirects=True)
|
||||
print(f"[FRED API] {series_id} HTTP {resp.status_code}", flush=True)
|
||||
resp.raise_for_status()
|
||||
from io import StringIO
|
||||
df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE")
|
||||
df.columns = ["value"]
|
||||
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
||||
df = df.dropna()
|
||||
df = df[df.index >= pd.Timestamp(from_date)]
|
||||
print(f"[FRED] {series_id}: {len(df)} rows from {from_date}", flush=True)
|
||||
return df.sort_index()
|
||||
data = resp.json()
|
||||
|
||||
if "error_message" in data:
|
||||
print(f"[FRED API] {series_id} API error: {data['error_message']}", flush=True)
|
||||
return None
|
||||
|
||||
records = []
|
||||
for obs in data.get("observations", []):
|
||||
val_str = obs.get("value", ".")
|
||||
if val_str == ".": # FRED uses "." for missing/unreleased values
|
||||
continue
|
||||
try:
|
||||
records.append({"date": pd.Timestamp(obs["date"]), "value": float(val_str)})
|
||||
except (ValueError, KeyError):
|
||||
continue
|
||||
|
||||
if not records:
|
||||
print(f"[FRED API] {series_id}: no valid observations", flush=True)
|
||||
return None
|
||||
|
||||
df = pd.DataFrame(records).set_index("date").sort_index()
|
||||
print(f"[FRED API] {series_id}: {len(df)} rows OK", flush=True)
|
||||
return df
|
||||
except Exception as e:
|
||||
print(f"[FRED] ERROR {series_id}: {type(e).__name__}: {e}", flush=True)
|
||||
logger.warning(f"[FRED] Failed to fetch {series_id}: {e}")
|
||||
print(f"[FRED API] ERROR {series_id}: {type(e).__name__}: {e}", flush=True)
|
||||
logger.warning(f"[FRED API] Failed {series_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
@@ -242,15 +256,27 @@ def bootstrap_fred(
|
||||
from_date: str = "2020-01-01",
|
||||
series_ids: Optional[List[str]] = None,
|
||||
force: bool = False,
|
||||
api_key: Optional[str] = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
Fetch and store FRED data from from_date to today.
|
||||
Transforms are applied before storing (YoY%, QoQ Ann., /1000).
|
||||
Returns summary dict with counts per series.
|
||||
Requires a free FRED API key (fred.stlouisfed.org/docs/api/api_key.html).
|
||||
If api_key is not passed, reads from DB config key 'fred_api_key'.
|
||||
"""
|
||||
from services.database import get_conn
|
||||
from services.database import get_conn, get_config
|
||||
import json
|
||||
|
||||
# Resolve API key
|
||||
key = api_key or get_config("fred_api_key") or ""
|
||||
if not key:
|
||||
return {
|
||||
"_error": (
|
||||
"Clé API FRED manquante. "
|
||||
"Inscris-toi gratuitement sur fred.stlouisfed.org/docs/api/api_key.html "
|
||||
"puis saisis la clé dans le panel Bootstrap."
|
||||
)
|
||||
}
|
||||
|
||||
target_series = series_ids or list(FRED_SERIES.keys())
|
||||
results: Dict[str, Any] = {}
|
||||
conn = get_conn()
|
||||
@@ -269,14 +295,12 @@ def bootstrap_fred(
|
||||
logger.warning(f"[FRED bootstrap] Unknown series: {sid}")
|
||||
continue
|
||||
|
||||
logger.info(f"[FRED bootstrap] Fetching {sid} ({meta['name']})")
|
||||
|
||||
# Warm-up period: extra years before from_date for transform + z-score
|
||||
transform = meta.get("transform")
|
||||
extra_years = 3 if transform == "qoq_annualized" else (2 if transform == "yoy_pct" else 1)
|
||||
fetch_from = str(date(int(from_date[:4]) - extra_years, 1, 1))
|
||||
|
||||
df = _fetch_fred_csv(sid, from_date=fetch_from)
|
||||
df = _fetch_fred_api(sid, from_date=fetch_from, api_key=key)
|
||||
if df is None or df.empty:
|
||||
results[sid] = {"status": "fetch_failed", "count": 0}
|
||||
continue
|
||||
|
||||
@@ -142,11 +142,37 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
|
||||
const [force, setForce] = useState(false)
|
||||
const [bsStatus, setBsStatus] = useState<BootstrapStatus>({ running: false, last_result: null })
|
||||
const [polling, setPolling] = useState(false)
|
||||
const [apiKey, setApiKey] = useState('')
|
||||
const [keyConfigured, setKeyConfigured] = useState<boolean | null>(null)
|
||||
const [keyPreview, setKeyPreview] = useState('')
|
||||
const [savingKey, setSavingKey] = useState(false)
|
||||
const onDoneRef = useRef(onDone)
|
||||
useEffect(() => { onDoneRef.current = onDone }, [onDone])
|
||||
|
||||
// Check if FRED key is already saved
|
||||
useEffect(() => {
|
||||
fetch('/api/eco/fred-key')
|
||||
.then(r => r.json())
|
||||
.then(d => { setKeyConfigured(d.configured); setKeyPreview(d.preview) })
|
||||
.catch(() => setKeyConfigured(false))
|
||||
}, [])
|
||||
|
||||
const saveKey = async () => {
|
||||
if (!apiKey.trim()) return
|
||||
setSavingKey(true)
|
||||
try {
|
||||
const r = await fetch(`/api/eco/fred-key?key=${encodeURIComponent(apiKey.trim())}`, { method: 'POST' })
|
||||
const d = await r.json()
|
||||
setKeyConfigured(true)
|
||||
setKeyPreview(d.preview)
|
||||
setApiKey('')
|
||||
} finally {
|
||||
setSavingKey(false)
|
||||
}
|
||||
}
|
||||
|
||||
const startBootstrap = async () => {
|
||||
const url = `${API_BASE}/api/eco/bootstrap?from_date=${fromDate}&force=${force}`
|
||||
const url = `/api/eco/bootstrap?from_date=${fromDate}&force=${force}`
|
||||
await fetch(url, { method: 'POST' })
|
||||
setPolling(true)
|
||||
setBsStatus(s => ({ ...s, running: true }))
|
||||
@@ -155,7 +181,7 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
|
||||
useEffect(() => {
|
||||
if (!polling) return
|
||||
const iv = setInterval(async () => {
|
||||
const r = await fetch(`${API_BASE}/api/eco/bootstrap/status`)
|
||||
const r = await fetch(`/api/eco/bootstrap/status`)
|
||||
const data: BootstrapStatus = await r.json()
|
||||
setBsStatus(data)
|
||||
if (!data.running) {
|
||||
@@ -168,7 +194,9 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
|
||||
|
||||
const rawResult = bsStatus.last_result as Record<string, unknown> | null
|
||||
const bootstrapError: string | null =
|
||||
rawResult && typeof (rawResult as { error?: unknown }).error === 'string'
|
||||
rawResult && typeof (rawResult as { _error?: unknown })._error === 'string'
|
||||
? (rawResult as { _error: string })._error
|
||||
: rawResult && typeof (rawResult as { error?: unknown }).error === 'string'
|
||||
? (rawResult as { error: string }).error
|
||||
: null
|
||||
const bootstrapEntries: [string, BootstrapSeriesResult][] =
|
||||
@@ -181,9 +209,46 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
|
||||
<div className="text-xs font-semibold text-blue-400 flex items-center gap-1">
|
||||
<Download className="w-3 h-3" /> Bootstrap FRED
|
||||
</div>
|
||||
<div className="text-xs text-slate-400">
|
||||
Télécharge les données FRED (endpoint CSV public, sans clé API) depuis la date choisie.
|
||||
Séries : NFP, Chômage, CPI, Core CPI, Core PCE, FOMC, Jobless Claims, GDP, HY Spread, T10Y2Y, T10Y3M.
|
||||
|
||||
{/* API Key section */}
|
||||
<div className="bg-dark-600 rounded border border-slate-700/50 p-3 space-y-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-semibold text-slate-300">Clé API FRED</span>
|
||||
{keyConfigured === true && (
|
||||
<span className="text-xs text-emerald-400 font-mono">{keyPreview} ✓</span>
|
||||
)}
|
||||
{keyConfigured === false && (
|
||||
<span className="text-xs text-orange-400">Non configurée</span>
|
||||
)}
|
||||
</div>
|
||||
{keyConfigured === false && (
|
||||
<div className="text-xs text-slate-500">
|
||||
L'endpoint CSV public est bloqué par CloudFlare sur les IPs serveur.
|
||||
L'API officielle FRED nécessite une clé gratuite :{' '}
|
||||
<span className="text-blue-400 font-mono">fred.stlouisfed.org → My Account → API Keys</span>
|
||||
</div>
|
||||
)}
|
||||
<div className="flex gap-2">
|
||||
<input
|
||||
type="text"
|
||||
value={apiKey}
|
||||
onChange={e => setApiKey(e.target.value)}
|
||||
placeholder={keyConfigured ? 'Nouvelle clé (optionnel)' : 'Colle ta clé FRED ici…'}
|
||||
className="flex-1 text-xs bg-dark-700 border border-slate-700 rounded px-2 py-1 text-white font-mono placeholder:text-slate-600"
|
||||
onKeyDown={e => e.key === 'Enter' && saveKey()}
|
||||
/>
|
||||
<button
|
||||
onClick={saveKey}
|
||||
disabled={!apiKey.trim() || savingKey}
|
||||
className="text-xs px-3 py-1 rounded bg-emerald-700 hover:bg-emerald-600 text-white font-semibold disabled:opacity-40"
|
||||
>
|
||||
{savingKey ? '…' : 'Sauver'}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="text-xs text-slate-500">
|
||||
Séries : NFP · Chômage · CPI · Core CPI · Core PCE · FOMC · Jobless Claims · GDP · HY Spread · T10Y2Y · T10Y3M
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<div>
|
||||
@@ -200,11 +265,14 @@ function BootstrapPanel({ onDone }: { onDone: () => void }) {
|
||||
</label>
|
||||
<button
|
||||
onClick={startBootstrap}
|
||||
disabled={bsStatus.running}
|
||||
disabled={bsStatus.running || keyConfigured === false}
|
||||
title={keyConfigured === false ? 'Configure ta clé FRED d\'abord' : ''}
|
||||
className={clsx(
|
||||
'mt-4 px-3 py-1.5 rounded text-xs font-semibold transition-colors',
|
||||
bsStatus.running
|
||||
? 'bg-blue-900/40 text-blue-400 cursor-wait'
|
||||
: keyConfigured === false
|
||||
? 'bg-slate-700 text-slate-500 cursor-not-allowed'
|
||||
: 'bg-blue-600 hover:bg-blue-500 text-white',
|
||||
)}
|
||||
>
|
||||
|
||||
Reference in New Issue
Block a user