diff --git a/backend/services/fred_bootstrap.py b/backend/services/fred_bootstrap.py index abf0143..2cbb974 100644 --- a/backend/services/fred_bootstrap.py +++ b/backend/services/fred_bootstrap.py @@ -159,10 +159,24 @@ CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()}) _FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}" +_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/124.0.0.0 Safari/537.36" + ), + "Accept": "text/csv,text/plain,*/*", + "Accept-Language": "en-US,en;q=0.9", +} + + def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]: url = _FRED_CSV_URL.format(series_id=series_id) + print(f"[FRED] Fetching {series_id} from {url}", flush=True) try: - resp = httpx.get(url, timeout=30, follow_redirects=True) + with httpx.Client(timeout=httpx.Timeout(10.0, connect=10.0), follow_redirects=True) as client: + resp = client.get(url, headers=_HEADERS) + print(f"[FRED] {series_id} status={resp.status_code} len={len(resp.content)}", flush=True) resp.raise_for_status() from io import StringIO df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE") @@ -170,8 +184,10 @@ def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]: df["value"] = pd.to_numeric(df["value"], errors="coerce") df = df.dropna() df = df[df.index >= pd.Timestamp(from_date)] + print(f"[FRED] {series_id}: {len(df)} rows from {from_date}", flush=True) return df.sort_index() except Exception as e: + print(f"[FRED] ERROR {series_id}: {type(e).__name__}: {e}", flush=True) logger.warning(f"[FRED] Failed to fetch {series_id}: {e}") return None