admin/OpenFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: FRED fetch — add browser User-Agent + print diagnostics to stdout

Browse Source 
FRED blocks requests without a valid User-Agent. Added headers matching
a real browser. Also added print() calls so fetch errors appear in
docker logs (logger.warning goes to system_logs table, not stdout).

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-26 15:33:06 +02:00
parent d25ca92694
commit 202a8ce97c
1 changed files with 17 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
backend/services/fred_bootstrap.py
View File

@@ -159,10 +159,24 @@ CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}" _FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
_HEADERS = {
"User-Agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
),
"Accept": "text/csv,text/plain,*/*",
"Accept-Language": "en-US,en;q=0.9",
}
def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]: def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
url = _FRED_CSV_URL.format(series_id=series_id) url = _FRED_CSV_URL.format(series_id=series_id)
print(f"[FRED] Fetching {series_id} from {url}", flush=True)
try: try:
resp = httpx.get(url, timeout=30, follow_redirects=True) with httpx.Client(timeout=httpx.Timeout(10.0, connect=10.0), follow_redirects=True) as client:
resp = client.get(url, headers=_HEADERS)
print(f"[FRED] {series_id} status={resp.status_code} len={len(resp.content)}", flush=True)
resp.raise_for_status() resp.raise_for_status()
from io import StringIO from io import StringIO
df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE") df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE")
@@ -170,8 +184,10 @@ def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
df["value"] = pd.to_numeric(df["value"], errors="coerce") df["value"] = pd.to_numeric(df["value"], errors="coerce")
df = df.dropna() df = df.dropna()
df = df[df.index >= pd.Timestamp(from_date)] df = df[df.index >= pd.Timestamp(from_date)]
print(f"[FRED] {series_id}: {len(df)} rows from {from_date}", flush=True)
return df.sort_index() return df.sort_index()
except Exception as e: except Exception as e:
print(f"[FRED] ERROR {series_id}: {type(e).__name__}: {e}", flush=True)
logger.warning(f"[FRED] Failed to fetch {series_id}: {e}") logger.warning(f"[FRED] Failed to fetch {series_id}: {e}")
return None return None