fix: FRED fetch — add browser User-Agent + print diagnostics to stdout
FRED blocks requests without a valid User-Agent. Added headers matching a real browser. Also added print() calls so fetch errors appear in docker logs (logger.warning goes to system_logs table, not stdout). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -159,10 +159,24 @@ CATEGORIES = sorted({v["category"] for v in FRED_SERIES.values()})
|
||||
_FRED_CSV_URL = "https://fred.stlouisfed.org/graph/fredgraph.csv?id={series_id}"
|
||||
|
||||
|
||||
_HEADERS = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "text/csv,text/plain,*/*",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
|
||||
url = _FRED_CSV_URL.format(series_id=series_id)
|
||||
print(f"[FRED] Fetching {series_id} from {url}", flush=True)
|
||||
try:
|
||||
resp = httpx.get(url, timeout=30, follow_redirects=True)
|
||||
with httpx.Client(timeout=httpx.Timeout(10.0, connect=10.0), follow_redirects=True) as client:
|
||||
resp = client.get(url, headers=_HEADERS)
|
||||
print(f"[FRED] {series_id} status={resp.status_code} len={len(resp.content)}", flush=True)
|
||||
resp.raise_for_status()
|
||||
from io import StringIO
|
||||
df = pd.read_csv(StringIO(resp.text), parse_dates=["DATE"], index_col="DATE")
|
||||
@@ -170,8 +184,10 @@ def _fetch_fred_csv(series_id: str, from_date: str) -> Optional[pd.DataFrame]:
|
||||
df["value"] = pd.to_numeric(df["value"], errors="coerce")
|
||||
df = df.dropna()
|
||||
df = df[df.index >= pd.Timestamp(from_date)]
|
||||
print(f"[FRED] {series_id}: {len(df)} rows from {from_date}", flush=True)
|
||||
return df.sort_index()
|
||||
except Exception as e:
|
||||
print(f"[FRED] ERROR {series_id}: {type(e).__name__}: {e}", flush=True)
|
||||
logger.warning(f"[FRED] Failed to fetch {series_id}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user