From bdbb87962d8490526c963e4d8100ec20262299bd Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Fri, 26 Jun 2026 17:54:15 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20FXStreet=20calendar=20=E2=80=94=20free?= =?UTF-8?q?=20upcoming=20events=20with=20forecasts=20(no=20API=20key)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FXStreet calendar.fxstreet.com/eventdate/ returns ~300-400 events over 6 weeks including consensus forecasts, no authentication required. FF HTML scraper is blocked by Cloudflare even on residential IPs. FMP free plan returns 403 on /economic_calendar (requires Starter plan). - Add backend/services/fxstreet_calendar.py: single GET request returning all major currencies; maps Volatility 0/1/2 → low/medium/high - Add POST /api/eco/fxs-sync + GET /api/eco/fxs-sync/status endpoints - Add FXStreet to daily background sync in main.py (runs every 24h) - Add "Sync Upcoming (FXStreet)" button in ImportPanel (no key needed) - Fix FMP 403 error message to say endpoint requires Starter plan - Keep FMP panel for users who upgrade to FMP Starter ($14.99/month) Co-Authored-By: Claude Sonnet 4.6 --- backend/main.py | 10 +- backend/routers/eco.py | 42 +++++++- backend/services/fmp_calendar.py | 2 +- backend/services/fxstreet_calendar.py | 147 ++++++++++++++++++++++++++ frontend/src/pages/CalendarPage.tsx | 54 ++++++++-- 5 files changed, 241 insertions(+), 14 deletions(-) create mode 100644 backend/services/fxstreet_calendar.py diff --git a/backend/main.py b/backend/main.py index 2345a21..99365ef 100644 --- a/backend/main.py +++ b/backend/main.py @@ -109,7 +109,15 @@ def startup(): print(f"[Daily] FF live sync error: {e}", flush=True) try: - # FMP — upcoming forecasts (only if key is set) + # FXStreet — upcoming forecasts (no API key needed) + from services.fxstreet_calendar import fetch_upcoming as fxs_fetch + r = fxs_fetch(weeks_ahead=6) + print(f"[Daily] FXStreet sync: {r.get('total_upserted', 0)} upserted", flush=True) + except Exception as e: + print(f"[Daily] FXStreet sync error: {e}", flush=True) + + try: + # FMP — upcoming forecasts (only if key is set + Starter plan) from services.database import get_config fmp_key = get_config("fmp_api_key") or "" if fmp_key: diff --git a/backend/routers/eco.py b/backend/routers/eco.py index 636151d..0570c42 100644 --- a/backend/routers/eco.py +++ b/backend/routers/eco.py @@ -293,10 +293,11 @@ def upcoming_events() -> List[Dict[str, Any]]: # ── Forex Factory calendar ──────────────────────────────────────────────────── -_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None} -_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None} -_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None} -_te_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None} +_te_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_fxs_sync_status: Dict[str, Any] = {"running": False, "last_result": None} # CSV search order: Docker image (/app), uploaded to /tmp, local dev paths _FF_CSV_CANDIDATES = [ @@ -469,6 +470,39 @@ def fmp_sync_status_ep() -> Dict[str, Any]: return _te_sync_status +# ── FXStreet calendar (no API key required) ─────────────────────────────────── + +def _run_fxs_sync(weeks_ahead: int): + global _fxs_sync_status + _fxs_sync_status["running"] = True + try: + from services.fxstreet_calendar import fetch_upcoming + result = fetch_upcoming(weeks_ahead=weeks_ahead) + _fxs_sync_status["last_result"] = result + except Exception as e: + logger.error(f"[eco/fxs-sync] Failed: {e}") + _fxs_sync_status["last_result"] = {"error": str(e)} + finally: + _fxs_sync_status["running"] = False + + +@router.post("/fxs-sync") +def fxs_sync( + background_tasks: BackgroundTasks, + weeks: int = Query(6, ge=1, le=12), +) -> Dict[str, Any]: + """Fetch upcoming events + forecasts from FXStreet (no API key needed).""" + if _fxs_sync_status["running"]: + raise HTTPException(409, "FXStreet sync already running") + background_tasks.add_task(_run_fxs_sync, weeks) + return {"status": "started", "weeks_ahead": weeks} + + +@router.get("/fxs-sync/status") +def fxs_sync_status_ep() -> Dict[str, Any]: + return _fxs_sync_status + + @router.get("/calendar") def ff_calendar( period: str = Query("recent", description="recent|today|tomorrow|yesterday|this_week|next_week|previous_week|this_month|next_month|previous_month|custom"), diff --git a/backend/services/fmp_calendar.py b/backend/services/fmp_calendar.py index e5aa9a2..bc20d18 100644 --- a/backend/services/fmp_calendar.py +++ b/backend/services/fmp_calendar.py @@ -89,7 +89,7 @@ def fetch_upcoming(weeks_ahead: int = 6) -> Dict[str, Any]: if resp.status_code == 401: return {"error": "Invalid FMP API key"} if resp.status_code == 403: - return {"error": "FMP API key quota exceeded"} + return {"error": "Economic Calendar not available on FMP free plan — requires Starter ($14.99/month) at financialmodelingprep.com/developer/docs"} resp.raise_for_status() events = resp.json() diff --git a/backend/services/fxstreet_calendar.py b/backend/services/fxstreet_calendar.py new file mode 100644 index 0000000..40294f2 --- /dev/null +++ b/backend/services/fxstreet_calendar.py @@ -0,0 +1,147 @@ +""" +FXStreet economic calendar — upcoming events with consensus forecasts. +No API key required. Endpoint: calendar.fxstreet.com/eventdate/ +Covers ~300-400 events over 6 weeks for all major currencies. +""" +import logging +from datetime import datetime, timedelta, date +from typing import Any, Dict + +import httpx + +logger = logging.getLogger(__name__) + +_FXS_URL = "https://calendar.fxstreet.com/eventdate/" + +_SUPPORTED_CCY = {"USD", "EUR", "GBP", "JPY", "AUD", "CAD", "NZD", "CHF", "CNY"} + +_VOLATILITY = {0: "low", 1: "medium", 2: "high"} + +_HEADERS = { + "Accept": "application/json", + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/125.0.0.0 Safari/537.36" + ), + "Origin": "https://www.fxstreet.com", + "Referer": "https://www.fxstreet.com/economic-calendar", +} + + +def _parse_dt(dt_str: str) -> tuple[str, str]: + """'2026-07-04T12:30:00Z' → ('2026-07-04', '12:30')""" + if not dt_str: + return "", "00:00" + try: + dt_str = dt_str.replace("Z", "+00:00") + dt = datetime.fromisoformat(dt_str) + return dt.strftime("%Y-%m-%d"), dt.strftime("%H:%M") + except Exception: + return dt_str[:10] if len(dt_str) >= 10 else "", "00:00" + + +def _fmt(v: Any) -> str | None: + if v is None: + return None + try: + f = float(v) + if f != f: + return None + return str(int(f)) if f == int(f) else str(round(f, 4)) + except (TypeError, ValueError): + s = str(v).strip() + return s or None + + +def fetch_upcoming(weeks_ahead: int = 6) -> Dict[str, Any]: + """ + Fetch upcoming economic events from FXStreet (no API key needed). + One request returns all currencies for the full date range. + """ + from services.database import get_conn + from services.ff_calendar import _upsert_batch, FF_TO_FRED + + today = date.today() + date_to = today + timedelta(weeks=weeks_ahead) + + params = { + "f": "json", + "v": "2", + "from": str(today), + "to": str(date_to), + "timezone": "UTC", + "culture": "en-GB", + } + + print(f"[FXS calendar] fetching {today} → {date_to}", flush=True) + + try: + resp = httpx.get( + _FXS_URL, params=params, headers=_HEADERS, + timeout=30, follow_redirects=True, + ) + print(f"[FXS calendar] HTTP {resp.status_code}", flush=True) + + if resp.status_code == 403: + return {"error": "FXStreet blocked this server IP (403). Try again later."} + if resp.status_code == 429: + return {"error": "FXStreet rate limit hit (429). Wait a few minutes."} + + resp.raise_for_status() + events = resp.json() + + if not isinstance(events, list): + return {"error": f"Unexpected FXStreet response: {str(events)[:200]}"} + + except Exception as e: + return {"error": f"FXStreet request failed: {e}"} + + batch = [] + skipped = 0 + for ev in events: + event_obj = ev.get("Event") or {} + + ccy = (event_obj.get("CurrencyId") or "").strip() + if ccy not in _SUPPORTED_CCY: + skipped += 1 + continue + + event_name = (event_obj.get("Name") or "").strip() + if not event_name: + continue + + dt_str = (ev.get("DateUtc") or "").strip() + ev_date, ev_time = _parse_dt(dt_str) + if not ev_date: + continue + + volatility = ev.get("Volatility") + impact = _VOLATILITY.get(volatility, "low") if volatility is not None else "low" + + actual = _fmt(ev.get("Actual")) + forecast = _fmt(ev.get("Consensus")) + # Use Revised previous if available, otherwise Previous + previous = _fmt(ev.get("Revised")) or _fmt(ev.get("Previous")) + + series_id = FF_TO_FRED.get(event_name) + + batch.append(( + ev_date, ev_time, ccy, impact, event_name, + actual, forecast, previous, + series_id, None, "fxstreet", + )) + + conn = get_conn() + if batch: + _upsert_batch(conn, batch) + conn.commit() + conn.close() + + print(f"[FXS calendar] {len(batch)} upserted, {skipped} skipped (non-major CCY)", flush=True) + return { + "total_upserted": len(batch), + "skipped": skipped, + "date_from": str(today), + "date_to": str(date_to), + } diff --git a/frontend/src/pages/CalendarPage.tsx b/frontend/src/pages/CalendarPage.tsx index 95ae262..1725099 100644 --- a/frontend/src/pages/CalendarPage.tsx +++ b/frontend/src/pages/CalendarPage.tsx @@ -133,11 +133,13 @@ function SurpriseChip({ ev }: { ev: FFEvent }) { // ── Import Panel ────────────────────────────────────────────────────────────── function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => void }) { - const [syncResult, setSyncResult] = useState(null) + const [syncResult, setSyncResult] = useState(null) + const [fxsMsg, setFxsMsg] = useState(null) + const [fxsSyncing, setFxsSyncing] = useState(false) const [importStatus, setImportStatus] = useState({ running: false, last_result: null }) - const pollRef = useRef | null>(null) + const pollRef = useRef | null>(null) + const fxsPollRef = useRef | null>(null) - // Poll import status on mount (auto-import may be running in background) useEffect(() => { const poll = async () => { const r: ImportStatus = await fetch(`${API}/api/eco/ff-import/status`).then(x => x.json()).catch(() => ({ running: false, last_result: null })) @@ -151,7 +153,10 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => } } poll() - return () => { if (pollRef.current) clearInterval(pollRef.current) } + return () => { + if (pollRef.current) clearInterval(pollRef.current) + if (fxsPollRef.current) clearInterval(fxsPollRef.current) + } }, [onImported]) const startSync = async () => { @@ -161,12 +166,29 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => await new Promise(res => setTimeout(res, 2500)) const s = await fetch(`${API}/api/eco/ff-sync/status`).then(x => x.json()) const res = s.last_result || {} - const tw = res.thisweek?.events ?? '?' - setSyncResult(`✓ live: ${tw} events`) + setSyncResult(`✓ live: ${res.thisweek?.events ?? '?'} events`) onImported() } catch { setSyncResult('sync error') } } + const startFxsSync = async () => { + setFxsSyncing(true) + setFxsMsg('syncing FXStreet…') + try { + await fetch(`${API}/api/eco/fxs-sync?weeks=6`, { method: 'POST' }) + fxsPollRef.current = setInterval(async () => { + const s = await fetch(`${API}/api/eco/fxs-sync/status`).then(x => x.json()) + if (!s.running) { + clearInterval(fxsPollRef.current!) + setFxsSyncing(false) + const r = s.last_result || {} + if (r.error) { setFxsMsg(`Error: ${r.error}`); return } + setFxsMsg(`✓ FXStreet: ${r.total_upserted} events (${r.date_from} → ${r.date_to})`) + onImported() + } + }, 2000) + } catch { setFxsSyncing(false); setFxsMsg('FXStreet error') } + } const hasData = stats.total > 0 const res = importStatus.last_result @@ -185,14 +207,25 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => - {/* Live sync — this week from faireconomy.media JSON */} + {/* FF live sync — this week actuals */} + + {/* FXStreet — upcoming 6 weeks + forecasts, no API key */} + {res && !importStatus.running && ( @@ -201,6 +234,11 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => : {res.inserted?.toLocaleString()} inserted )} {syncResult && {syncResult}} + {fxsMsg && ( + + {fxsMsg} + + )} ) }