feat: FXStreet calendar — free upcoming events with forecasts (no API key)

FXStreet calendar.fxstreet.com/eventdate/ returns ~300-400 events over 6 weeks including consensus forecasts, no authentication required. FF HTML scraper is blocked by Cloudflare even on residential IPs. FMP free plan returns 403 on /economic_calendar (requires Starter plan). - Add backend/services/fxstreet_calendar.py: single GET request returning all major currencies; maps Volatility 0/1/2 → low/medium/high - Add POST /api/eco/fxs-sync + GET /api/eco/fxs-sync/status endpoints - Add FXStreet to daily background sync in main.py (runs every 24h) - Add "Sync Upcoming (FXStreet)" button in ImportPanel (no key needed) - Fix FMP 403 error message to say endpoint requires Starter plan - Keep FMP panel for users who upgrade to FMP Starter ($14.99/month) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-26 17:54:15 +02:00
parent 5c86d6e715
commit bdbb87962d
5 changed files with 241 additions and 14 deletions
--- a/backend/main.py
+++ b/backend/main.py
@@ -109,7 +109,15 @@ def startup():
                print(f"[Daily] FF live sync error: {e}", flush=True)

            try:
-                # FMP — upcoming forecasts (only if key is set)
+                # FXStreet — upcoming forecasts (no API key needed)
+                from services.fxstreet_calendar import fetch_upcoming as fxs_fetch
+                r = fxs_fetch(weeks_ahead=6)
+                print(f"[Daily] FXStreet sync: {r.get('total_upserted', 0)} upserted", flush=True)
+            except Exception as e:
+                print(f"[Daily] FXStreet sync error: {e}", flush=True)
+
+            try:
+                # FMP — upcoming forecasts (only if key is set + Starter plan)
                from services.database import get_config
                fmp_key = get_config("fmp_api_key") or ""
                if fmp_key:
--- a/backend/routers/eco.py
+++ b/backend/routers/eco.py
@@ -293,10 +293,11 @@ def upcoming_events() -> List[Dict[str, Any]]:

 # ── Forex Factory calendar ────────────────────────────────────────────────────

-_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None}
-_ff_sync_status:   Dict[str, Any] = {"running": False, "last_result": None}
-_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None}
-_te_sync_status:   Dict[str, Any] = {"running": False, "last_result": None}
+_ff_import_status:  Dict[str, Any] = {"running": False, "last_result": None}
+_ff_sync_status:    Dict[str, Any] = {"running": False, "last_result": None}
+_ff_scrape_status:  Dict[str, Any] = {"running": False, "last_result": None}
+_te_sync_status:    Dict[str, Any] = {"running": False, "last_result": None}
+_fxs_sync_status:   Dict[str, Any] = {"running": False, "last_result": None}

 # CSV search order: Docker image (/app), uploaded to /tmp, local dev paths
 _FF_CSV_CANDIDATES = [
@@ -469,6 +470,39 @@ def fmp_sync_status_ep() -> Dict[str, Any]:
    return _te_sync_status


+# ── FXStreet calendar (no API key required) ───────────────────────────────────
+
+def _run_fxs_sync(weeks_ahead: int):
+    global _fxs_sync_status
+    _fxs_sync_status["running"] = True
+    try:
+        from services.fxstreet_calendar import fetch_upcoming
+        result = fetch_upcoming(weeks_ahead=weeks_ahead)
+        _fxs_sync_status["last_result"] = result
+    except Exception as e:
+        logger.error(f"[eco/fxs-sync] Failed: {e}")
+        _fxs_sync_status["last_result"] = {"error": str(e)}
+    finally:
+        _fxs_sync_status["running"] = False
+
+
+@router.post("/fxs-sync")
+def fxs_sync(
+    background_tasks: BackgroundTasks,
+    weeks: int = Query(6, ge=1, le=12),
+) -> Dict[str, Any]:
+    """Fetch upcoming events + forecasts from FXStreet (no API key needed)."""
+    if _fxs_sync_status["running"]:
+        raise HTTPException(409, "FXStreet sync already running")
+    background_tasks.add_task(_run_fxs_sync, weeks)
+    return {"status": "started", "weeks_ahead": weeks}
+
+
+@router.get("/fxs-sync/status")
+def fxs_sync_status_ep() -> Dict[str, Any]:
+    return _fxs_sync_status
+
+
@router.get("/calendar")
 def ff_calendar(
    period:     str            = Query("recent", description="recent|today|tomorrow|yesterday|this_week|next_week|previous_week|this_month|next_month|previous_month|custom"),
--- a/backend/services/fmp_calendar.py
+++ b/backend/services/fmp_calendar.py
@@ -89,7 +89,7 @@ def fetch_upcoming(weeks_ahead: int = 6) -> Dict[str, Any]:
        if resp.status_code == 401:
            return {"error": "Invalid FMP API key"}
        if resp.status_code == 403:
-            return {"error": "FMP API key quota exceeded"}
+            return {"error": "Economic Calendar not available on FMP free plan — requires Starter ($14.99/month) at financialmodelingprep.com/developer/docs"}

        resp.raise_for_status()
        events = resp.json()
--- a/backend/services/fxstreet_calendar.py
+++ b/backend/services/fxstreet_calendar.py
@@ -0,0 +1,147 @@
+"""
+FXStreet economic calendar — upcoming events with consensus forecasts.
+No API key required. Endpoint: calendar.fxstreet.com/eventdate/
+Covers ~300-400 events over 6 weeks for all major currencies.
+"""
+import logging
+from datetime import datetime, timedelta, date
+from typing import Any, Dict
+
+import httpx
+
+logger = logging.getLogger(__name__)
+
+_FXS_URL = "https://calendar.fxstreet.com/eventdate/"
+
+_SUPPORTED_CCY = {"USD", "EUR", "GBP", "JPY", "AUD", "CAD", "NZD", "CHF", "CNY"}
+
+_VOLATILITY = {0: "low", 1: "medium", 2: "high"}
+
+_HEADERS = {
+    "Accept": "application/json",
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/125.0.0.0 Safari/537.36"
+    ),
+    "Origin": "https://www.fxstreet.com",
+    "Referer": "https://www.fxstreet.com/economic-calendar",
+}
+
+
+def _parse_dt(dt_str: str) -> tuple[str, str]:
+    """'2026-07-04T12:30:00Z' → ('2026-07-04', '12:30')"""
+    if not dt_str:
+        return "", "00:00"
+    try:
+        dt_str = dt_str.replace("Z", "+00:00")
+        dt = datetime.fromisoformat(dt_str)
+        return dt.strftime("%Y-%m-%d"), dt.strftime("%H:%M")
+    except Exception:
+        return dt_str[:10] if len(dt_str) >= 10 else "", "00:00"
+
+
+def _fmt(v: Any) -> str | None:
+    if v is None:
+        return None
+    try:
+        f = float(v)
+        if f != f:
+            return None
+        return str(int(f)) if f == int(f) else str(round(f, 4))
+    except (TypeError, ValueError):
+        s = str(v).strip()
+        return s or None
+
+
+def fetch_upcoming(weeks_ahead: int = 6) -> Dict[str, Any]:
+    """
+    Fetch upcoming economic events from FXStreet (no API key needed).
+    One request returns all currencies for the full date range.
+    """
+    from services.database import get_conn
+    from services.ff_calendar import _upsert_batch, FF_TO_FRED
+
+    today = date.today()
+    date_to = today + timedelta(weeks=weeks_ahead)
+
+    params = {
+        "f":        "json",
+        "v":        "2",
+        "from":     str(today),
+        "to":       str(date_to),
+        "timezone": "UTC",
+        "culture":  "en-GB",
+    }
+
+    print(f"[FXS calendar] fetching {today} → {date_to}", flush=True)
+
+    try:
+        resp = httpx.get(
+            _FXS_URL, params=params, headers=_HEADERS,
+            timeout=30, follow_redirects=True,
+        )
+        print(f"[FXS calendar] HTTP {resp.status_code}", flush=True)
+
+        if resp.status_code == 403:
+            return {"error": "FXStreet blocked this server IP (403). Try again later."}
+        if resp.status_code == 429:
+            return {"error": "FXStreet rate limit hit (429). Wait a few minutes."}
+
+        resp.raise_for_status()
+        events = resp.json()
+
+        if not isinstance(events, list):
+            return {"error": f"Unexpected FXStreet response: {str(events)[:200]}"}
+
+    except Exception as e:
+        return {"error": f"FXStreet request failed: {e}"}
+
+    batch = []
+    skipped = 0
+    for ev in events:
+        event_obj = ev.get("Event") or {}
+
+        ccy = (event_obj.get("CurrencyId") or "").strip()
+        if ccy not in _SUPPORTED_CCY:
+            skipped += 1
+            continue
+
+        event_name = (event_obj.get("Name") or "").strip()
+        if not event_name:
+            continue
+
+        dt_str = (ev.get("DateUtc") or "").strip()
+        ev_date, ev_time = _parse_dt(dt_str)
+        if not ev_date:
+            continue
+
+        volatility = ev.get("Volatility")
+        impact = _VOLATILITY.get(volatility, "low") if volatility is not None else "low"
+
+        actual   = _fmt(ev.get("Actual"))
+        forecast = _fmt(ev.get("Consensus"))
+        # Use Revised previous if available, otherwise Previous
+        previous = _fmt(ev.get("Revised")) or _fmt(ev.get("Previous"))
+
+        series_id = FF_TO_FRED.get(event_name)
+
+        batch.append((
+            ev_date, ev_time, ccy, impact, event_name,
+            actual, forecast, previous,
+            series_id, None, "fxstreet",
+        ))
+
+    conn = get_conn()
+    if batch:
+        _upsert_batch(conn, batch)
+        conn.commit()
+    conn.close()
+
+    print(f"[FXS calendar] {len(batch)} upserted, {skipped} skipped (non-major CCY)", flush=True)
+    return {
+        "total_upserted": len(batch),
+        "skipped":        skipped,
+        "date_from":      str(today),
+        "date_to":        str(date_to),
+    }
--- a/frontend/src/pages/CalendarPage.tsx
+++ b/frontend/src/pages/CalendarPage.tsx
@@ -133,11 +133,13 @@ function SurpriseChip({ ev }: { ev: FFEvent }) {
 // ── Import Panel ──────────────────────────────────────────────────────────────

 function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => void }) {
-  const [syncResult, setSyncResult] = useState<string | null>(null)
+  const [syncResult, setSyncResult]   = useState<string | null>(null)
+  const [fxsMsg, setFxsMsg]           = useState<string | null>(null)
+  const [fxsSyncing, setFxsSyncing]   = useState(false)
  const [importStatus, setImportStatus] = useState<ImportStatus>({ running: false, last_result: null })
-  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
+  const pollRef    = useRef<ReturnType<typeof setInterval> | null>(null)
+  const fxsPollRef = useRef<ReturnType<typeof setInterval> | null>(null)

-  // Poll import status on mount (auto-import may be running in background)
  useEffect(() => {
    const poll = async () => {
      const r: ImportStatus = await fetch(`${API}/api/eco/ff-import/status`).then(x => x.json()).catch(() => ({ running: false, last_result: null }))
@@ -151,7 +153,10 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
      }
    }
    poll()
-    return () => { if (pollRef.current) clearInterval(pollRef.current) }
+    return () => {
+      if (pollRef.current)    clearInterval(pollRef.current)
+      if (fxsPollRef.current) clearInterval(fxsPollRef.current)
+    }
  }, [onImported])

  const startSync = async () => {
@@ -161,12 +166,29 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
      await new Promise(res => setTimeout(res, 2500))
      const s = await fetch(`${API}/api/eco/ff-sync/status`).then(x => x.json())
      const res = s.last_result || {}
-      const tw = res.thisweek?.events ?? '?'
-      setSyncResult(`✓ live: ${tw} events`)
+      setSyncResult(`✓ live: ${res.thisweek?.events ?? '?'} events`)
      onImported()
    } catch { setSyncResult('sync error') }
  }

+  const startFxsSync = async () => {
+    setFxsSyncing(true)
+    setFxsMsg('syncing FXStreet…')
+    try {
+      await fetch(`${API}/api/eco/fxs-sync?weeks=6`, { method: 'POST' })
+      fxsPollRef.current = setInterval(async () => {
+        const s = await fetch(`${API}/api/eco/fxs-sync/status`).then(x => x.json())
+        if (!s.running) {
+          clearInterval(fxsPollRef.current!)
+          setFxsSyncing(false)
+          const r = s.last_result || {}
+          if (r.error) { setFxsMsg(`Error: ${r.error}`); return }
+          setFxsMsg(`✓ FXStreet: ${r.total_upserted} events (${r.date_from} → ${r.date_to})`)
+          onImported()
+        }
+      }, 2000)
+    } catch { setFxsSyncing(false); setFxsMsg('FXStreet error') }
+  }

  const hasData = stats.total > 0
  const res = importStatus.last_result
@@ -185,14 +207,25 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
        </div>
      </div>

-      {/* Live sync — this week from faireconomy.media JSON */}
+      {/* FF live sync — this week actuals */}
      <button
        onClick={startSync}
        className="px-3 py-1.5 rounded bg-slate-600 hover:bg-slate-500 flex items-center gap-1.5 text-white"
        title="Update this week actuals from faireconomy.media (Forex Factory JSON)"
      >
        <RefreshCw size={13} />
-        Sync Live
+        Sync Live (FF)
+      </button>
+
+      {/* FXStreet — upcoming 6 weeks + forecasts, no API key */}
+      <button
+        onClick={startFxsSync}
+        disabled={fxsSyncing}
+        className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
+        title="Fetch upcoming 6 weeks with consensus forecasts from FXStreet (no API key required)"
+      >
+        <RefreshCw size={13} className={clsx(fxsSyncing && 'animate-spin')} />
+        Sync Upcoming (FXStreet)
      </button>

      {res && !importStatus.running && (
@@ -201,6 +234,11 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
          : <span className="text-emerald-400 text-xs flex items-center gap-1"><Check size={12}/> {res.inserted?.toLocaleString()} inserted</span>
      )}
      {syncResult && <span className="text-sky-400 text-xs">{syncResult}</span>}
+      {fxsMsg && (
+        <span className={clsx('text-xs', fxsMsg.startsWith('✓') ? 'text-emerald-400' : fxsMsg.startsWith('Error') ? 'text-red-400' : 'text-yellow-400')}>
+          {fxsMsg}
+        </span>
+      )}
    </div>
  )
 }