From 5f23141992ea2bca5a342dd5e8bd5c3b09d8725e Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Tue, 21 Jul 2026 14:49:59 +0200 Subject: [PATCH] feat: calendar eco --- backend/services/calendar_sync.py | 10 ++++--- backend/services/ff_calendar.py | 48 ++++++++++++++++++++++++++----- backend/services/te_calendar.py | 6 +++- deploy/docker-compose.yml | 14 +++++++++ frontend/src/pages/Config.tsx | 2 +- 5 files changed, 67 insertions(+), 13 deletions(-) diff --git a/backend/services/calendar_sync.py b/backend/services/calendar_sync.py index 730ea50..8227b99 100644 --- a/backend/services/calendar_sync.py +++ b/backend/services/calendar_sync.py @@ -3,10 +3,12 @@ Unified calendar sync. 1. FF live JSON (nfs.faireconomy.media) → this week + next week (actual values, precise) 2. FF HTML scraper → weeks 3..N ahead (upcoming forecasts) 3. Fallback for weeks 3..N, only if step 2 got blocked (ForexFactory now 403s the - scraper — likely Cloudflare): Trading Economics first (services.te_calendar, free - tier, no card required), then FMP (services.fmp_calendar, free tier doesn't include - the economic_calendar endpoint — needs their paid Starter plan) — whichever has a - configured key and actually returns events. + scraper — likely Cloudflare): Trading Economics first (services.te_calendar), then + FMP (services.fmp_calendar) — whichever has a configured key and actually returns + events. NOTE: neither is actually free for this endpoint — TE's calendar needs + their $149/mo base plan, FMP's free tier excludes economic_calendar (needs their + $14.99/mo Starter). Kept wired for whichever the user ends up paying for; look for + a cheaper source before recommending either. Called by the background loop in main.py and manually via POST /api/eco/calendar-sync. """ import logging diff --git a/backend/services/ff_calendar.py b/backend/services/ff_calendar.py index d85ad55..2ed1075 100644 --- a/backend/services/ff_calendar.py +++ b/backend/services/ff_calendar.py @@ -435,18 +435,52 @@ def _parse_impact(impact_td) -> str: return "low" +def _fetch_via_flaresolverr(url: str) -> Optional[str]: + """ + Fetch a page through FlareSolverr (a headless-browser proxy that solves Cloudflare's + JS challenge) instead of a plain HTTP client. FF started 403'ing our direct requests + — the response is Cloudflare's "Just a moment..." interstitial, which no amount of + header/cookie spoofing can pass (confirmed: replaying a real logged-in session's + cf_clearance cookie from a different IP still 403s — it's IP-bound). FlareSolverr runs + as a sibling container (see deploy/docker-compose.yml) and is reached over the internal + docker network; not configured (e.g. local dev without that container) → returns None + so the caller falls back to the direct httpx path unchanged. + """ + import os + base = os.environ.get("FLARESOLVERR_URL", "http://flaresolverr:8191/v1") + try: + resp = httpx.post( + base, + json={"cmd": "request.get", "url": url, "maxTimeout": 60000}, + timeout=70, + ) + resp.raise_for_status() + data = resp.json() + if data.get("status") != "ok": + print(f"[FF scrape] FlareSolverr error for {url}: {data.get('message')}", flush=True) + return None + return data.get("solution", {}).get("response") + except Exception as e: + print(f"[FF scrape] FlareSolverr unavailable ({e}) — falling back to direct fetch", flush=True) + return None + + def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]: """Fetch and parse one week of FF calendar. Returns list of row tuples.""" url = _week_url(monday) print(f"[FF scrape] GET {url}", flush=True) - try: - resp = client.get(url, timeout=20) - resp.raise_for_status() - except Exception as e: - print(f"[FF scrape] fetch error {url}: {e}", flush=True) - return [] - soup = BeautifulSoup(resp.text, "lxml") + html = _fetch_via_flaresolverr(url) + if html is None: + try: + resp = client.get(url, timeout=20) + resp.raise_for_status() + html = resp.text + except Exception as e: + print(f"[FF scrape] fetch error {url}: {e}", flush=True) + return [] + + soup = BeautifulSoup(html, "lxml") table = soup.find("table", class_="calendar__table") if not table: # Try alternate selector (FF occasionally restructures) diff --git a/backend/services/te_calendar.py b/backend/services/te_calendar.py index 3e86a19..280c891 100644 --- a/backend/services/te_calendar.py +++ b/backend/services/te_calendar.py @@ -1,6 +1,10 @@ """ Trading Economics calendar service — upcoming events with consensus forecasts. -Free API: https://tradingeconomics.com/api/login (500 calls/month) +NOTE: confirmed (2026-07) that the calendar endpoint is NOT actually free despite the +signup page — their base paid plan is $149/month. Kept wired in case a key is ever +available, but don't recommend this as a free option (same dead end as FMP, whose +free tier also excludes the economic_calendar endpoint — needs their $14.99/mo Starter). +Signup: https://tradingeconomics.com/api/login Endpoint: GET https://api.tradingeconomics.com/calendar/country/united%20states?c=KEY """ import logging diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 1ec0a2f..2abda3e 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -6,9 +6,12 @@ services: dockerfile: Dockerfile environment: - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - FLARESOLVERR_URL=http://flaresolverr:8191/v1 volumes: - db_data:/app/data restart: unless-stopped + depends_on: + - flaresolverr networks: - internal healthcheck: @@ -17,6 +20,17 @@ services: timeout: 10s retries: 3 + # Solves Cloudflare's JS challenge via a headless browser — ForexFactory started 403'ing + # the backend's direct scraper requests (datacenter IP gets a much harder challenge than + # a residential browser). Internal-only, reached by the backend over the docker network. + flaresolverr: + image: ghcr.io/flaresolverr/flaresolverr:latest + environment: + - LOG_LEVEL=info + restart: unless-stopped + networks: + - internal + frontend: build: context: ../frontend diff --git a/frontend/src/pages/Config.tsx b/frontend/src/pages/Config.tsx index 1f33814..d4e5ef0 100644 --- a/frontend/src/pages/Config.tsx +++ b/frontend/src/pages/Config.tsx @@ -1646,7 +1646,7 @@ export default function Config() { { label: 'NewsAPI Key', value: newsapiKey, setter: setNewsapiKey, placeholder: 'Get at newsapi.org', configKey: 'newsapi_key' }, { label: 'EIA API Key', value: eiaKey, setter: setEiaKey, placeholder: 'Get at eia.gov', configKey: 'eia_api_key' }, { label: 'FRED API Key', value: fredKey, setter: setFredKey, placeholder: 'Get at fred.stlouisfed.org', configKey: 'fred_api_key' }, - { label: 'Trading Economics Key', value: teKey, setter: setTeKey, placeholder: 'Calendar fallback (free) — tradingeconomics.com/api/login', configKey: 'te_api_key' }, + { label: 'Trading Economics Key', value: teKey, setter: setTeKey, placeholder: 'Calendar fallback ($149/mo plan) — tradingeconomics.com/api/login', configKey: 'te_api_key' }, { label: 'FMP API Key', value: fmpKey, setter: setFmpKey, placeholder: 'Calendar fallback (paid tier) — financialmodelingprep.com', configKey: 'fmp_api_key' }, ].map(({ label, value, setter, placeholder, configKey }) => (