feat: calendar eco
This commit is contained in:
@@ -3,10 +3,12 @@ Unified calendar sync.
|
|||||||
1. FF live JSON (nfs.faireconomy.media) → this week + next week (actual values, precise)
|
1. FF live JSON (nfs.faireconomy.media) → this week + next week (actual values, precise)
|
||||||
2. FF HTML scraper → weeks 3..N ahead (upcoming forecasts)
|
2. FF HTML scraper → weeks 3..N ahead (upcoming forecasts)
|
||||||
3. Fallback for weeks 3..N, only if step 2 got blocked (ForexFactory now 403s the
|
3. Fallback for weeks 3..N, only if step 2 got blocked (ForexFactory now 403s the
|
||||||
scraper — likely Cloudflare): Trading Economics first (services.te_calendar, free
|
scraper — likely Cloudflare): Trading Economics first (services.te_calendar), then
|
||||||
tier, no card required), then FMP (services.fmp_calendar, free tier doesn't include
|
FMP (services.fmp_calendar) — whichever has a configured key and actually returns
|
||||||
the economic_calendar endpoint — needs their paid Starter plan) — whichever has a
|
events. NOTE: neither is actually free for this endpoint — TE's calendar needs
|
||||||
configured key and actually returns events.
|
their $149/mo base plan, FMP's free tier excludes economic_calendar (needs their
|
||||||
|
$14.99/mo Starter). Kept wired for whichever the user ends up paying for; look for
|
||||||
|
a cheaper source before recommending either.
|
||||||
Called by the background loop in main.py and manually via POST /api/eco/calendar-sync.
|
Called by the background loop in main.py and manually via POST /api/eco/calendar-sync.
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
|||||||
@@ -435,18 +435,52 @@ def _parse_impact(impact_td) -> str:
|
|||||||
return "low"
|
return "low"
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_via_flaresolverr(url: str) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
Fetch a page through FlareSolverr (a headless-browser proxy that solves Cloudflare's
|
||||||
|
JS challenge) instead of a plain HTTP client. FF started 403'ing our direct requests
|
||||||
|
— the response is Cloudflare's "Just a moment..." interstitial, which no amount of
|
||||||
|
header/cookie spoofing can pass (confirmed: replaying a real logged-in session's
|
||||||
|
cf_clearance cookie from a different IP still 403s — it's IP-bound). FlareSolverr runs
|
||||||
|
as a sibling container (see deploy/docker-compose.yml) and is reached over the internal
|
||||||
|
docker network; not configured (e.g. local dev without that container) → returns None
|
||||||
|
so the caller falls back to the direct httpx path unchanged.
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
base = os.environ.get("FLARESOLVERR_URL", "http://flaresolverr:8191/v1")
|
||||||
|
try:
|
||||||
|
resp = httpx.post(
|
||||||
|
base,
|
||||||
|
json={"cmd": "request.get", "url": url, "maxTimeout": 60000},
|
||||||
|
timeout=70,
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
data = resp.json()
|
||||||
|
if data.get("status") != "ok":
|
||||||
|
print(f"[FF scrape] FlareSolverr error for {url}: {data.get('message')}", flush=True)
|
||||||
|
return None
|
||||||
|
return data.get("solution", {}).get("response")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[FF scrape] FlareSolverr unavailable ({e}) — falling back to direct fetch", flush=True)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]:
|
def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]:
|
||||||
"""Fetch and parse one week of FF calendar. Returns list of row tuples."""
|
"""Fetch and parse one week of FF calendar. Returns list of row tuples."""
|
||||||
url = _week_url(monday)
|
url = _week_url(monday)
|
||||||
print(f"[FF scrape] GET {url}", flush=True)
|
print(f"[FF scrape] GET {url}", flush=True)
|
||||||
try:
|
|
||||||
resp = client.get(url, timeout=20)
|
|
||||||
resp.raise_for_status()
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
|
|
||||||
return []
|
|
||||||
|
|
||||||
soup = BeautifulSoup(resp.text, "lxml")
|
html = _fetch_via_flaresolverr(url)
|
||||||
|
if html is None:
|
||||||
|
try:
|
||||||
|
resp = client.get(url, timeout=20)
|
||||||
|
resp.raise_for_status()
|
||||||
|
html = resp.text
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
|
||||||
|
return []
|
||||||
|
|
||||||
|
soup = BeautifulSoup(html, "lxml")
|
||||||
table = soup.find("table", class_="calendar__table")
|
table = soup.find("table", class_="calendar__table")
|
||||||
if not table:
|
if not table:
|
||||||
# Try alternate selector (FF occasionally restructures)
|
# Try alternate selector (FF occasionally restructures)
|
||||||
|
|||||||
@@ -1,6 +1,10 @@
|
|||||||
"""
|
"""
|
||||||
Trading Economics calendar service — upcoming events with consensus forecasts.
|
Trading Economics calendar service — upcoming events with consensus forecasts.
|
||||||
Free API: https://tradingeconomics.com/api/login (500 calls/month)
|
NOTE: confirmed (2026-07) that the calendar endpoint is NOT actually free despite the
|
||||||
|
signup page — their base paid plan is $149/month. Kept wired in case a key is ever
|
||||||
|
available, but don't recommend this as a free option (same dead end as FMP, whose
|
||||||
|
free tier also excludes the economic_calendar endpoint — needs their $14.99/mo Starter).
|
||||||
|
Signup: https://tradingeconomics.com/api/login
|
||||||
Endpoint: GET https://api.tradingeconomics.com/calendar/country/united%20states?c=KEY
|
Endpoint: GET https://api.tradingeconomics.com/calendar/country/united%20states?c=KEY
|
||||||
"""
|
"""
|
||||||
import logging
|
import logging
|
||||||
|
|||||||
@@ -6,9 +6,12 @@ services:
|
|||||||
dockerfile: Dockerfile
|
dockerfile: Dockerfile
|
||||||
environment:
|
environment:
|
||||||
- OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
- OPENAI_API_KEY=${OPENAI_API_KEY:-}
|
||||||
|
- FLARESOLVERR_URL=http://flaresolverr:8191/v1
|
||||||
volumes:
|
volumes:
|
||||||
- db_data:/app/data
|
- db_data:/app/data
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
|
depends_on:
|
||||||
|
- flaresolverr
|
||||||
networks:
|
networks:
|
||||||
- internal
|
- internal
|
||||||
healthcheck:
|
healthcheck:
|
||||||
@@ -17,6 +20,17 @@ services:
|
|||||||
timeout: 10s
|
timeout: 10s
|
||||||
retries: 3
|
retries: 3
|
||||||
|
|
||||||
|
# Solves Cloudflare's JS challenge via a headless browser — ForexFactory started 403'ing
|
||||||
|
# the backend's direct scraper requests (datacenter IP gets a much harder challenge than
|
||||||
|
# a residential browser). Internal-only, reached by the backend over the docker network.
|
||||||
|
flaresolverr:
|
||||||
|
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||||
|
environment:
|
||||||
|
- LOG_LEVEL=info
|
||||||
|
restart: unless-stopped
|
||||||
|
networks:
|
||||||
|
- internal
|
||||||
|
|
||||||
frontend:
|
frontend:
|
||||||
build:
|
build:
|
||||||
context: ../frontend
|
context: ../frontend
|
||||||
|
|||||||
@@ -1646,7 +1646,7 @@ export default function Config() {
|
|||||||
{ label: 'NewsAPI Key', value: newsapiKey, setter: setNewsapiKey, placeholder: 'Get at newsapi.org', configKey: 'newsapi_key' },
|
{ label: 'NewsAPI Key', value: newsapiKey, setter: setNewsapiKey, placeholder: 'Get at newsapi.org', configKey: 'newsapi_key' },
|
||||||
{ label: 'EIA API Key', value: eiaKey, setter: setEiaKey, placeholder: 'Get at eia.gov', configKey: 'eia_api_key' },
|
{ label: 'EIA API Key', value: eiaKey, setter: setEiaKey, placeholder: 'Get at eia.gov', configKey: 'eia_api_key' },
|
||||||
{ label: 'FRED API Key', value: fredKey, setter: setFredKey, placeholder: 'Get at fred.stlouisfed.org', configKey: 'fred_api_key' },
|
{ label: 'FRED API Key', value: fredKey, setter: setFredKey, placeholder: 'Get at fred.stlouisfed.org', configKey: 'fred_api_key' },
|
||||||
{ label: 'Trading Economics Key', value: teKey, setter: setTeKey, placeholder: 'Calendar fallback (free) — tradingeconomics.com/api/login', configKey: 'te_api_key' },
|
{ label: 'Trading Economics Key', value: teKey, setter: setTeKey, placeholder: 'Calendar fallback ($149/mo plan) — tradingeconomics.com/api/login', configKey: 'te_api_key' },
|
||||||
{ label: 'FMP API Key', value: fmpKey, setter: setFmpKey, placeholder: 'Calendar fallback (paid tier) — financialmodelingprep.com', configKey: 'fmp_api_key' },
|
{ label: 'FMP API Key', value: fmpKey, setter: setFmpKey, placeholder: 'Calendar fallback (paid tier) — financialmodelingprep.com', configKey: 'fmp_api_key' },
|
||||||
].map(({ label, value, setter, placeholder, configKey }) => (
|
].map(({ label, value, setter, placeholder, configKey }) => (
|
||||||
<div key={label}>
|
<div key={label}>
|
||||||
|
|||||||
Reference in New Issue
Block a user