diff --git a/backend/main.py b/backend/main.py index 4d93811..f65c0d8 100644 --- a/backend/main.py +++ b/backend/main.py @@ -202,7 +202,9 @@ def startup(): except Exception as _e: print(f"[Startup] FXStreet purge skipped: {_e}", flush=True) - # Calendar sync loop — FF only (live JSON + HTML scraper); interval from calendar_refresh_h config (default 6h) + # Calendar sync loop — FF live JSON every calendar_refresh_h (default 6h); the FF HTML + # scraper (heavier, one FlareSolverr browser fetch per week) is throttled separately + # inside calendar_sync() via calendar_scrape_refresh_h (default 24h, once/day) import threading, time as _time def _calendar_sync_loop(): _time.sleep(60) # let server fully boot diff --git a/backend/routers/config.py b/backend/routers/config.py index 588a625..5f0acd7 100644 --- a/backend/routers/config.py +++ b/backend/routers/config.py @@ -15,6 +15,7 @@ class ApiKeysRequest(BaseModel): fmp_api_key: Optional[str] = None te_api_key: Optional[str] = None calendar_refresh_h: Optional[str] = None + calendar_scrape_refresh_h: Optional[str] = None class SourcesRequest(BaseModel): @@ -73,6 +74,9 @@ def update_api_keys(req: ApiKeysRequest): if req.calendar_refresh_h is not None: set_config("calendar_refresh_h", req.calendar_refresh_h) updated.append("calendar_refresh_h") + if req.calendar_scrape_refresh_h is not None: + set_config("calendar_scrape_refresh_h", req.calendar_scrape_refresh_h) + updated.append("calendar_scrape_refresh_h") return {"status": "ok", "updated": updated} diff --git a/backend/routers/eco.py b/backend/routers/eco.py index b139aef..38f7da8 100644 --- a/backend/routers/eco.py +++ b/backend/routers/eco.py @@ -543,7 +543,7 @@ def ff_page_import_status() -> Dict[str, Any]: def _run_calendar_sync(weeks_ahead: int): from services.calendar_sync import calendar_sync try: - calendar_sync(weeks_ahead=weeks_ahead) + calendar_sync(weeks_ahead=weeks_ahead, force_scrape=True) except Exception as e: logger.error(f"[eco/calendar-sync] Failed: {e}") @@ -553,7 +553,9 @@ def calendar_sync_ep( background_tasks: BackgroundTasks, weeks: int = Query(8, ge=1, le=12), ) -> Dict[str, Any]: - """Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in background.""" + """Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in + background. Manual trigger always runs the scraper regardless of its own throttle — + only the automatic background loop respects calendar_scrape_refresh_h.""" from services.calendar_sync import get_sync_status if get_sync_status()["running"]: raise HTTPException(409, "Calendar sync already running") diff --git a/backend/services/calendar_sync.py b/backend/services/calendar_sync.py index 8227b99..b03ead7 100644 --- a/backend/services/calendar_sync.py +++ b/backend/services/calendar_sync.py @@ -26,11 +26,17 @@ _sync_status: Dict[str, Any] = { } -def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]: +def calendar_sync(weeks_ahead: int = 8, force_scrape: bool = False) -> Dict[str, Any]: """ Sync upcoming economic events from ForexFactory only: - - FF live JSON covers this week + next week (with actuals as they publish) - - FF HTML scraper covers weeks 3..weeks_ahead for forecasts + - FF live JSON covers this week + next week (with actuals as they publish) — cheap, + runs every background-loop tick (calendar_refresh_h). + - FF HTML scraper (via FlareSolverr) covers weeks 3..weeks_ahead for forecasts — each + call launches a real headless-browser fetch per week, which is heavier and riskier + to run often (more requests FF's Cloudflare sees from us = more chance of getting + re-flagged), so it's throttled to its own interval (calendar_scrape_refresh_h, + config, default once/day) independent of the live-JSON cadence. `force_scrape=True` + (manual "Sync Now" in the UI) bypasses the throttle. """ global _sync_status _sync_status["running"] = True @@ -41,6 +47,7 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]: try: from services.ff_calendar import sync_live, scrape_upcoming + from services.database import get_config, set_config # Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time print("[calendar_sync] step 1/3 — FF live JSON (thisweek + nextweek)", flush=True) @@ -48,13 +55,31 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]: result["live"] = r_live print(f"[calendar_sync] live result: {r_live}", flush=True) - # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet) + # Throttle: is the scraper (+ TE/FMP fallback) actually due? + scrape_interval_h = float(get_config("calendar_scrape_refresh_h") or 24) + last_scrape_iso = get_config("calendar_last_scrape_at") + elapsed_h = None + if last_scrape_iso: + try: + elapsed_h = (datetime.now(timezone.utc) - datetime.fromisoformat(last_scrape_iso)).total_seconds() / 3600 + except ValueError: + elapsed_h = None + scrape_due = force_scrape or elapsed_h is None or elapsed_h >= scrape_interval_h + scrape_weeks = max(0, weeks_ahead - 2) - if scrape_weeks > 0: + if scrape_weeks == 0: + result["scrape"] = {"skipped": True} + elif not scrape_due: + wait_h = round(scrape_interval_h - (elapsed_h or 0), 1) + print(f"[calendar_sync] step 2/3 — scrape not due yet (next in ~{wait_h}h), skipping", flush=True) + result["scrape"] = {"skipped": "not_due", "next_due_in_h": wait_h} + else: + # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet) print(f"[calendar_sync] step 2/3 — FF HTML scrape, {scrape_weeks} week(s) ahead", flush=True) r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks) result["scrape"] = r_scrape print(f"[calendar_sync] scrape result: {r_scrape}", flush=True) + set_config("calendar_last_scrape_at", datetime.now(timezone.utc).isoformat()) # FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can # silently come back empty — fall back to Trading Economics (free tier), then @@ -88,8 +113,6 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]: result["fmp_fallback"] = {"skipped": "no_key"} else: print("[calendar_sync] step 3/3 — scrape got events, TE/FMP fallback not needed", flush=True) - else: - result["scrape"] = {"skipped": True} except Exception as e: logger.error(f"[calendar_sync] FF sync failed: {e}") diff --git a/frontend/src/pages/CalendarPage.tsx b/frontend/src/pages/CalendarPage.tsx index 09284d9..a60ef40 100644 --- a/frontend/src/pages/CalendarPage.tsx +++ b/frontend/src/pages/CalendarPage.tsx @@ -217,7 +217,9 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => const tw = twStatus === 'ok' ? r.live.thisweek.events : (r.live?.thisweek?.error ? `err` : (twStatus ?? '?')) const nw = nwStatus === 'ok' ? r.live.nextweek.events : (r.live?.nextweek?.error ? `err` : (nwStatus ?? '?')) const sc = r.scrape?.total_upserted ?? 0 - let msg = `✓ FF: ${tw}+${nw} live, ${sc} upcoming` + let msg = r.scrape?.skipped === 'not_due' + ? `✓ FF: ${tw}+${nw} live · scrape skipped (next in ~${r.scrape.next_due_in_h}h)` + : `✓ FF: ${tw}+${nw} live, ${sc} upcoming` if (r.te_fallback) { if (r.te_fallback.skipped === 'no_key') msg += ' · TE fallback skipped (no key)' else if (r.te_fallback.error) msg += ` · TE error: ${r.te_fallback.error}` @@ -294,7 +296,7 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => onClick={startCalendarSync} disabled={calStatus.running} className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white" - title="Sync economic calendar (FF live JSON + HTML scraper; FMP fallback for weeks 3+ if FF scraper is blocked)" + title="Sync economic calendar (live JSON + HTML scraper for weeks 3+, always forced on manual sync regardless of its own throttle in Config)" > {calStatus.running ? 'Syncing…' : 'Sync Now'} diff --git a/frontend/src/pages/Config.tsx b/frontend/src/pages/Config.tsx index d4e5ef0..e52cfb7 100644 --- a/frontend/src/pages/Config.tsx +++ b/frontend/src/pages/Config.tsx @@ -801,6 +801,7 @@ export default function Config() { const [fmpKey, setFmpKey] = useState('') const [teKey, setTeKey] = useState('') const [calendarRefreshH, setCalendarRefreshH] = useState(6) + const [calendarScrapeRefreshH, setCalendarScrapeRefreshH] = useState(24) const [showKeys, setShowKeys] = useState(false) const [savedMsg, setSavedMsg] = useState('') const [configTab, setConfigTab] = useState<'ia' | 'cycle' | 'sources' | 'options' | 'profiles' | 'watchlist' | 'data'>('ia') @@ -930,6 +931,10 @@ export default function Config() { const h = parseFloat(config.calendar_refresh_h) if (!isNaN(h)) setCalendarRefreshH(h) } + if (config?.calendar_scrape_refresh_h) { + const h = parseFloat(config.calendar_scrape_refresh_h) + if (!isNaN(h)) setCalendarScrapeRefreshH(h) + } }, [config]) useEffect(() => { @@ -1601,10 +1606,10 @@ export default function Config() { Calendar Auto-Sync
- Interval between automatic economic calendar syncs (Forex Factory live JSON + HTML scraper; FMP API used as fallback for weeks 3+ if the FF scraper gets blocked — needs an FMP key below). + Live JSON (this week's actuals/forecasts) is cheap and safe to poll often. The HTML scraper (weeks 3+, one FlareSolverr headless-browser fetch per week) is heavier and more likely to get re-flagged by Cloudflare if run too often — kept on its own, slower interval.
- +
{[2, 4, 6, 12, 24].map(h => ( ))}
-
+
+ +
+ {[24, 48, 72, 168].map(h => ( + + ))} +
+
+ )} diff --git a/frontend/src/pages/Dashboard.tsx b/frontend/src/pages/Dashboard.tsx index 19aaf99..14fb30a 100644 --- a/frontend/src/pages/Dashboard.tsx +++ b/frontend/src/pages/Dashboard.tsx @@ -51,12 +51,6 @@ const IMPACT_DOT: Record = { high: 'bg-red-500', medium: 'bg-yellow-400', low: 'bg-slate-400', } -const isTodayISO = (dateStr: string) => { - const t = new Date() - const utc = `${t.getUTCFullYear()}-${String(t.getUTCMonth() + 1).padStart(2, '0')}-${String(t.getUTCDate()).padStart(2, '0')}` - return dateStr === utc -} - const formatDateShort = (dateStr: string) => { const d = new Date(dateStr + 'T00:00:00Z') return d.toLocaleDateString('fr-FR', { weekday: 'short', day: 'numeric', month: 'short', timeZone: 'UTC' }) @@ -123,10 +117,29 @@ function QuoteRow({ q }: { q: Quote }) { ) } +function EcoEventRow({ ev, showActual }: { ev: any; showActual: boolean }) { + return ( +
+ {CURRENCY_FLAGS[ev.currency] ?? ev.currency} + + {ev.event_name} + {showActual && ev.actual_value ? ( + + {ev.actual_value} + {ev.forecast_value && /{ev.forecast_value}} + + ) : ev.forecast_value ? ( + F {ev.forecast_value} + ) : null} + {ev.event_time && {ev.event_time}} +
+ ) +} + export default function Dashboard() { const { data: riskScore, isLoading: riskLoading } = useGeoRiskScore() const { data: allQuotes } = useAllQuotes() - const { data: ecoCalendarData } = useEcoCalendar({ period: 'recent', limit: 30 }) + const { data: ecoCalendarData } = useEcoCalendar({ period: 'recent', limit: 100 }) const { data: portfolio } = usePortfolioSummary() const { data: lastScoresData } = useLastScores() const { data: allPatternsData } = useAllPatterns() @@ -259,20 +272,32 @@ export default function Dashboard() { }) }, [watchlistQuotesData]) - // Upcoming FF economic events, chronological, grouped by date (fills available height, no fixed cap) - const upcomingEventsGrouped = useMemo(() => { + // FF economic events — today (with actual/forecast as they release) + rest of the + // current week (Mon-Sun), grouped by date. Past events beyond today are dropped — + // the point of this card is "am I up to date", not a historical log. + const { todayEvents, restOfWeekGrouped } = useMemo(() => { const events: any[] = (ecoCalendarData as any)?.events ?? [] - const todayISO = new Date().toISOString().slice(0, 10) - const sorted = [...events] - .filter((ev: any) => ev.event_date >= todayISO) + const now = new Date() + const todayISO = now.toISOString().slice(0, 10) + const dow = now.getUTCDay() // 0=Sun..6=Sat + const sunday = new Date(now) + sunday.setUTCDate(now.getUTCDate() + (dow === 0 ? 0 : 7 - dow)) + const weekEndISO = sunday.toISOString().slice(0, 10) + + const today = events + .filter((ev: any) => ev.event_date === todayISO) + .sort((a: any, b: any) => (a.event_time ?? '').localeCompare(b.event_time ?? '')) + + const restOfWeek = events + .filter((ev: any) => ev.event_date > todayISO && ev.event_date <= weekEndISO) .sort((a: any, b: any) => `${a.event_date}${a.event_time ?? ''}`.localeCompare(`${b.event_date}${b.event_time ?? ''}`)) const groups: { date: string; events: any[] }[] = [] - for (const ev of sorted) { + for (const ev of restOfWeek) { const last = groups[groups.length - 1] if (!last || last.date !== ev.event_date) groups.push({ date: ev.event_date, events: [ev] }) else last.events.push(ev) } - return groups + return { todayEvents: today, restOfWeekGrouped: groups } }, [ecoCalendarData]) return ( @@ -542,31 +567,33 @@ export default function Dashboard() { ) })()} - {/* Economic Events — real Forex Factory feed, grouped by date */} + {/* Economic Events — real Forex Factory feed: today (actual as it releases) + rest of week */}
Economic Events
- {upcomingEventsGrouped.length > 0 ? ( + {(todayEvents.length > 0 || restOfWeekGrouped.length > 0) ? (
- {upcomingEventsGrouped.map(group => ( + {todayEvents.length > 0 && ( +
+
+ Aujourd'hui TODAY +
+
+ {todayEvents.map((ev: any, i: number) => )} +
+
+ )} + {restOfWeekGrouped.map(group => (
{formatDateShort(group.date)} - {isTodayISO(group.date) && TODAY}
- {group.events.map((ev: any, i: number) => ( -
- {CURRENCY_FLAGS[ev.currency] ?? ev.currency} - - {ev.event_name} - {ev.event_time && {ev.event_time}} -
- ))} + {group.events.map((ev: any, i: number) => )}
))}
- ) :
No upcoming events
} + ) :
No events this week
}