feat: event eco

backend/main.py

@@ -202,7 +202,9 @@ def startup():
     except Exception as _e:
         print(f"[Startup] FXStreet purge skipped: {_e}", flush=True)
 
-    # Calendar sync loop — FF only (live JSON + HTML scraper); interval from calendar_refresh_h config (default 6h)
+    # Calendar sync loop — FF live JSON every calendar_refresh_h (default 6h); the FF HTML
+    # scraper (heavier, one FlareSolverr browser fetch per week) is throttled separately
+    # inside calendar_sync() via calendar_scrape_refresh_h (default 24h, once/day)
     import threading, time as _time
     def _calendar_sync_loop():
         _time.sleep(60)  # let server fully boot

backend/routers/config.py

@@ -15,6 +15,7 @@ class ApiKeysRequest(BaseModel):
     fmp_api_key: Optional[str] = None
     te_api_key: Optional[str] = None
     calendar_refresh_h: Optional[str] = None
+    calendar_scrape_refresh_h: Optional[str] = None
 
 
 class SourcesRequest(BaseModel):
@@ -73,6 +74,9 @@ def update_api_keys(req: ApiKeysRequest):
     if req.calendar_refresh_h is not None:
         set_config("calendar_refresh_h", req.calendar_refresh_h)
         updated.append("calendar_refresh_h")
+    if req.calendar_scrape_refresh_h is not None:
+        set_config("calendar_scrape_refresh_h", req.calendar_scrape_refresh_h)
+        updated.append("calendar_scrape_refresh_h")
     return {"status": "ok", "updated": updated}
 
 

backend/routers/eco.py

@@ -543,7 +543,7 @@ def ff_page_import_status() -> Dict[str, Any]:
 def _run_calendar_sync(weeks_ahead: int):
     from services.calendar_sync import calendar_sync
     try:
-        calendar_sync(weeks_ahead=weeks_ahead)
+        calendar_sync(weeks_ahead=weeks_ahead, force_scrape=True)
     except Exception as e:
         logger.error(f"[eco/calendar-sync] Failed: {e}")
 
@@ -553,7 +553,9 @@ def calendar_sync_ep(
     background_tasks: BackgroundTasks,
     weeks: int = Query(8, ge=1, le=12),
 ) -> Dict[str, Any]:
-    """Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in background."""
+    """Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in
+    background. Manual trigger always runs the scraper regardless of its own throttle —
+    only the automatic background loop respects calendar_scrape_refresh_h."""
     from services.calendar_sync import get_sync_status
     if get_sync_status()["running"]:
         raise HTTPException(409, "Calendar sync already running")

backend/services/calendar_sync.py

@@ -26,11 +26,17 @@ _sync_status: Dict[str, Any] = {
 }
 
 
-def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
+def calendar_sync(weeks_ahead: int = 8, force_scrape: bool = False) -> Dict[str, Any]:
     """
     Sync upcoming economic events from ForexFactory only:
-    - FF live JSON covers this week + next week (with actuals as they publish)
-    - FF HTML scraper covers weeks 3..weeks_ahead for forecasts
+    - FF live JSON covers this week + next week (with actuals as they publish) — cheap,
+      runs every background-loop tick (calendar_refresh_h).
+    - FF HTML scraper (via FlareSolverr) covers weeks 3..weeks_ahead for forecasts — each
+      call launches a real headless-browser fetch per week, which is heavier and riskier
+      to run often (more requests FF's Cloudflare sees from us = more chance of getting
+      re-flagged), so it's throttled to its own interval (calendar_scrape_refresh_h,
+      config, default once/day) independent of the live-JSON cadence. `force_scrape=True`
+      (manual "Sync Now" in the UI) bypasses the throttle.
     """
     global _sync_status
     _sync_status["running"] = True
@@ -41,6 +47,7 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
 
     try:
         from services.ff_calendar import sync_live, scrape_upcoming
+        from services.database import get_config, set_config
 
         # Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
         print("[calendar_sync] step 1/3 — FF live JSON (thisweek + nextweek)", flush=True)
@@ -48,13 +55,31 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
         result["live"] = r_live
         print(f"[calendar_sync] live result: {r_live}", flush=True)
 
-        # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
+        # Throttle: is the scraper (+ TE/FMP fallback) actually due?
+        scrape_interval_h = float(get_config("calendar_scrape_refresh_h") or 24)
+        last_scrape_iso = get_config("calendar_last_scrape_at")
+        elapsed_h = None
+        if last_scrape_iso:
+            try:
+                elapsed_h = (datetime.now(timezone.utc) - datetime.fromisoformat(last_scrape_iso)).total_seconds() / 3600
+            except ValueError:
+                elapsed_h = None
+        scrape_due = force_scrape or elapsed_h is None or elapsed_h >= scrape_interval_h
+
         scrape_weeks = max(0, weeks_ahead - 2)
-        if scrape_weeks > 0:
+        if scrape_weeks == 0:
+            result["scrape"] = {"skipped": True}
+        elif not scrape_due:
+            wait_h = round(scrape_interval_h - (elapsed_h or 0), 1)
+            print(f"[calendar_sync] step 2/3 — scrape not due yet (next in ~{wait_h}h), skipping", flush=True)
+            result["scrape"] = {"skipped": "not_due", "next_due_in_h": wait_h}
+        else:
+            # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
             print(f"[calendar_sync] step 2/3 — FF HTML scrape, {scrape_weeks} week(s) ahead", flush=True)
             r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
             result["scrape"] = r_scrape
             print(f"[calendar_sync] scrape result: {r_scrape}", flush=True)
+            set_config("calendar_last_scrape_at", datetime.now(timezone.utc).isoformat())
 
             # FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can
             # silently come back empty — fall back to Trading Economics (free tier), then
@@ -88,8 +113,6 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
                         result["fmp_fallback"] = {"skipped": "no_key"}
             else:
                 print("[calendar_sync] step 3/3 — scrape got events, TE/FMP fallback not needed", flush=True)
-        else:
-            result["scrape"] = {"skipped": True}
 
     except Exception as e:
         logger.error(f"[calendar_sync] FF sync failed: {e}")

frontend/src/pages/CalendarPage.tsx

@@ -217,7 +217,9 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
             const tw = twStatus === 'ok' ? r.live.thisweek.events : (r.live?.thisweek?.error ? `err` : (twStatus ?? '?'))
             const nw = nwStatus === 'ok' ? r.live.nextweek.events : (r.live?.nextweek?.error ? `err` : (nwStatus ?? '?'))
             const sc = r.scrape?.total_upserted ?? 0
-            let msg = `✓ FF: ${tw}+${nw} live, ${sc} upcoming`
+            let msg = r.scrape?.skipped === 'not_due'
+              ? `✓ FF: ${tw}+${nw} live · scrape skipped (next in ~${r.scrape.next_due_in_h}h)`
+              : `✓ FF: ${tw}+${nw} live, ${sc} upcoming`
             if (r.te_fallback) {
               if (r.te_fallback.skipped === 'no_key') msg += ' · TE fallback skipped (no key)'
               else if (r.te_fallback.error) msg += ` · TE error: ${r.te_fallback.error}`
@@ -294,7 +296,7 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
         onClick={startCalendarSync}
         disabled={calStatus.running}
         className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
-        title="Sync economic calendar (FF live JSON + HTML scraper; FMP fallback for weeks 3+ if FF scraper is blocked)"
+        title="Sync economic calendar (live JSON + HTML scraper for weeks 3+, always forced on manual sync regardless of its own throttle in Config)"
       >
         <RefreshCw size={13} className={clsx(calStatus.running && 'animate-spin')} />
         {calStatus.running ? 'Syncing…' : 'Sync Now'}

frontend/src/pages/Config.tsx

@@ -801,6 +801,7 @@ export default function Config() {
   const [fmpKey, setFmpKey] = useState('')
   const [teKey, setTeKey] = useState('')
   const [calendarRefreshH, setCalendarRefreshH] = useState(6)
+  const [calendarScrapeRefreshH, setCalendarScrapeRefreshH] = useState(24)
   const [showKeys, setShowKeys] = useState(false)
   const [savedMsg, setSavedMsg] = useState('')
   const [configTab, setConfigTab] = useState<'ia' | 'cycle' | 'sources' | 'options' | 'profiles' | 'watchlist' | 'data'>('ia')
@@ -930,6 +931,10 @@ export default function Config() {
       const h = parseFloat(config.calendar_refresh_h)
       if (!isNaN(h)) setCalendarRefreshH(h)
     }
+    if (config?.calendar_scrape_refresh_h) {
+      const h = parseFloat(config.calendar_scrape_refresh_h)
+      if (!isNaN(h)) setCalendarScrapeRefreshH(h)
+    }
   }, [config])
 
   useEffect(() => {
@@ -1601,10 +1606,10 @@ export default function Config() {
               <RefreshCw className="w-4 h-4 text-indigo-400" /> Calendar Auto-Sync
             </h2>
             <div className="text-xs text-slate-400 mb-3">
-              Interval between automatic economic calendar syncs (Forex Factory live JSON + HTML scraper; FMP API used as fallback for weeks 3+ if the FF scraper gets blocked — needs an FMP key below).
+              Live JSON (this week's actuals/forecasts) is cheap and safe to poll often. The HTML scraper (weeks 3+, one FlareSolverr headless-browser fetch per week) is heavier and more likely to get re-flagged by Cloudflare if run too often — kept on its own, slower interval.
             </div>
             <div className="flex items-center gap-3">
-              <label className="text-xs text-slate-500 whitespace-nowrap">Refresh every</label>
+              <label className="text-xs text-slate-500 whitespace-nowrap w-32 shrink-0">Live JSON every</label>
               <div className="flex gap-1">
                 {[2, 4, 6, 12, 24].map(h => (
                   <button key={h} onClick={() => setCalendarRefreshH(h)}
@@ -1616,16 +1621,30 @@ export default function Config() {
                   </button>
                 ))}
               </div>
-              <button
-                onClick={() => updateApiKeys({ calendar_refresh_h: String(calendarRefreshH) }, {
-                  onSuccess: () => { setSavedMsg('Calendar refresh saved'); setTimeout(() => setSavedMsg(''), 2000) }
-                })}
-                disabled={savingKeys}
-                className="flex items-center gap-1.5 bg-indigo-700 hover:bg-indigo-600 disabled:opacity-40 text-white px-3 py-1.5 rounded text-xs font-semibold">
-                <Save className="w-3.5 h-3.5" />
-                {savingKeys ? 'Saving…' : 'Save'}
-              </button>
             </div>
+            <div className="flex items-center gap-3 mt-2">
+              <label className="text-xs text-slate-500 whitespace-nowrap w-32 shrink-0">Scrape (weeks 3+) every</label>
+              <div className="flex gap-1">
+                {[24, 48, 72, 168].map(h => (
+                  <button key={h} onClick={() => setCalendarScrapeRefreshH(h)}
+                    className={clsx('px-3 py-1.5 rounded text-xs transition-colors', {
+                      'bg-indigo-700 text-white': calendarScrapeRefreshH === h,
+                      'bg-dark-700 text-slate-400 hover:text-slate-200': calendarScrapeRefreshH !== h,
+                    })}>
+                    {h < 168 ? `${h}h` : '7j'}
+                  </button>
+                ))}
+              </div>
+            </div>
+            <button
+              onClick={() => updateApiKeys({ calendar_refresh_h: String(calendarRefreshH), calendar_scrape_refresh_h: String(calendarScrapeRefreshH) }, {
+                onSuccess: () => { setSavedMsg('Calendar refresh saved'); setTimeout(() => setSavedMsg(''), 2000) }
+              })}
+              disabled={savingKeys}
+              className="mt-3 flex items-center gap-1.5 bg-indigo-700 hover:bg-indigo-600 disabled:opacity-40 text-white px-3 py-1.5 rounded text-xs font-semibold">
+              <Save className="w-3.5 h-3.5" />
+              {savingKeys ? 'Saving…' : 'Save'}
+            </button>
           </div>
         </div>
       )}

frontend/src/pages/Dashboard.tsx

@@ -51,12 +51,6 @@ const IMPACT_DOT: Record<string, string> = {
   high: 'bg-red-500', medium: 'bg-yellow-400', low: 'bg-slate-400',
 }
 
-const isTodayISO = (dateStr: string) => {
-  const t = new Date()
-  const utc = `${t.getUTCFullYear()}-${String(t.getUTCMonth() + 1).padStart(2, '0')}-${String(t.getUTCDate()).padStart(2, '0')}`
-  return dateStr === utc
-}
-
 const formatDateShort = (dateStr: string) => {
   const d = new Date(dateStr + 'T00:00:00Z')
   return d.toLocaleDateString('fr-FR', { weekday: 'short', day: 'numeric', month: 'short', timeZone: 'UTC' })
@@ -123,10 +117,29 @@ function QuoteRow({ q }: { q: Quote }) {
   )
 }
 
+function EcoEventRow({ ev, showActual }: { ev: any; showActual: boolean }) {
+  return (
+    <div className="flex items-center gap-1.5 text-[10px]">
+      <span className="shrink-0">{CURRENCY_FLAGS[ev.currency] ?? ev.currency}</span>
+      <span className={clsx('w-1.5 h-1.5 rounded-full shrink-0', IMPACT_DOT[ev.impact] ?? 'bg-slate-400')} />
+      <span className="text-white truncate flex-1 line-clamp-1">{ev.event_name}</span>
+      {showActual && ev.actual_value ? (
+        <span className="text-[9px] font-mono shrink-0">
+          <span className="text-white font-semibold">{ev.actual_value}</span>
+          {ev.forecast_value && <span className="text-slate-600"> /{ev.forecast_value}</span>}
+        </span>
+      ) : ev.forecast_value ? (
+        <span className="text-[9px] font-mono text-slate-600 shrink-0">F {ev.forecast_value}</span>
+      ) : null}
+      {ev.event_time && <span className="text-slate-700 text-[9px] shrink-0 font-mono">{ev.event_time}</span>}
+    </div>
+  )
+}
+
 export default function Dashboard() {
   const { data: riskScore, isLoading: riskLoading } = useGeoRiskScore()
   const { data: allQuotes } = useAllQuotes()
-  const { data: ecoCalendarData } = useEcoCalendar({ period: 'recent', limit: 30 })
+  const { data: ecoCalendarData } = useEcoCalendar({ period: 'recent', limit: 100 })
   const { data: portfolio } = usePortfolioSummary()
   const { data: lastScoresData } = useLastScores()
   const { data: allPatternsData } = useAllPatterns()
@@ -259,20 +272,32 @@ export default function Dashboard() {
     })
   }, [watchlistQuotesData])
 
-  // Upcoming FF economic events, chronological, grouped by date (fills available height, no fixed cap)
-  const upcomingEventsGrouped = useMemo(() => {
+  // FF economic events — today (with actual/forecast as they release) + rest of the
+  // current week (Mon-Sun), grouped by date. Past events beyond today are dropped —
+  // the point of this card is "am I up to date", not a historical log.
+  const { todayEvents, restOfWeekGrouped } = useMemo(() => {
     const events: any[] = (ecoCalendarData as any)?.events ?? []
-    const todayISO = new Date().toISOString().slice(0, 10)
-    const sorted = [...events]
-      .filter((ev: any) => ev.event_date >= todayISO)
+    const now = new Date()
+    const todayISO = now.toISOString().slice(0, 10)
+    const dow = now.getUTCDay() // 0=Sun..6=Sat
+    const sunday = new Date(now)
+    sunday.setUTCDate(now.getUTCDate() + (dow === 0 ? 0 : 7 - dow))
+    const weekEndISO = sunday.toISOString().slice(0, 10)
+
+    const today = events
+      .filter((ev: any) => ev.event_date === todayISO)
+      .sort((a: any, b: any) => (a.event_time ?? '').localeCompare(b.event_time ?? ''))
+
+    const restOfWeek = events
+      .filter((ev: any) => ev.event_date > todayISO && ev.event_date <= weekEndISO)
       .sort((a: any, b: any) => `${a.event_date}${a.event_time ?? ''}`.localeCompare(`${b.event_date}${b.event_time ?? ''}`))
     const groups: { date: string; events: any[] }[] = []
-    for (const ev of sorted) {
+    for (const ev of restOfWeek) {
       const last = groups[groups.length - 1]
       if (!last || last.date !== ev.event_date) groups.push({ date: ev.event_date, events: [ev] })
       else last.events.push(ev)
     }
-    return groups
+    return { todayEvents: today, restOfWeekGrouped: groups }
   }, [ecoCalendarData])
 
   return (
@@ -542,31 +567,33 @@ export default function Dashboard() {
           )
         })()}
 
-        {/* Economic Events — real Forex Factory feed, grouped by date */}
+        {/* Economic Events — real Forex Factory feed: today (actual as it releases) + rest of week */}
         <div className="card col-span-1 flex flex-col overflow-hidden" style={row1Height ? { height: row1Height } : undefined}>
           <div className="section-title flex items-center gap-1 shrink-0"><Clock className="w-3 h-3" /> Economic Events</div>
-          {upcomingEventsGrouped.length > 0 ? (
+          {(todayEvents.length > 0 || restOfWeekGrouped.length > 0) ? (
             <div className="mt-2 space-y-2 overflow-y-auto flex-1 min-h-0">
-              {upcomingEventsGrouped.map(group => (
+              {todayEvents.length > 0 && (
+                <div>
+                  <div className="flex items-center gap-1.5 text-[9px] font-semibold text-slate-300 bg-dark-700/50 rounded px-1.5 py-0.5 mb-1">
+                    Aujourd'hui <span className="badge-blue text-[8px] px-1 py-0">TODAY</span>
+                  </div>
+                  <div className="space-y-1 pl-0.5">
+                    {todayEvents.map((ev: any, i: number) => <EcoEventRow key={i} ev={ev} showActual />)}
+                  </div>
+                </div>
+              )}
+              {restOfWeekGrouped.map(group => (
                 <div key={group.date}>
                   <div className="flex items-center gap-1.5 text-[9px] font-semibold text-slate-300 bg-dark-700/50 rounded px-1.5 py-0.5 mb-1">
                     {formatDateShort(group.date)}
-                    {isTodayISO(group.date) && <span className="badge-blue text-[8px] px-1 py-0">TODAY</span>}
                   </div>
                   <div className="space-y-1 pl-0.5">
-                    {group.events.map((ev: any, i: number) => (
-                      <div key={i} className="flex items-center gap-1.5 text-[10px]">
-                        <span className="shrink-0">{CURRENCY_FLAGS[ev.currency] ?? ev.currency}</span>
-                        <span className={clsx('w-1.5 h-1.5 rounded-full shrink-0', IMPACT_DOT[ev.impact] ?? 'bg-slate-400')} />
-                        <span className="text-white truncate flex-1 line-clamp-1">{ev.event_name}</span>
-                        {ev.event_time && <span className="text-slate-700 text-[9px] shrink-0 font-mono">{ev.event_time}</span>}
-                      </div>
-                    ))}
+                    {group.events.map((ev: any, i: number) => <EcoEventRow key={i} ev={ev} showActual={false} />)}
                   </div>
                 </div>
               ))}
             </div>
-          ) : <div className="text-slate-600 text-xs mt-2">No upcoming events</div>}
+          ) : <div className="text-slate-600 text-xs mt-2">No events this week</div>}
         </div>
       </div>