feat: event eco

backend/services/calendar_sync.py

@@ -26,11 +26,17 @@ _sync_status: Dict[str, Any] = {
 }
 
 
-def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
+def calendar_sync(weeks_ahead: int = 8, force_scrape: bool = False) -> Dict[str, Any]:
     """
     Sync upcoming economic events from ForexFactory only:
-    - FF live JSON covers this week + next week (with actuals as they publish)
-    - FF HTML scraper covers weeks 3..weeks_ahead for forecasts
+    - FF live JSON covers this week + next week (with actuals as they publish) — cheap,
+      runs every background-loop tick (calendar_refresh_h).
+    - FF HTML scraper (via FlareSolverr) covers weeks 3..weeks_ahead for forecasts — each
+      call launches a real headless-browser fetch per week, which is heavier and riskier
+      to run often (more requests FF's Cloudflare sees from us = more chance of getting
+      re-flagged), so it's throttled to its own interval (calendar_scrape_refresh_h,
+      config, default once/day) independent of the live-JSON cadence. `force_scrape=True`
+      (manual "Sync Now" in the UI) bypasses the throttle.
     """
     global _sync_status
     _sync_status["running"] = True
@@ -41,6 +47,7 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
 
     try:
         from services.ff_calendar import sync_live, scrape_upcoming
+        from services.database import get_config, set_config
 
         # Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
         print("[calendar_sync] step 1/3 — FF live JSON (thisweek + nextweek)", flush=True)
@@ -48,13 +55,31 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
         result["live"] = r_live
         print(f"[calendar_sync] live result: {r_live}", flush=True)
 
-        # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
+        # Throttle: is the scraper (+ TE/FMP fallback) actually due?
+        scrape_interval_h = float(get_config("calendar_scrape_refresh_h") or 24)
+        last_scrape_iso = get_config("calendar_last_scrape_at")
+        elapsed_h = None
+        if last_scrape_iso:
+            try:
+                elapsed_h = (datetime.now(timezone.utc) - datetime.fromisoformat(last_scrape_iso)).total_seconds() / 3600
+            except ValueError:
+                elapsed_h = None
+        scrape_due = force_scrape or elapsed_h is None or elapsed_h >= scrape_interval_h
+
         scrape_weeks = max(0, weeks_ahead - 2)
-        if scrape_weeks > 0:
+        if scrape_weeks == 0:
+            result["scrape"] = {"skipped": True}
+        elif not scrape_due:
+            wait_h = round(scrape_interval_h - (elapsed_h or 0), 1)
+            print(f"[calendar_sync] step 2/3 — scrape not due yet (next in ~{wait_h}h), skipping", flush=True)
+            result["scrape"] = {"skipped": "not_due", "next_due_in_h": wait_h}
+        else:
+            # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
             print(f"[calendar_sync] step 2/3 — FF HTML scrape, {scrape_weeks} week(s) ahead", flush=True)
             r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
             result["scrape"] = r_scrape
             print(f"[calendar_sync] scrape result: {r_scrape}", flush=True)
+            set_config("calendar_last_scrape_at", datetime.now(timezone.utc).isoformat())
 
             # FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can
             # silently come back empty — fall back to Trading Economics (free tier), then
@@ -88,8 +113,6 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
                         result["fmp_fallback"] = {"skipped": "no_key"}
             else:
                 print("[calendar_sync] step 3/3 — scrape got events, TE/FMP fallback not needed", flush=True)
-        else:
-            result["scrape"] = {"skipped": True}
 
     except Exception as e:
         logger.error(f"[calendar_sync] FF sync failed: {e}")