feat: calendar

backend/services/calendar_sync.py

@@ -1,5 +1,7 @@
 """
-Unified calendar sync — FXStreet (primary) with FF live fallback.
+Unified calendar sync — ForexFactory only.
+  1. FF live JSON (nfs.faireconomy.media) → this week + next week  (actual values, precise)
+  2. FF HTML scraper → weeks 3..N ahead                            (upcoming forecasts)
 Called by the background loop in main.py and manually via POST /api/eco/calendar-sync.
 """
 import logging
@@ -19,35 +21,38 @@ _sync_status: Dict[str, Any] = {
 
 def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
     """
-    Sync upcoming economic events:
-    1. Try FXStreet (no API key, ~300-400 events over weeks_ahead weeks)
-    2. Fall back to FF live JSON if FXStreet returns 403 or fails
+    Sync upcoming economic events from ForexFactory only:
+    - FF live JSON covers this week + next week (with actuals as they publish)
+    - FF HTML scraper covers weeks 3..weeks_ahead for forecasts
     """
     global _sync_status
     _sync_status["running"] = True
     _sync_status["last_run"] = datetime.now(timezone.utc).isoformat()
 
     result: Dict[str, Any] = {}
-    source = "unknown"
+    source = "ff"
 
     try:
-        from services.fxstreet_calendar import fetch_upcoming as fxs_fetch
-        r = fxs_fetch(weeks_ahead=weeks_ahead)
-        if r.get("error"):
-            logger.warning(f"[calendar_sync] FXStreet error: {r['error']} — falling back to FF live")
-            from services.ff_calendar import sync_live
-            r_ff = sync_live()
-            result = {**r_ff, "_fallback": True, "_fxs_error": r["error"]}
-            source = "ff_fallback"
+        from services.ff_calendar import sync_live, scrape_upcoming
+
+        # Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
+        r_live = sync_live()
+        result["live"] = r_live
+
+        # Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
+        scrape_weeks = max(0, weeks_ahead - 2)
+        if scrape_weeks > 0:
+            r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
+            result["scrape"] = r_scrape
         else:
-            result = r
-            source = "fxstreet"
+            result["scrape"] = {"skipped": True}
+
     except Exception as e:
-        logger.error(f"[calendar_sync] Exception: {e}")
+        logger.error(f"[calendar_sync] FF sync failed: {e}")
         result = {"error": str(e)}
         source = "error"
 
-    # After calendar upsert, log any forecast/actual changes to macro_series_log
+    # Log forecast/actual changes to macro_series_log
     try:
         from services.database import get_conn
         from services.macro_series_log import scan_and_log_all

backend/services/ff_calendar.py

@@ -163,6 +163,7 @@ def _upsert_batch(conn, batch: list):
              forecast_value = COALESCE(excluded.forecast_value, ff_calendar.forecast_value),
              previous_value = COALESCE(excluded.previous_value, ff_calendar.previous_value),
              series_id      = COALESCE(excluded.series_id, ff_calendar.series_id),
+             source         = excluded.source,
              fetched_at     = datetime('now')
         """,
         batch,

backend/services/ff_page_parser.py

@@ -0,0 +1,166 @@
+"""
+Parser for ForexFactory calendar page source (HTML).
+FF embeds a rich JSON object directly in the page:
+  window.calendarComponentStates[1] = { days: [...] }
+
+Each event has: dateline (unix ts UTC), currency (ISO), name, impactName,
+actual, forecast, previous, revision.
+
+Usage:
+  rows = parse_ff_page(html_string)
+  # → list of tuples ready for ff_calendar._upsert_batch()
+"""
+import json
+import logging
+import re
+from datetime import datetime, timezone
+
+logger = logging.getLogger(__name__)
+
+# Currencies we care about (same set as ff_calendar.py)
+_SUPPORTED = {"USD", "EUR", "GBP", "JPY", "AUD", "CAD", "NZD", "CHF", "CNY"}
+
+# Impact labels FF uses (impactName field in the embedded JSON)
+_IMPACT_OK = {"high", "medium", "low"}
+
+
+def _extract_days_json(html: str) -> list:
+    """
+    Pull the value of `days` array from window.calendarComponentStates[1].
+    Returns parsed list or raises ValueError.
+    """
+    # Find the start of the days array
+    m = re.search(r'window\.calendarComponentStates\s*\[\s*1\s*\]\s*=\s*\{\s*days\s*:\s*(\[)', html)
+    if not m:
+        raise ValueError("window.calendarComponentStates[1] not found in HTML")
+
+    start = m.start(1)
+    depth = 0
+    end = start
+    in_string = False
+    escape = False
+
+    for i in range(start, len(html)):
+        ch = html[i]
+        if escape:
+            escape = False
+            continue
+        if ch == '\\' and in_string:
+            escape = True
+            continue
+        if ch == '"':
+            in_string = not in_string
+            continue
+        if in_string:
+            continue
+        if ch == '[':
+            depth += 1
+        elif ch == ']':
+            depth -= 1
+            if depth == 0:
+                end = i + 1
+                break
+
+    raw = html[start:end]
+    return json.loads(raw)
+
+
+def parse_ff_page(html: str) -> tuple[list, dict]:
+    """
+    Parse FF calendar HTML page source.
+    Returns (rows, stats) where rows are tuples for _upsert_batch().
+    """
+    days = _extract_days_json(html)
+
+    rows = []
+    skipped = 0
+    date_min = date_max = None
+
+    for day in days:
+        for ev in day.get("events", []):
+            dateline = ev.get("dateline")
+            if not dateline:
+                skipped += 1
+                continue
+
+            currency = (ev.get("currency") or "").strip()
+            if currency not in _SUPPORTED:
+                skipped += 1
+                continue
+
+            impact = (ev.get("impactName") or "low").lower()
+            if impact not in _IMPACT_OK:
+                skipped += 1
+                continue
+
+            name = (ev.get("name") or "").strip()
+            if not name:
+                skipped += 1
+                continue
+
+            dt = datetime.fromtimestamp(int(dateline), tz=timezone.utc)
+            event_date = dt.strftime("%Y-%m-%d")
+            # "All Day" events: FF sets timeMasked=true — normalize to "00:00"
+            # so they match rows inserted by the HTML scraper (same convention)
+            event_time = "00:00" if ev.get("timeMasked") else dt.strftime("%H:%M")
+
+            if date_min is None or event_date < date_min:
+                date_min = event_date
+            if date_max is None or event_date > date_max:
+                date_max = event_date
+
+            actual   = (ev.get("actual")   or "").strip() or None
+            forecast = (ev.get("forecast") or "").strip() or None
+            revision = (ev.get("revision") or "").strip()
+            previous = revision or (ev.get("previous") or "").strip() or None
+
+            rows.append((
+                event_date, event_time, currency, impact, name,
+                actual, forecast, previous,
+                None, None, "ff_page",
+            ))
+
+    stats = {
+        "total": len(rows) + skipped,
+        "inserted": len(rows),
+        "skipped": skipped,
+        "date_from": date_min,
+        "date_to": date_max,
+    }
+    return rows, stats
+
+
+def import_ff_page_file(path: str) -> dict:
+    """Read an FF calendar HTML file and upsert into ff_calendar."""
+    from services.database import get_conn
+    from services.ff_calendar import _upsert_batch
+
+    try:
+        with open(path, encoding="utf-8", errors="replace") as f:
+            html = f.read()
+    except Exception as e:
+        return {"error": f"Cannot read file: {e}"}
+
+    try:
+        rows, stats = parse_ff_page(html)
+    except ValueError as e:
+        return {"error": str(e)}
+    except Exception as e:
+        logger.error(f"[ff_page_parser] parse error: {e}")
+        return {"error": f"Parse error: {e}"}
+
+    if not rows:
+        return {**stats, "error": "No usable events found — check the HTML is a full FF calendar page"}
+
+    conn = get_conn()
+    try:
+        _upsert_batch(conn, rows)
+        conn.commit()
+        logger.info(f"[ff_page_parser] {stats['inserted']} rows upserted ({stats['date_from']} → {stats['date_to']})")
+    except Exception as e:
+        logger.error(f"[ff_page_parser] DB error: {e}")
+        return {**stats, "error": f"DB error: {e}"}
+    finally:
+        conn.close()
+
+    return stats