feat: FF HTML scraper for upcoming weeks with forecasts

- scrape_upcoming(weeks_ahead=5) in ff_calendar.py:
  fetches forexfactory.com/calendar?week=... HTML for N weeks ahead,
  parses calendar__table (date/time/currency/impact/event/forecast/previous),
  converts ET times to UTC, upserts into ff_calendar
- Daily scheduler in main.py: runs scrape_upcoming at startup (after 30s delay)
  then every 24h — no manual action needed
- New endpoints: POST /api/eco/ff-scrape?weeks=5, GET /api/eco/ff-scrape/status
- CalendarPage: "Scrape Upcoming (5w)" button (indigo) with polling + result

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/services/ff_calendar.py

@@ -1,18 +1,23 @@
 """
 Forex Factory calendar service.
 - CSV bulk import from forex_factory_cache.csv (2007-2025)
-- Live sync from nfs.faireconomy.media JSON endpoint (this week / next week)
+- Live sync from nfs.faireconomy.media JSON endpoint (this week)
+- HTML scraper for upcoming weeks (forexfactory.com/calendar?week=...)
 """
 import csv
 import json
 import logging
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, date, timezone, timedelta
 from typing import Any, Dict, List, Optional
+from zoneinfo import ZoneInfo
 
 import httpx
+from bs4 import BeautifulSoup
 
 logger = logging.getLogger(__name__)
 
+_ET = ZoneInfo("America/New_York")   # FF publishes times in US Eastern
+
 _FF_THIS_WEEK_URL = "https://nfs.faireconomy.media/ff_calendar_thisweek.json"
 _FF_NEXT_WEEK_URL = "https://nfs.faireconomy.media/ff_calendar_nextweek.json"
 
@@ -343,3 +348,210 @@ def get_calendar(
         }
     finally:
         conn.close()
+
+
+# ── FF HTML scraper for upcoming weeks ────────────────────────────────────────
+#
+# Forex Factory renders its calendar server-side; the HTML table is parseable
+# without JavaScript.  Times are in US Eastern Time (ET).
+#
+# URL pattern: https://www.forexfactory.com/calendar?week=jun26.2026
+# (any date in the week works; we use each Monday)
+
+_FF_BASE = "https://www.forexfactory.com"
+
+_FF_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/125.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Connection": "keep-alive",
+    "Upgrade-Insecure-Requests": "1",
+    "Referer": "https://www.forexfactory.com/",
+}
+
+# FF impact icon CSS class suffix → our normalized impact label
+_FF_IMPACT_CSS = {
+    "red": "high",
+    "ora": "medium",   # orange
+    "yel": "low",      # yellow
+    "gra": "non-economic",
+}
+
+
+def _week_url(monday: date) -> str:
+    """Return FF calendar URL for the week containing 'monday'."""
+    month = monday.strftime("%b").lower()   # "jun", "jul", …
+    return f"{_FF_BASE}/calendar?week={month}{monday.day}.{monday.year}"
+
+
+def _parse_et_time(time_str: str, ref_date: date) -> tuple[str, str]:
+    """
+    Parse an FF time string (e.g. '8:30am', 'Tentative', 'All Day')
+    expressed in US Eastern Time and return (event_date_utc, event_time_utc).
+    """
+    s = (time_str or "").strip().lower()
+    if not s or s in ("all day", "tentative", ""):
+        return str(ref_date), "00:00"
+    try:
+        # FF uses "8:30am" / "12:45pm" format
+        fmt = "%I:%M%p" if ":" in s else "%I%p"
+        t = datetime.strptime(s, fmt)
+        dt_et = datetime(
+            ref_date.year, ref_date.month, ref_date.day,
+            t.hour, t.minute, tzinfo=_ET,
+        )
+        dt_utc = dt_et.astimezone(timezone.utc)
+        return dt_utc.strftime("%Y-%m-%d"), dt_utc.strftime("%H:%M")
+    except Exception:
+        return str(ref_date), "00:00"
+
+
+def _parse_impact(impact_td) -> str:
+    if impact_td is None:
+        return "low"
+    span = impact_td.find("span", class_=lambda c: c and "icon--ff-impact" in c)
+    if span:
+        cls = " ".join(span.get("class", []))
+        for suffix, label in _FF_IMPACT_CSS.items():
+            if suffix in cls:
+                return label
+    # Fallback: check title attribute
+    title = (impact_td.get("title") or "").lower()
+    if "high" in title:   return "high"
+    if "medium" in title: return "medium"
+    return "low"
+
+
+def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]:
+    """Fetch and parse one week of FF calendar. Returns list of row tuples."""
+    url = _week_url(monday)
+    print(f"[FF scrape] GET {url}", flush=True)
+    try:
+        resp = client.get(url, timeout=20)
+        resp.raise_for_status()
+    except Exception as e:
+        print(f"[FF scrape] fetch error {url}: {e}", flush=True)
+        return []
+
+    soup = BeautifulSoup(resp.text, "lxml")
+    table = soup.find("table", class_="calendar__table")
+    if not table:
+        # Try alternate selector (FF occasionally restructures)
+        table = soup.find("table", attrs={"id": "calendar"})
+    if not table:
+        print(f"[FF scrape] table not found for {url}", flush=True)
+        return []
+
+    rows = table.find_all("tr", class_="calendar__row")
+    batch: list[tuple] = []
+    current_date: date = monday   # carry forward
+
+    for tr in rows:
+        # ── Date cell (only filled on day-breaker rows) ─────────────────
+        date_td = tr.find("td", class_="calendar__date")
+        if date_td and date_td.get_text(strip=True):
+            raw = date_td.get_text(" ", strip=True)  # e.g. "Fri Jun 27"
+            for fmt in ("%a %b %d", "%A %b %d", "%a %B %d"):
+                try:
+                    parsed = datetime.strptime(raw, fmt)
+                    current_date = date(monday.year, parsed.month, parsed.day)
+                    # Handle year wrap (Dec → Jan)
+                    if current_date < monday - timedelta(days=6):
+                        current_date = date(monday.year + 1, parsed.month, parsed.day)
+                    break
+                except ValueError:
+                    continue
+
+        # ── Time ────────────────────────────────────────────────────────
+        time_td = tr.find("td", class_="calendar__time")
+        time_str = time_td.get_text(strip=True) if time_td else ""
+
+        # ── Currency ────────────────────────────────────────────────────
+        cur_td = tr.find("td", class_="calendar__currency")
+        currency = cur_td.get_text(strip=True).upper() if cur_td else ""
+        if currency not in SUPPORTED_CURRENCIES:
+            continue
+
+        # ── Impact ──────────────────────────────────────────────────────
+        impact_td = tr.find("td", class_="calendar__impact")
+        impact = _parse_impact(impact_td)
+        if impact == "non-economic":
+            continue
+
+        # ── Event name ──────────────────────────────────────────────────
+        event_td = tr.find("td", class_="calendar__event")
+        if not event_td:
+            continue
+        title_span = event_td.find("span", class_="calendar__event-title")
+        event_name = (title_span or event_td).get_text(strip=True)
+        if not event_name:
+            continue
+
+        # ── Values ──────────────────────────────────────────────────────
+        actual   = _td_val(tr, "calendar__actual")
+        forecast = _td_val(tr, "calendar__forecast")
+        previous = _td_val(tr, "calendar__previous")
+
+        # ── Convert time ET → UTC ────────────────────────────────────────
+        ev_date, ev_time = _parse_et_time(time_str, current_date)
+
+        series_id = FF_TO_FRED.get(event_name)
+        batch.append((
+            ev_date, ev_time, currency, impact, event_name,
+            actual or None, forecast or None, previous or None,
+            series_id, None, "ff_scrape",
+        ))
+
+    return batch
+
+
+def _td_val(tr, cls: str) -> str:
+    td = tr.find("td", class_=cls)
+    if not td:
+        return ""
+    span = td.find("span")
+    text = (span or td).get_text(strip=True)
+    return text if text not in ("", "\xa0") else ""
+
+
+def scrape_upcoming(weeks_ahead: int = 5) -> Dict[str, Any]:
+    """
+    Scrape FF calendar HTML for the next N weeks and upsert into ff_calendar.
+    Includes the current week so we catch events added after the JSON sync.
+    """
+    from services.database import get_conn
+
+    today_date = date.today()
+    monday = today_date - timedelta(days=today_date.weekday())
+
+    total_inserted = 0
+    week_results = {}
+
+    conn = get_conn()
+    with httpx.Client(headers=_FF_HEADERS, follow_redirects=True, timeout=30) as client:
+        # Warm up session — get cookies from homepage
+        try:
+            client.get(_FF_BASE, timeout=10)
+        except Exception:
+            pass
+
+        for w in range(weeks_ahead):
+            week_monday = monday + timedelta(weeks=w)
+            batch = _scrape_week(client, week_monday)
+
+            if batch:
+                _upsert_batch(conn, batch)
+                conn.commit()
+                total_inserted += len(batch)
+
+            week_results[str(week_monday)] = len(batch)
+            print(f"[FF scrape] week {week_monday}: {len(batch)} events", flush=True)
+
+    conn.close()
+    print(f"[FF scrape] total upserted: {total_inserted}", flush=True)
+    return {"weeks_scraped": weeks_ahead, "total_upserted": total_inserted, "by_week": week_results}