feat: FF HTML scraper for upcoming weeks with forecasts

- scrape_upcoming(weeks_ahead=5) in ff_calendar.py:
  fetches forexfactory.com/calendar?week=... HTML for N weeks ahead,
  parses calendar__table (date/time/currency/impact/event/forecast/previous),
  converts ET times to UTC, upserts into ff_calendar
- Daily scheduler in main.py: runs scrape_upcoming at startup (after 30s delay)
  then every 24h — no manual action needed
- New endpoints: POST /api/eco/ff-scrape?weeks=5, GET /api/eco/ff-scrape/status
- CalendarPage: "Scrape Upcoming (5w)" button (indigo) with polling + result

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-26 17:14:00 +02:00
parent a69ced8fff
commit c36b2b2198
4 changed files with 305 additions and 5 deletions

View File

@@ -295,6 +295,7 @@ def upcoming_events() -> List[Dict[str, Any]]:
_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None}
_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None}
_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None}
# CSV search order: Docker image (/app), uploaded to /tmp, local dev paths
_FF_CSV_CANDIDATES = [
@@ -387,6 +388,37 @@ def ff_sync_status_ep() -> Dict[str, Any]:
return _ff_sync_status
def _run_ff_scrape(weeks_ahead: int = 5):
global _ff_scrape_status
_ff_scrape_status["running"] = True
try:
from services.ff_calendar import scrape_upcoming
result = scrape_upcoming(weeks_ahead=weeks_ahead)
_ff_scrape_status["last_result"] = result
except Exception as e:
logger.error(f"[eco/ff-scrape] Failed: {e}")
_ff_scrape_status["last_result"] = {"error": str(e)}
finally:
_ff_scrape_status["running"] = False
@router.post("/ff-scrape")
def ff_scrape(
background_tasks: BackgroundTasks,
weeks: int = Query(5, ge=1, le=8, description="Number of weeks ahead to scrape"),
) -> Dict[str, Any]:
"""Scrape forexfactory.com HTML calendar for upcoming N weeks (incl. forecasts)."""
if _ff_scrape_status["running"]:
raise HTTPException(409, "Scrape already running")
background_tasks.add_task(_run_ff_scrape, weeks)
return {"status": "started", "weeks_ahead": weeks}
@router.get("/ff-scrape/status")
def ff_scrape_status_ep() -> Dict[str, Any]:
return _ff_scrape_status
@router.get("/calendar")
def ff_calendar(
period: str = Query("recent", description="recent|today|tomorrow|yesterday|this_week|next_week|previous_week|this_month|next_month|previous_month"),