feat: FF HTML scraper for upcoming weeks with forecasts
- scrape_upcoming(weeks_ahead=5) in ff_calendar.py: fetches forexfactory.com/calendar?week=... HTML for N weeks ahead, parses calendar__table (date/time/currency/impact/event/forecast/previous), converts ET times to UTC, upserts into ff_calendar - Daily scheduler in main.py: runs scrape_upcoming at startup (after 30s delay) then every 24h — no manual action needed - New endpoints: POST /api/eco/ff-scrape?weeks=5, GET /api/eco/ff-scrape/status - CalendarPage: "Scrape Upcoming (5w)" button (indigo) with polling + result Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -295,6 +295,7 @@ def upcoming_events() -> List[Dict[str, Any]]:
|
||||
|
||||
_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None}
|
||||
_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None}
|
||||
_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None}
|
||||
|
||||
# CSV search order: Docker image (/app), uploaded to /tmp, local dev paths
|
||||
_FF_CSV_CANDIDATES = [
|
||||
@@ -387,6 +388,37 @@ def ff_sync_status_ep() -> Dict[str, Any]:
|
||||
return _ff_sync_status
|
||||
|
||||
|
||||
def _run_ff_scrape(weeks_ahead: int = 5):
|
||||
global _ff_scrape_status
|
||||
_ff_scrape_status["running"] = True
|
||||
try:
|
||||
from services.ff_calendar import scrape_upcoming
|
||||
result = scrape_upcoming(weeks_ahead=weeks_ahead)
|
||||
_ff_scrape_status["last_result"] = result
|
||||
except Exception as e:
|
||||
logger.error(f"[eco/ff-scrape] Failed: {e}")
|
||||
_ff_scrape_status["last_result"] = {"error": str(e)}
|
||||
finally:
|
||||
_ff_scrape_status["running"] = False
|
||||
|
||||
|
||||
@router.post("/ff-scrape")
|
||||
def ff_scrape(
|
||||
background_tasks: BackgroundTasks,
|
||||
weeks: int = Query(5, ge=1, le=8, description="Number of weeks ahead to scrape"),
|
||||
) -> Dict[str, Any]:
|
||||
"""Scrape forexfactory.com HTML calendar for upcoming N weeks (incl. forecasts)."""
|
||||
if _ff_scrape_status["running"]:
|
||||
raise HTTPException(409, "Scrape already running")
|
||||
background_tasks.add_task(_run_ff_scrape, weeks)
|
||||
return {"status": "started", "weeks_ahead": weeks}
|
||||
|
||||
|
||||
@router.get("/ff-scrape/status")
|
||||
def ff_scrape_status_ep() -> Dict[str, Any]:
|
||||
return _ff_scrape_status
|
||||
|
||||
|
||||
@router.get("/calendar")
|
||||
def ff_calendar(
|
||||
period: str = Query("recent", description="recent|today|tomorrow|yesterday|this_week|next_week|previous_week|this_month|next_month|previous_month"),
|
||||
|
||||
Reference in New Issue
Block a user