admin/OpenFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: event eco

Browse Source
This commit is contained in:
OpenSquared
2026-07-21 15:59:53 +02:00
parent 5f23141992
commit 24e5e4b03b
7 changed files with 129 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/main.py
View File

@@ -202,7 +202,9 @@ def startup():
except Exception as _e:
print(f"[Startup] FXStreet purge skipped: {_e}", flush=True)
# Calendar sync loop — FF only (live JSON + HTML scraper); interval from calendar_refresh_h config (default 6h)
# Calendar sync loop — FF live JSON every calendar_refresh_h (default 6h); the FF HTML
# scraper (heavier, one FlareSolverr browser fetch per week) is throttled separately
# inside calendar_sync() via calendar_scrape_refresh_h (default 24h, once/day)
import threading, time as _time
def _calendar_sync_loop():
_time.sleep(60) # let server fully boot

4
backend/routers/config.py
View File

@@ -15,6 +15,7 @@ class ApiKeysRequest(BaseModel):
fmp_api_key: Optional[str] = None
te_api_key: Optional[str] = None
calendar_refresh_h: Optional[str] = None
calendar_scrape_refresh_h: Optional[str] = None
class SourcesRequest(BaseModel):
@@ -73,6 +74,9 @@ def update_api_keys(req: ApiKeysRequest):
if req.calendar_refresh_h is not None:
set_config("calendar_refresh_h", req.calendar_refresh_h)
updated.append("calendar_refresh_h")
if req.calendar_scrape_refresh_h is not None:
set_config("calendar_scrape_refresh_h", req.calendar_scrape_refresh_h)
updated.append("calendar_scrape_refresh_h")
return {"status": "ok", "updated": updated}

6
backend/routers/eco.py
View File

@@ -543,7 +543,7 @@ def ff_page_import_status() -> Dict[str, Any]:
def _run_calendar_sync(weeks_ahead: int):
from services.calendar_sync import calendar_sync
try:
calendar_sync(weeks_ahead=weeks_ahead)
calendar_sync(weeks_ahead=weeks_ahead, force_scrape=True)
except Exception as e:
logger.error(f"[eco/calendar-sync] Failed: {e}")
@@ -553,7 +553,9 @@ def calendar_sync_ep(
background_tasks: BackgroundTasks,
weeks: int = Query(8, ge=1, le=12),
) -> Dict[str, Any]:
"""Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in background."""
"""Trigger unified calendar sync (FF live JSON thisweek+nextweek + FF HTML scraper) in
background. Manual trigger always runs the scraper regardless of its own throttle —
only the automatic background loop respects calendar_scrape_refresh_h."""
from services.calendar_sync import get_sync_status
if get_sync_status()["running"]:
raise HTTPException(409, "Calendar sync already running")

37
backend/services/calendar_sync.py
View File

@@ -26,11 +26,17 @@ _sync_status: Dict[str, Any] = {
}
def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
def calendar_sync(weeks_ahead: int = 8, force_scrape: bool = False) -> Dict[str, Any]:
"""
Sync upcoming economic events from ForexFactory only:
- FF live JSON covers this week + next week (with actuals as they publish)
- FF HTML scraper covers weeks 3..weeks_ahead for forecasts
- FF live JSON covers this week + next week (with actuals as they publish) — cheap,
runs every background-loop tick (calendar_refresh_h).
- FF HTML scraper (via FlareSolverr) covers weeks 3..weeks_ahead for forecasts — each
call launches a real headless-browser fetch per week, which is heavier and riskier
to run often (more requests FF's Cloudflare sees from us = more chance of getting
re-flagged), so it's throttled to its own interval (calendar_scrape_refresh_h,
config, default once/day) independent of the live-JSON cadence. `force_scrape=True`
(manual "Sync Now" in the UI) bypasses the throttle.
"""
global _sync_status
_sync_status["running"] = True
@@ -41,6 +47,7 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
try:
from services.ff_calendar import sync_live, scrape_upcoming
from services.database import get_config, set_config
# Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
print("[calendar_sync] step 1/3 — FF live JSON (thisweek + nextweek)", flush=True)
@@ -48,13 +55,31 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
result["live"] = r_live
print(f"[calendar_sync] live result: {r_live}", flush=True)
# Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
# Throttle: is the scraper (+ TE/FMP fallback) actually due?
scrape_interval_h = float(get_config("calendar_scrape_refresh_h") or 24)
last_scrape_iso = get_config("calendar_last_scrape_at")
elapsed_h = None
if last_scrape_iso:
try:
elapsed_h = (datetime.now(timezone.utc) - datetime.fromisoformat(last_scrape_iso)).total_seconds() / 3600
except ValueError:
elapsed_h = None
scrape_due = force_scrape or elapsed_h is None or elapsed_h >= scrape_interval_h
scrape_weeks = max(0, weeks_ahead - 2)
if scrape_weeks > 0:
if scrape_weeks == 0:
result["scrape"] = {"skipped": True}
elif not scrape_due:
wait_h = round(scrape_interval_h - (elapsed_h or 0), 1)
print(f"[calendar_sync] step 2/3 — scrape not due yet (next in ~{wait_h}h), skipping", flush=True)
result["scrape"] = {"skipped": "not_due", "next_due_in_h": wait_h}
else:
# Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
print(f"[calendar_sync] step 2/3 — FF HTML scrape, {scrape_weeks} week(s) ahead", flush=True)
r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
result["scrape"] = r_scrape
print(f"[calendar_sync] scrape result: {r_scrape}", flush=True)
set_config("calendar_last_scrape_at", datetime.now(timezone.utc).isoformat())
# FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can
# silently come back empty — fall back to Trading Economics (free tier), then
@@ -88,8 +113,6 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
result["fmp_fallback"] = {"skipped": "no_key"}
else:
print("[calendar_sync] step 3/3 — scrape got events, TE/FMP fallback not needed", flush=True)
else:
result["scrape"] = {"skipped": True}
except Exception as e:
logger.error(f"[calendar_sync] FF sync failed: {e}")