diff --git a/backend/routers/instrument_models.py b/backend/routers/instrument_models.py index 32b1816..29ed0bf 100644 --- a/backend/routers/instrument_models.py +++ b/backend/routers/instrument_models.py @@ -809,3 +809,30 @@ def get_all_overrides(instrument: str) -> List[Dict[str, Any]]: return [dict(r) for r in rows] finally: conn.close() + + +class CalendarBackfillBody(BaseModel): + from_date: str # ISO date string "YYYY-MM-DD" + to_date: Optional[str] = None # defaults to today + + +@router.post("/ff-calendar/backfill") +def backfill_calendar(body: CalendarBackfillBody) -> Dict[str, Any]: + """ + Backfill ff_calendar with historical data from ForexFactory HTML scraper. + Scrapes week by week from from_date to to_date (or today). + Skips weeks already populated. Runs synchronously — may take several minutes for long ranges. + """ + from datetime import date as date_type + from services.ff_calendar import sync_historical_range + try: + from_d = date_type.fromisoformat(body.from_date[:10]) + to_d = date_type.fromisoformat(body.to_date[:10]) if body.to_date else date_type.today() + except ValueError as e: + raise HTTPException(status_code=400, detail=f"Invalid date: {e}") + + if (to_d - from_d).days > 730: + raise HTTPException(status_code=400, detail="Range too large (max 2 years)") + + result = sync_historical_range(from_d, to_d) + return result diff --git a/backend/services/ff_calendar.py b/backend/services/ff_calendar.py index c8aedf8..d85ad55 100644 --- a/backend/services/ff_calendar.py +++ b/backend/services/ff_calendar.py @@ -563,3 +563,93 @@ def scrape_upcoming(weeks_ahead: int = 5) -> Dict[str, Any]: conn.close() print(f"[FF scrape] total upserted: {total_inserted}", flush=True) return {"weeks_scraped": weeks_ahead, "total_upserted": total_inserted, "by_week": week_results} + + +def sync_historical_range(from_date: date, to_date: Optional[date] = None) -> Dict[str, Any]: + """ + Backfill ff_calendar for a historical date range by scraping FF HTML week by week. + Skips weeks that already have events with actual_value populated (avoids redundant requests). + Adds a small delay between requests to be respectful of the server. + Returns progress stats. + """ + import time + from services.database import get_conn + + if to_date is None: + to_date = date.today() + + # Align from_date to the Monday of its week + monday_start = from_date - timedelta(days=from_date.weekday()) + monday_today = to_date - timedelta(days=to_date.weekday()) + + # Collect all Mondays in range + mondays: list[date] = [] + cur = monday_start + while cur <= monday_today: + mondays.append(cur) + cur += timedelta(weeks=1) + + if not mondays: + return {"weeks_scraped": 0, "total_upserted": 0, "skipped": 0, "by_week": {}} + + conn = get_conn() + + # Pre-check which weeks already have actual values to avoid redundant scraping + # A week is "done" if it has ≥5 events with actual_value for supported currencies + weeks_with_actuals: set[str] = set() + for m in mondays: + sunday = m + timedelta(days=6) + cnt = conn.execute( + """SELECT COUNT(*) FROM ff_calendar + WHERE event_date >= ? AND event_date <= ? + AND actual_value IS NOT NULL""", + (str(m), str(sunday)) + ).fetchone()[0] + if cnt >= 5: + weeks_with_actuals.add(str(m)) + + total_inserted = 0 + skipped = 0 + week_results: Dict[str, Any] = {} + + with httpx.Client(headers=_FF_HEADERS, follow_redirects=True, timeout=30) as client: + # Warm up session + try: + client.get(_FF_BASE, timeout=10) + time.sleep(1.0) + except Exception: + pass + + for monday in mondays: + monday_str = str(monday) + if monday_str in weeks_with_actuals: + week_results[monday_str] = {"status": "skipped", "count": 0} + skipped += 1 + print(f"[FF backfill] {monday_str}: already populated, skipping", flush=True) + continue + + batch = _scrape_week(client, monday) + + if batch: + _upsert_batch(conn, batch) + conn.commit() + total_inserted += len(batch) + week_results[monday_str] = {"status": "ok", "count": len(batch)} + else: + week_results[monday_str] = {"status": "empty", "count": 0} + + print(f"[FF backfill] {monday_str}: {len(batch)} events", flush=True) + time.sleep(1.5) # respectful delay between requests + + conn.close() + total_weeks = len(mondays) + print(f"[FF backfill] done: {total_inserted} events upserted, {skipped}/{total_weeks} weeks skipped", flush=True) + return { + "from_date": str(from_date), + "to_date": str(to_date), + "weeks_total": total_weeks, + "weeks_scraped": total_weeks - skipped, + "weeks_skipped": skipped, + "total_upserted": total_inserted, + "by_week": week_results, + } diff --git a/backend/services/instrument_models.py b/backend/services/instrument_models.py index 95fb5f9..8ecf8c0 100644 --- a/backend/services/instrument_models.py +++ b/backend/services/instrument_models.py @@ -1679,6 +1679,23 @@ def simulate_timeline( if v0 is not None: macro_last[node_id] = v0 + # Catégories d'events gérées par les nœuds input_event du graphe + # Les catégories sans nœud correspondant sont appliquées comme choc direct sur l'output. + graph_event_cats: set[str] = { + n.get("event_category", "") + for n in graph_def.get("nodes", []) + if n.get("node_type") == "input_event" + } - {""} + + # Baseline linéaire pour la sensibilité temporelle des nœuds macro + # Utilisé uniquement pour calculer le DELTA par rapport au point d'ancrage. + # Le linéaire amplifie la sensibilité aux changements de taux/CPI/NFP qui + # sinon disparaissent dans la zone de saturation tanh. + base_linear_pips = float(evaluate_graph( + _graph_json_for_eval(graph_def, {}), + _build_inputs(graph_def, overrides, {}, saturation=False) + ).get(output_id, 0.0)) if has_macro else 0.0 + timeline = [] cur = date_from while cur <= today: @@ -1706,6 +1723,10 @@ def simulate_timeline( "category": cat, }) + # Split ev_by_cat : catégories avec nœud dédié vs choc direct + routed_ev = {cat: v for cat, v in ev_by_cat.items() if cat in graph_event_cats} + direct_shock = sum(v for cat, v in ev_by_cat.items() if cat not in graph_event_cats) + # ── Overrides pour ce jour (statiques + time-varying macro) ─────────── if has_macro: cur_overrides = dict(overrides) @@ -1718,18 +1739,26 @@ def simulate_timeline( if v is not None: cur_overrides[node_id] = {"value": v, "note": "macro_guidance", "set_at": ""} - # Structural pips time-varying (sans events, avec macro overrides du jour) + # Structural pips : baseline saturé + delta linéaire (sensibilité amplifiée) gj_s = _graph_json_for_eval(graph_def, {}) in_s = _build_inputs(graph_def, cur_overrides, {}, saturation=True) vs_s = evaluate_graph(gj_s, in_s) - structural_pips_t = round(float(vs_s.get(output_id, 0.0)), 1) + sat_pips_t = float(vs_s.get(output_id, 0.0)) + + # Delta linéaire = (linéaire_courant − linéaire_base) → sensibilité temporelle + in_s_lin = _build_inputs(graph_def, cur_overrides, {}, saturation=False) + vs_s_lin = evaluate_graph(gj_s, in_s_lin) + lin_pips_t = float(vs_s_lin.get(output_id, 0.0)) + delta_lin = lin_pips_t - base_linear_pips + + structural_pips_t = round(sat_pips_t + delta_lin, 1) if ev_by_cat: ri = detect_regime(ev_by_cat) gj_ = _graph_json_for_eval(graph_def, ri["weights"]) - in_ = _build_inputs(graph_def, cur_overrides, ev_by_cat, saturation=True) + in_ = _build_inputs(graph_def, cur_overrides, routed_ev, saturation=True) vals = evaluate_graph(gj_, in_) - net = round(float(vals.get(output_id, 0.0)), 1) + net = round(float(vals.get(output_id, 0.0)) + delta_lin + direct_shock, 1) regime_label = ri["regime"] else: vals = vs_s @@ -1741,9 +1770,9 @@ def simulate_timeline( if ev_by_cat: ri = detect_regime(ev_by_cat) gj = _graph_json_for_eval(graph_def, ri["weights"]) - inputs = _build_inputs(graph_def, overrides, ev_by_cat, saturation=True) + inputs = _build_inputs(graph_def, overrides, routed_ev, saturation=True) vals = evaluate_graph(gj, inputs) - net = round(float(vals.get(output_id, 0.0)), 1) + net = round(float(vals.get(output_id, 0.0)) + direct_shock, 1) regime_label = ri["regime"] else: vals = vals_struct diff --git a/frontend/src/pages/InstrumentModels.tsx b/frontend/src/pages/InstrumentModels.tsx index 2ae8e74..f8ca380 100644 --- a/frontend/src/pages/InstrumentModels.tsx +++ b/frontend/src/pages/InstrumentModels.tsx @@ -985,6 +985,7 @@ function TimelineView({ instrument }: { instrument: string }) { const [whatifLoading, setWhatifLoading] = useState(false) const [whatifData, setWhatifData] = useState(null) const [importingCal, setImportingCal] = useState(false) + const [backfilling, setBackfilling] = useState(false) const [calMsg, setCalMsg] = useState(null) // What-if filters const [filterImpact, setFilterImpact] = useState(['high', 'medium']) @@ -1510,6 +1511,31 @@ function TimelineView({ instrument }: { instrument: string }) { className="flex items-center gap-1 text-xs text-sky-400 hover:text-sky-300 border border-sky-700/40 rounded px-2 py-0.5 transition-colors disabled:opacity-40"> {importingCal ? '…' : 'Import calendrier'} +