feat: system logs page + dynamic IV watchlist with auto-add from cycle

Backend:
- DB: add system_logs table (level/source/cycle_id/ticker/message) and
  iv_watchlist table (ticker/added_by/is_active); seed builtin 18 tickers
- DBLogHandler attached at startup — all WARNING+ logs auto-persist to DB
- log_system_event() helper for structured manual events
- New router /api/logs: GET with filters (level, source, cycle_id, ticker,
  date range), GET /sources, GET /cycles for dropdowns, DELETE /clear
- iv_watchlist now read from DB instead of hardcoded constant; options_vol
  watchlist/refresh/bootstrap endpoints all use get_watchlist_tickers()
- New endpoints: POST/DELETE /options-vol/watchlist-tickers/{ticker} to
  add/remove tickers; adding triggers background 1-year bootstrap
- auto_cycle: after log_trade_entries(), auto-detect new underlying proxies
  not yet in watchlist, add them and bootstrap their IV history

Frontend:
- New page SystemLogs (/logs): log table with level/source/cycle/ticker/date
  filters, color-coded rows, expandable JSON details, auto-refresh 30s
- Options Lab: WatchlistManager section — add ticker input, chip list with
  builtin/auto/manual color coding, remove button for non-builtins
- Sidebar: Logs Système nav link (ScrollText icon)
- useApi: useSystemLogs, useLogSources, useLogCycles, useClearLogs,
  useWatchlistTickers, useAddWatchlistTicker, useRemoveWatchlistTicker

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/main.py

@@ -1,10 +1,30 @@
 from fastapi import FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from routers import market_data, geopolitical, options, backtest, ai, portfolio, config, patterns, journal, cycle as cycle_router, profiles as profiles_router, reasoning as reasoning_router, knowledge as knowledge_router, options_vol as options_vol_router, analytics as analytics_router, risk as risk_router
+from routers import logs as logs_router
 from services.database import init_db, get_config, cleanup_stale_running_cycles
 import os
+import logging
 import uvicorn
 
+
+class _DBLogHandler(logging.Handler):
+    """Persist WARNING+ log records to system_logs table."""
+    def emit(self, record: logging.LogRecord):
+        if record.levelno < logging.WARNING:
+            return
+        try:
+            from services.database import log_system_event
+            log_system_event(
+                level=record.levelname,
+                source=record.name,
+                message=self.format(record),
+                cycle_id=getattr(record, "cycle_id", None),
+                ticker=getattr(record, "ticker", None),
+            )
+        except Exception:
+            pass  # Never raise from inside a log handler
+
 app = FastAPI(
     title="GeoOptions Intelligence",
     description="Geopolitical Options Trading Cockpit API",
@@ -22,7 +42,11 @@ app.add_middleware(
 
 @app.on_event("startup")
 def startup():
-    import logging
+    # Attach DB log handler to root logger (captures WARNING+ from all modules)
+    _db_handler = _DBLogHandler()
+    _db_handler.setFormatter(logging.Formatter("%(message)s"))
+    logging.getLogger().addHandler(_db_handler)
+
     _log = logging.getLogger(__name__)
 
     init_db()
@@ -74,6 +98,7 @@ app.include_router(knowledge_router.router)
 app.include_router(options_vol_router.router)
 app.include_router(analytics_router.router)
 app.include_router(risk_router.router)
+app.include_router(logs_router.router)
 
 
 @app.get("/")

backend/routers/logs.py

@@ -0,0 +1,75 @@
+"""
+System logs endpoint — exposes structured log events stored in system_logs table.
+"""
+from fastapi import APIRouter
+from typing import Optional
+
+router = APIRouter(prefix="/api/logs", tags=["logs"])
+
+
+@router.get("")
+def get_logs(
+    level: Optional[str] = None,
+    source: Optional[str] = None,
+    cycle_id: Optional[str] = None,
+    ticker: Optional[str] = None,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 300,
+):
+    """
+    Query system logs with optional filters.
+    - level: INFO | WARNING | ERROR | CRITICAL
+    - source: logger name substring (e.g. 'auto_cycle', 'iv_engine')
+    - cycle_id: exact run_id UUID
+    - ticker: exact ticker symbol
+    - date_from / date_to: ISO date strings YYYY-MM-DD
+    - limit: max rows returned (default 300, max 1000)
+    """
+    from services.database import get_system_logs
+    limit = min(limit, 1000)
+    logs = get_system_logs(
+        level=level,
+        source=source,
+        cycle_id=cycle_id,
+        ticker=ticker,
+        date_from=date_from,
+        date_to=date_to,
+        limit=limit,
+    )
+    return {"logs": logs, "count": len(logs)}
+
+
+@router.get("/sources")
+def get_log_sources():
+    """List distinct source names present in system_logs (for filter dropdown)."""
+    from services.database import get_conn
+    conn = get_conn()
+    rows = conn.execute(
+        "SELECT DISTINCT source FROM system_logs WHERE source IS NOT NULL ORDER BY source"
+    ).fetchall()
+    conn.close()
+    return {"sources": [r["source"] for r in rows]}
+
+
+@router.get("/cycles")
+def get_log_cycles(limit: int = 50):
+    """List distinct cycle_ids (most recent first) for filter dropdown."""
+    from services.database import get_conn
+    conn = get_conn()
+    rows = conn.execute(
+        """SELECT DISTINCT cycle_id, MIN(ts) as first_ts
+           FROM system_logs WHERE cycle_id IS NOT NULL
+           GROUP BY cycle_id ORDER BY first_ts DESC LIMIT ?""",
+        (limit,)
+    ).fetchall()
+    conn.close()
+    return {"cycles": [{"cycle_id": r["cycle_id"], "first_ts": r["first_ts"]} for r in rows]}
+
+
+@router.delete("/clear")
+def clear_old_logs(older_than_days: int = 30):
+    """Delete logs older than N days."""
+    from services.database import clear_system_logs
+    deleted = clear_system_logs(older_than_days)
+    return {"deleted": deleted, "older_than_days": older_than_days}

backend/routers/options_vol.py

@@ -68,14 +68,14 @@ def get_iv_watchlist():
     IV for the core IV watchlist (portfolio-relevant tickers).
     Returns a summary list sorted by IV Rank descending.
     """
-    from services.iv_engine import IV_WATCHLIST, get_atm_iv, _resolve_ticker
-    from services.database import get_iv_rank_percentile, save_iv_snapshot, get_iv_history
+    from services.iv_engine import get_atm_iv, _resolve_ticker
+    from services.database import get_iv_rank_percentile, save_iv_snapshot, get_iv_history, get_watchlist_tickers
     from datetime import date
 
     results = []
     today = date.today().isoformat()
 
-    for ticker in IV_WATCHLIST:
+    for ticker in get_watchlist_tickers():
         key = f"watchlist:{ticker}"
         if not _is_stale(key):
             results.append(_iv_cache[key]["data"])
@@ -128,16 +128,48 @@ def get_iv_history_endpoint(ticker: str, days: int = 90):
     return {"ticker": ticker, "proxy": proxy, "history": history, "count": len(history)}
 
 
+@router.get("/watchlist-tickers")
+def list_watchlist_tickers():
+    """List all watchlist tickers (active + inactive) with metadata."""
+    from services.database import get_watchlist_entries
+    return {"tickers": get_watchlist_entries()}
+
+
+@router.post("/watchlist-tickers/{ticker}")
+def add_to_watchlist(ticker: str, background_tasks: BackgroundTasks):
+    """Add a ticker to the IV watchlist and start background bootstrap."""
+    from services.database import add_watchlist_ticker
+    from services.iv_engine import _resolve_ticker
+    proxy = _resolve_ticker(ticker.upper())
+    is_new = add_watchlist_ticker(proxy, added_by="manual")
+    if is_new:
+        def _bootstrap():
+            from services.iv_engine import bootstrap_iv_history
+            bootstrap_iv_history(tickers=[proxy], min_existing=0)
+            logger.info(f"[Watchlist] Bootstrap done for new ticker {proxy}")
+        background_tasks.add_task(_bootstrap)
+    return {"ticker": proxy, "is_new": is_new, "bootstrap": is_new}
+
+
+@router.delete("/watchlist-tickers/{ticker}")
+def remove_from_watchlist(ticker: str):
+    """Remove (soft-delete) a ticker from the watchlist."""
+    from services.database import remove_watchlist_ticker
+    from services.iv_engine import _resolve_ticker
+    proxy = _resolve_ticker(ticker.upper())
+    ok = remove_watchlist_ticker(proxy)
+    return {"ticker": proxy, "removed": ok}
+
+
 @router.post("/refresh-watchlist")
 def refresh_watchlist(background_tasks: BackgroundTasks):
-    """
-    Trigger a background refresh of IV data for all watchlist tickers.
-    Call this once daily to build up the 52-week IV history.
-    """
+    """Trigger a background refresh of IV data for all watchlist tickers."""
     def _refresh():
-        from services.iv_engine import IV_WATCHLIST, get_full_iv_snapshot
-        logger.info(f"[IVRefresh] Refreshing IV for {len(IV_WATCHLIST)} tickers")
-        for ticker in IV_WATCHLIST:
+        from services.iv_engine import get_full_iv_snapshot
+        from services.database import get_watchlist_tickers
+        tickers = get_watchlist_tickers()
+        logger.info(f"[IVRefresh] Refreshing IV for {len(tickers)} tickers")
+        for ticker in tickers:
             try:
                 snap = get_full_iv_snapshot(ticker)
                 key = f"snapshot:{ticker}"
@@ -148,21 +180,21 @@ def refresh_watchlist(background_tasks: BackgroundTasks):
                 logger.warning(f"[IVRefresh] {ticker} failed: {e}")
         logger.info("[IVRefresh] Done")
 
+    from services.database import get_watchlist_tickers
+    count = len(get_watchlist_tickers())
     background_tasks.add_task(_refresh)
-    return {"status": "refresh started", "tickers": len(__import__("services.iv_engine", fromlist=["IV_WATCHLIST"]).IV_WATCHLIST)}
+    return {"status": "refresh started", "tickers": count}
 
 
 @router.post("/bootstrap-history")
 def bootstrap_iv_history_endpoint(background_tasks: BackgroundTasks, min_existing: int = 30):
-    """
-    Bootstrap iv_history with 1 year of realized volatility for all watchlist tickers.
-    Only fills missing dates — safe to call multiple times.
-    Runs in background (takes ~30-60s for 18 tickers).
-    """
+    """Bootstrap iv_history with 1 year of realized vol for all watchlist tickers."""
     def _run():
-        from services.iv_engine import bootstrap_iv_history, IV_WATCHLIST
-        logger.info(f"[IV Bootstrap] Starting for {len(IV_WATCHLIST)} tickers")
-        result = bootstrap_iv_history(tickers=IV_WATCHLIST, min_existing=min_existing)
+        from services.iv_engine import bootstrap_iv_history
+        from services.database import get_watchlist_tickers
+        tickers = get_watchlist_tickers()
+        logger.info(f"[IV Bootstrap] Starting for {len(tickers)} tickers")
+        result = bootstrap_iv_history(tickers=tickers, min_existing=min_existing)
         total = sum(v.get("inserted", 0) for v in result.values() if isinstance(v, dict))
         logger.info(f"[IV Bootstrap] Done — {total} rows inserted across {len(result)} tickers")
 

backend/services/auto_cycle.py

@@ -116,7 +116,7 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
             get_config, get_custom_patterns, save_custom_pattern,
             save_pattern_scores, log_macro_regime, log_geo_alert, log_trade_entries,
             add_cycle_run, update_cycle_run, save_reasoning_trace,
-            get_latest_portfolio_lessons,
+            get_latest_portfolio_lessons, log_system_event,
         )
         from services.data_fetcher import fetch_geo_news, get_all_quotes, get_macro_gauges, score_macro_scenarios
         from services.geo_analyzer import compute_geo_risk_score
@@ -466,6 +466,28 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
 
         log_trade_entries(run_id=scoring_run_id, scored_patterns=scored, quotes=quotes)
 
+        # Auto-add any new underlying tickers to the IV watchlist
+        try:
+            from services.database import add_watchlist_ticker, get_watchlist_tickers
+            from services.iv_engine import _resolve_ticker, bootstrap_iv_history
+            existing = set(get_watchlist_tickers())
+            new_proxies = set()
+            for sp in scored:
+                for trade in (sp.get("trade_rankings") or sp.get("suggested_trades") or []):
+                    underlying = trade.get("underlying") or sp.get("underlying") or ""
+                    if underlying:
+                        proxy = _resolve_ticker(underlying)
+                        if proxy not in existing:
+                            new_proxies.add(proxy)
+            for proxy in new_proxies:
+                if add_watchlist_ticker(proxy, added_by="cycle"):
+                    _log.info(f"[Watchlist] Auto-added new ticker: {proxy}")
+                    log_system_event("INFO", "auto_cycle", f"Nouveau ticker ajouté à la watchlist IV: {proxy}", cycle_id=scoring_run_id, ticker=proxy)
+            if new_proxies:
+                bootstrap_iv_history(tickers=list(new_proxies), min_existing=0)
+        except Exception as _we:
+            _log.warning(f"[Watchlist] Auto-add failed: {_we}")
+
         gauges_summary = {
             k: {"value": v.get("value"), "change_pct": v.get("change_pct"), "label": v.get("label")}
             for k, v in gauges.items()

backend/services/database.py

@@ -342,10 +342,31 @@ def init_db():
         created_at TEXT DEFAULT (datetime('now'))
     )""")
 
+    c.execute("""CREATE TABLE IF NOT EXISTS iv_watchlist (
+        ticker TEXT PRIMARY KEY,
+        added_date TEXT NOT NULL DEFAULT (date('now')),
+        added_by TEXT DEFAULT 'builtin',
+        is_active INTEGER DEFAULT 1
+    )""")
+
+    c.execute("""CREATE TABLE IF NOT EXISTS system_logs (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        ts TEXT NOT NULL DEFAULT (datetime('now')),
+        level TEXT NOT NULL,
+        source TEXT,
+        cycle_id TEXT,
+        ticker TEXT,
+        message TEXT NOT NULL,
+        details TEXT
+    )""")
+
     try:
         c.execute("CREATE INDEX IF NOT EXISTS idx_kb_category ON knowledge_base(category, status)")
         c.execute("CREATE INDEX IF NOT EXISTS idx_rs_version ON reasoning_state(version DESC)")
         c.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_iv_history_ticker_date ON iv_history(ticker, recorded_date)")
+        c.execute("CREATE INDEX IF NOT EXISTS idx_syslog_ts ON system_logs(ts DESC)")
+        c.execute("CREATE INDEX IF NOT EXISTS idx_syslog_level ON system_logs(level, ts DESC)")
+        c.execute("CREATE INDEX IF NOT EXISTS idx_syslog_cycle ON system_logs(cycle_id, ts DESC)")
     except Exception:
         pass
 
@@ -361,6 +382,18 @@ def init_db():
                 (_key, _val)
             )
 
+    # Seed built-in watchlist tickers (idempotent)
+    _builtin_tickers = [
+        "SPY", "QQQ", "GLD", "SLV", "USO", "BNO", "UNG",
+        "XLE", "UUP", "TLT", "GDX", "EWJ", "FEZ",
+        "XOM", "CVX", "LMT", "RTX", "BA",
+    ]
+    for _t in _builtin_tickers:
+        c.execute(
+            "INSERT OR IGNORE INTO iv_watchlist (ticker, added_by) VALUES (?, 'builtin')",
+            (_t,)
+        )
+
     conn.commit()
     conn.close()
 
@@ -1637,6 +1670,134 @@ def get_iv_history(ticker: str, days: int = 90) -> List[Dict]:
     return [dict(r) for r in rows]
 
 
+# ── IV Watchlist ──────────────────────────────────────────────────────────────
+
+def get_watchlist_tickers() -> List[str]:
+    conn = get_conn()
+    rows = conn.execute(
+        "SELECT ticker FROM iv_watchlist WHERE is_active=1 ORDER BY added_by='builtin' DESC, added_date ASC"
+    ).fetchall()
+    conn.close()
+    return [r["ticker"] for r in rows]
+
+
+def get_watchlist_entries() -> List[Dict]:
+    conn = get_conn()
+    rows = conn.execute(
+        "SELECT ticker, added_date, added_by, is_active FROM iv_watchlist ORDER BY added_by, ticker"
+    ).fetchall()
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def add_watchlist_ticker(ticker: str, added_by: str = "manual") -> bool:
+    """Add ticker to watchlist. Returns True if newly inserted, False if already existed."""
+    conn = get_conn()
+    existing = conn.execute("SELECT ticker, is_active FROM iv_watchlist WHERE ticker=?", (ticker.upper(),)).fetchone()
+    if existing:
+        if not existing["is_active"]:
+            conn.execute("UPDATE iv_watchlist SET is_active=1, added_by=? WHERE ticker=?", (added_by, ticker.upper()))
+            conn.commit()
+            conn.close()
+            return True
+        conn.close()
+        return False
+    conn.execute(
+        "INSERT INTO iv_watchlist (ticker, added_date, added_by) VALUES (?, date('now'), ?)",
+        (ticker.upper(), added_by)
+    )
+    conn.commit()
+    conn.close()
+    return True
+
+
+def remove_watchlist_ticker(ticker: str) -> bool:
+    conn = get_conn()
+    conn.execute("UPDATE iv_watchlist SET is_active=0 WHERE ticker=?", (ticker.upper(),))
+    changed = conn.total_changes > 0
+    conn.commit()
+    conn.close()
+    return changed
+
+
+# ── System Logs ───────────────────────────────────────────────────────────────
+
+def log_system_event(
+    level: str,
+    source: str,
+    message: str,
+    cycle_id: Optional[str] = None,
+    ticker: Optional[str] = None,
+    details: Optional[Dict] = None,
+) -> None:
+    try:
+        conn = get_conn()
+        conn.execute(
+            """INSERT INTO system_logs (level, source, cycle_id, ticker, message, details)
+               VALUES (?, ?, ?, ?, ?, ?)""",
+            (
+                level.upper(),
+                source,
+                cycle_id,
+                ticker.upper() if ticker else None,
+                message,
+                json.dumps(details, default=str) if details else None,
+            ),
+        )
+        conn.commit()
+        conn.close()
+    except Exception:
+        pass  # Never raise from logging
+
+
+def get_system_logs(
+    level: Optional[str] = None,
+    source: Optional[str] = None,
+    cycle_id: Optional[str] = None,
+    ticker: Optional[str] = None,
+    date_from: Optional[str] = None,
+    date_to: Optional[str] = None,
+    limit: int = 300,
+) -> List[Dict]:
+    conn = get_conn()
+    clauses = []
+    params: List[Any] = []
+    if level:
+        clauses.append("level = ?")
+        params.append(level.upper())
+    if source:
+        clauses.append("source LIKE ?")
+        params.append(f"%{source}%")
+    if cycle_id:
+        clauses.append("cycle_id = ?")
+        params.append(cycle_id)
+    if ticker:
+        clauses.append("ticker = ?")
+        params.append(ticker.upper())
+    if date_from:
+        clauses.append("ts >= ?")
+        params.append(date_from)
+    if date_to:
+        clauses.append("ts <= ?")
+        params.append(date_to + "T23:59:59")
+    where = ("WHERE " + " AND ".join(clauses)) if clauses else ""
+    rows = conn.execute(
+        f"SELECT id, ts, level, source, cycle_id, ticker, message, details FROM system_logs {where} ORDER BY ts DESC LIMIT ?",
+        params + [limit],
+    ).fetchall()
+    conn.close()
+    return [dict(r) for r in rows]
+
+
+def clear_system_logs(older_than_days: int = 30) -> int:
+    conn = get_conn()
+    conn.execute(f"DELETE FROM system_logs WHERE ts < datetime('now', '-{older_than_days} days')")
+    deleted = conn.total_changes
+    conn.commit()
+    conn.close()
+    return deleted
+
+
 # ── Knowledge Base Decay ──────────────────────────────────────────────────────
 
 def decay_kb_confidence() -> int: