feat: saxo

backend/services/database.py

@@ -70,9 +70,11 @@ def init_db():
         gamma REAL,
         theta REAL,
         vega REAL,
+        is_synthetic INTEGER DEFAULT 0,
         created_at TEXT DEFAULT (datetime('now'))
     )""")
     c.execute("CREATE INDEX IF NOT EXISTS idx_saxo_snap_symbol_date ON saxo_option_snapshots(symbol, snapshot_date)")
+    c.execute("CREATE INDEX IF NOT EXISTS idx_saxo_snap_contract ON saxo_option_snapshots(symbol, expiry_date, strike, option_type, created_at)")
 
     c.execute("""CREATE TABLE IF NOT EXISTS saxo_instrument_catalog (
         uic INTEGER PRIMARY KEY,
@@ -172,6 +174,7 @@ def init_db():
             added_at TEXT DEFAULT (datetime('now'))
         )""",
         "ALTER TABLE market_watchlist ADD COLUMN asset_class TEXT DEFAULT 'custom'",
+        "ALTER TABLE saxo_option_snapshots ADD COLUMN is_synthetic INTEGER DEFAULT 0",
         # Dashboard — instruments watchlist ("radar"), independent from market_watchlist
         """CREATE TABLE IF NOT EXISTS instruments_watchlist (
             ticker TEXT PRIMARY KEY,
@@ -6089,26 +6092,83 @@ def clear_saxo_tokens():
 
 
 def save_saxo_snapshot_rows(rows: List[Dict[str, Any]]):
+    """Inserts one row per (expiry, strike, type) — but skips contracts whose quote/IV
+    hasn't moved since the last stored snapshot, so an illiquid contract (bid=ask=0,
+    static IV) doesn't accumulate a near-duplicate row every 5 minutes forever. Greeks
+    drift a little from theta decay alone even when the quote is static, so they're
+    intentionally excluded from the comparison."""
     if not rows:
         return
     import uuid
     conn = get_conn()
+
+    to_insert = []
+    for r in rows:
+        last = conn.execute("""
+            SELECT bid, ask, mid, volatility_pct FROM saxo_option_snapshots
+            WHERE symbol=? AND expiry_date=? AND strike=? AND option_type=?
+            ORDER BY created_at DESC LIMIT 1
+        """, (r["symbol"], r["expiry_date"], r["strike"], r["option_type"])).fetchone()
+        if last is not None and (
+            last["bid"] == r.get("bid") and last["ask"] == r.get("ask")
+            and last["mid"] == r.get("mid") and last["volatility_pct"] == r.get("volatility_pct")
+        ):
+            continue
+        to_insert.append(r)
+
+    if not to_insert:
+        conn.close()
+        return
+
     conn.executemany("""INSERT INTO saxo_option_snapshots (
         id, symbol, snapshot_date, spot, expiry_date, strike, option_type,
-        bid, ask, mid, volatility_pct, delta, gamma, theta, vega
-    ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", [
+        bid, ask, mid, volatility_pct, delta, gamma, theta, vega, is_synthetic
+    ) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", [
         (
             f"SNAP-{uuid.uuid4().hex[:12]}",
             r["symbol"], r["snapshot_date"], r.get("spot"), r["expiry_date"], r["strike"], r["option_type"],
             r.get("bid"), r.get("ask"), r.get("mid"), r.get("volatility_pct"),
             r.get("delta"), r.get("gamma"), r.get("theta"), r.get("vega"),
+            1 if r.get("is_synthetic") else 0,
         )
-        for r in rows
+        for r in to_insert
     ])
     conn.commit()
     conn.close()
 
 
+def dedupe_saxo_snapshots() -> int:
+    """One-off cleanup for rows accumulated before the save-time dedup existed: collapses
+    consecutive snapshots per (symbol, expiry, strike, type) that carry the same
+    bid/ask/mid/IV as the row immediately before them, keeping only the first of each run.
+    Safe to call repeatedly — a no-op once the history is already collapsed. Returns the
+    number of rows deleted."""
+    conn = get_conn()
+    rows = conn.execute("""
+        WITH ranked AS (
+            SELECT id, bid, ask, mid, volatility_pct,
+                LAG(bid) OVER w AS prev_bid,
+                LAG(ask) OVER w AS prev_ask,
+                LAG(mid) OVER w AS prev_mid,
+                LAG(volatility_pct) OVER w AS prev_iv
+            FROM saxo_option_snapshots
+            WINDOW w AS (PARTITION BY symbol, expiry_date, strike, option_type ORDER BY created_at)
+        )
+        SELECT id FROM ranked
+        WHERE prev_bid IS NOT NULL
+          AND bid IS prev_bid AND ask IS prev_ask AND mid IS prev_mid AND volatility_pct IS prev_iv
+    """).fetchall()
+    ids = [r["id"] for r in rows]
+    if not ids:
+        conn.close()
+        return 0
+    placeholders = ",".join("?" for _ in ids)
+    conn.execute(f"DELETE FROM saxo_option_snapshots WHERE id IN ({placeholders})", ids)
+    conn.commit()
+    conn.close()
+    return len(ids)
+
+
 def get_saxo_snapshots(symbol: Optional[str] = None, date_from: Optional[str] = None, date_to: Optional[str] = None) -> List[Dict[str, Any]]:
     conn = get_conn()
     query = "SELECT * FROM saxo_option_snapshots WHERE 1=1"