feat: Specialist Desks v2 — COT, Forward Curves, Surprise Index, Hawk/Dove scorer

- COT Positioning: CFTC disaggregated + financial futures (19 markets) via Socrata free API net MM position % OI + weekly change stored in cot_data table - Forward Curves: yfinance front-month vs +3M slope (8 commodities) contango/backwardation/flat stored in forward_curve_data table - Surprise Index: consensus_estimate + actual_value on specialist_reports auto-computes surprise_score = actual - consensus on save - Hawk/Dove Text Scorer: GPT-4o-mini endpoint for CB statements score -1..+1, label, summary, key_phrases (forex/bonds: hawk/dove; commodities: bull/bear) - AI context injection: COT net positioning, forward curve structure, surprise scores, upcoming consensus estimates injected into all desk blocks - Frontend: COT panel (net% bars), Forward Curves panel, SurpriseInput on report cards, Hawk/Dove scorer in forex/bonds config tab - auto_cycle.py: non-blocking COT + curve refresh before each cycle Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-06-23 18:00:46 +02:00
parent 70a9e2b569
commit 3b7fa35456
9 changed files with 1965 additions and 40 deletions
--- a/backend/services/database.py
+++ b/backend/services/database.py
@@ -108,6 +108,32 @@ def init_db():
        except Exception:
            pass

+    # Specialist Reports — surprise index + text sentiment columns
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN consensus_estimate REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN actual_value REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN surprise_score REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_score REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_label TEXT")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_summary TEXT")
+    except Exception:
+        pass
+
    # Pattern Lab — historical backtest runs
    c.execute("""CREATE TABLE IF NOT EXISTS backtest_lab_runs (
        id TEXT PRIMARY KEY,
@@ -677,6 +703,36 @@ def init_db():
        PRIMARY KEY (report_id, asset_class)
    )""")

+    # ── COT & Forward Curve data ──────────────────────────────────────────────
+    c.execute("""CREATE TABLE IF NOT EXISTS cot_data (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        market_name TEXT NOT NULL,
+        commodity TEXT NOT NULL,
+        asset_class TEXT NOT NULL,
+        report_date TEXT NOT NULL,
+        mm_long INTEGER DEFAULT 0,
+        mm_short INTEGER DEFAULT 0,
+        open_interest INTEGER DEFAULT 0,
+        net_position INTEGER DEFAULT 0,
+        net_pct_oi REAL DEFAULT 0.0,
+        change_net INTEGER DEFAULT 0,
+        fetched_at TEXT NOT NULL DEFAULT (datetime('now')),
+        UNIQUE(commodity, report_date)
+    )""")
+
+    c.execute("""CREATE TABLE IF NOT EXISTS forward_curve_data (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        asset TEXT NOT NULL,
+        asset_class TEXT NOT NULL,
+        front_price REAL,
+        far_price REAL,
+        slope_pct REAL,
+        structure TEXT NOT NULL DEFAULT 'unknown',
+        months_spread INTEGER DEFAULT 3,
+        fetched_at TEXT NOT NULL DEFAULT (datetime('now')),
+        UNIQUE(asset, DATE(fetched_at))
+    )""")
+
    # Seed desk defaults (idempotent)
    _desk_count = c.execute("SELECT COUNT(*) FROM asset_class_configs").fetchone()[0]
    if _desk_count == 0:
@@ -4268,3 +4324,109 @@ def upsert_asset_class_config(
        return True
    finally:
        conn.close()
+
+
+# ── COT Data ──────────────────────────────────────────────────────────────────
+
+def save_cot_data(entries: List[Dict[str, Any]]) -> int:
+    conn = get_conn()
+    try:
+        saved = 0
+        for e in entries:
+            conn.execute("""INSERT INTO cot_data
+                (market_name, commodity, asset_class, report_date, mm_long, mm_short,
+                 open_interest, net_position, net_pct_oi, change_net)
+                VALUES (?,?,?,?,?,?,?,?,?,?)
+                ON CONFLICT(commodity, report_date) DO UPDATE SET
+                    mm_long=excluded.mm_long, mm_short=excluded.mm_short,
+                    open_interest=excluded.open_interest, net_position=excluded.net_position,
+                    net_pct_oi=excluded.net_pct_oi, change_net=excluded.change_net,
+                    fetched_at=datetime('now')""",
+                (e["market_name"], e["commodity"], e["asset_class"], e["report_date"],
+                 e.get("mm_long", 0), e.get("mm_short", 0), e.get("open_interest", 0),
+                 e.get("net_position", 0), e.get("net_pct_oi", 0.0), e.get("change_net", 0)))
+            saved += 1
+        conn.commit()
+        return saved
+    finally:
+        conn.close()
+
+
+def get_latest_cot_data() -> List[Dict[str, Any]]:
+    conn = get_conn()
+    try:
+        rows = conn.execute("""
+            SELECT * FROM cot_data
+            WHERE (commodity, report_date) IN (
+                SELECT commodity, MAX(report_date) FROM cot_data GROUP BY commodity
+            )
+            ORDER BY asset_class, net_pct_oi DESC""").fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+# ── Forward Curve Data ────────────────────────────────────────────────────────
+
+def save_forward_curves(entries: List[Dict[str, Any]]) -> int:
+    conn = get_conn()
+    try:
+        saved = 0
+        for e in entries:
+            conn.execute("""INSERT INTO forward_curve_data
+                (asset, asset_class, front_price, far_price, slope_pct, structure, months_spread)
+                VALUES (?,?,?,?,?,?,?)
+                ON CONFLICT(asset, DATE(fetched_at)) DO UPDATE SET
+                    front_price=excluded.front_price, far_price=excluded.far_price,
+                    slope_pct=excluded.slope_pct, structure=excluded.structure,
+                    fetched_at=datetime('now')""",
+                (e["asset"], e["asset_class"], e.get("front_price"), e.get("far_price"),
+                 e.get("slope_pct"), e.get("structure", "unknown"), e.get("months_spread", 3)))
+            saved += 1
+        conn.commit()
+        return saved
+    finally:
+        conn.close()
+
+
+def get_latest_forward_curves() -> List[Dict[str, Any]]:
+    conn = get_conn()
+    try:
+        rows = conn.execute("""
+            SELECT * FROM forward_curve_data
+            WHERE (asset, DATE(fetched_at)) IN (
+                SELECT asset, MAX(DATE(fetched_at)) FROM forward_curve_data GROUP BY asset
+            )
+            ORDER BY asset_class, asset""").fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+# ── Surprise Index ────────────────────────────────────────────────────────────
+
+def update_report_result(
+    report_id: str,
+    actual_value: Optional[float],
+    consensus_estimate: Optional[float],
+    text_sentiment_score: Optional[float] = None,
+    text_sentiment_label: Optional[str] = None,
+    text_sentiment_summary: Optional[str] = None,
+) -> bool:
+    conn = get_conn()
+    try:
+        surprise = None
+        if actual_value is not None and consensus_estimate is not None:
+            surprise = round(actual_value - consensus_estimate, 4)
+        conn.execute("""UPDATE specialist_reports SET
+            actual_value=?, consensus_estimate=?, surprise_score=?,
+            text_sentiment_score=?, text_sentiment_label=?, text_sentiment_summary=?,
+            updated_at=datetime('now')
+            WHERE id=?""",
+            (actual_value, consensus_estimate, surprise,
+             text_sentiment_score, text_sentiment_label, text_sentiment_summary,
+             report_id))
+        conn.commit()
+        return True
+    finally:
+        conn.close()