feat: Specialist Desks v2 — COT, Forward Curves, Surprise Index, Hawk/Dove scorer

- COT Positioning: CFTC disaggregated + financial futures (19 markets) via Socrata free API
  net MM position % OI + weekly change stored in cot_data table
- Forward Curves: yfinance front-month vs +3M slope (8 commodities)
  contango/backwardation/flat stored in forward_curve_data table
- Surprise Index: consensus_estimate + actual_value on specialist_reports
  auto-computes surprise_score = actual - consensus on save
- Hawk/Dove Text Scorer: GPT-4o-mini endpoint for CB statements
  score -1..+1, label, summary, key_phrases (forex/bonds: hawk/dove; commodities: bull/bear)
- AI context injection: COT net positioning, forward curve structure,
  surprise scores, upcoming consensus estimates injected into all desk blocks
- Frontend: COT panel (net% bars), Forward Curves panel, SurpriseInput
  on report cards, Hawk/Dove scorer in forex/bonds config tab
- auto_cycle.py: non-blocking COT + curve refresh before each cycle

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

backend/routers/specialist_desks.py

@@ -3,10 +3,13 @@ Specialist Desks — per asset-class fundamental configs + report catalogue.
 Prefix: /api/specialist-desks
 """
 import uuid
+import logging
 from typing import Optional, List
 from fastapi import APIRouter, HTTPException
 from pydantic import BaseModel
 
+logger = logging.getLogger(__name__)
+
 router = APIRouter(prefix="/api/specialist-desks", tags=["specialist-desks"])
 
 
@@ -45,6 +48,20 @@ class ReportUpsert(BaseModel):
     desks: List[str] = []
 
 
+class ReportResultUpdate(BaseModel):
+    actual_value: Optional[float] = None
+    consensus_estimate: Optional[float] = None
+    text_sentiment_score: Optional[float] = None
+    text_sentiment_label: Optional[str] = None
+    text_sentiment_summary: Optional[str] = None
+
+
+class ScoreTextRequest(BaseModel):
+    text: str
+    report_name: str = ""
+    desk: str = "forex"
+
+
 # ── Configs ────────────────────────────────────────────────────────────────────
 
 @router.get("/configs")
@@ -142,3 +159,79 @@ def unlink_from_desk(report_id: str, asset_class: str):
     from services.database import unlink_report_desk
     unlink_report_desk(report_id, asset_class)
     return {"unlinked": report_id, "desk": asset_class}
+
+
+# ── Surprise Index ─────────────────────────────────────────────────────────────
+
+@router.put("/reports/{report_id}/result")
+def update_report_result(report_id: str, body: ReportResultUpdate):
+    from services.database import update_report_result as _upd
+    _upd(
+        report_id=report_id,
+        actual_value=body.actual_value,
+        consensus_estimate=body.consensus_estimate,
+        text_sentiment_score=body.text_sentiment_score,
+        text_sentiment_label=body.text_sentiment_label,
+        text_sentiment_summary=body.text_sentiment_summary,
+    )
+    return {"saved": report_id}
+
+
+# ── COT Data ───────────────────────────────────────────────────────────────────
+
+@router.get("/cot")
+def get_cot_data():
+    from services.database import get_latest_cot_data
+    return get_latest_cot_data()
+
+
+@router.post("/cot/fetch")
+def fetch_cot():
+    from services.cot_fetcher import fetch_all_cot
+    from services.database import save_cot_data
+    try:
+        data = fetch_all_cot()
+        if data:
+            saved = save_cot_data(data)
+            return {"fetched": len(data), "saved": saved}
+        return {"fetched": 0, "saved": 0, "warning": "No COT data returned"}
+    except Exception as e:
+        logger.error(f"COT fetch error: {e}")
+        raise HTTPException(500, str(e))
+
+
+# ── Forward Curves ─────────────────────────────────────────────────────────────
+
+@router.get("/forward-curves")
+def get_forward_curves():
+    from services.database import get_latest_forward_curves
+    return get_latest_forward_curves()
+
+
+@router.post("/forward-curves/fetch")
+def fetch_forward_curves_endpoint():
+    from services.forward_curve import fetch_forward_curves
+    from services.database import save_forward_curves
+    try:
+        data = fetch_forward_curves()
+        if data:
+            saved = save_forward_curves(data)
+            return {"fetched": len(data), "saved": saved}
+        return {"fetched": 0, "saved": 0, "warning": "No curve data returned"}
+    except Exception as e:
+        logger.error(f"Forward curve fetch error: {e}")
+        raise HTTPException(500, str(e))
+
+
+# ── Hawk / Dove Text Scorer ────────────────────────────────────────────────────
+
+@router.post("/score-text")
+def score_text(body: ScoreTextRequest):
+    """Score a central bank / commodity report excerpt via GPT-4o-mini."""
+    try:
+        from services.ai_analyzer import score_report_text
+        result = score_report_text(body.text, body.report_name, body.desk)
+        return result
+    except Exception as e:
+        logger.error(f"Text scoring error: {e}")
+        raise HTTPException(500, str(e))

backend/services/ai_analyzer.py

@@ -377,10 +377,21 @@ SCORE DISTRIBUTION RULE (MANDATORY):
 
 
 def _build_specialist_context_block(asset_classes: set) -> str:
-    """Build a SPECIALIST DESK CONTEXT block for the given asset classes."""
+    """Build SPECIALIST DESK CONTEXT block including COT, forward curves, surprise scores."""
     try:
-        from services.database import get_asset_class_config, get_desk_reports
+        from services.database import get_asset_class_config, get_desk_reports, get_latest_cot_data, get_latest_forward_curves
         from datetime import datetime as _dt
+
+        # Fetch COT and forward curve data once
+        try:
+            all_cot = get_latest_cot_data()
+        except Exception:
+            all_cot = []
+        try:
+            all_curves = get_latest_forward_curves()
+        except Exception:
+            all_curves = []
+
         blocks = []
         for ac in sorted(asset_classes):
             cfg = get_asset_class_config(ac)
@@ -388,16 +399,19 @@ def _build_specialist_context_block(asset_classes: set) -> str:
                 continue
             reports = get_desk_reports(ac)
             lines = [f"\n## SPECIALIST DESK — {cfg['display_name'].upper()} {cfg.get('icon','')}"]
+
             if cfg.get("fundamentals"):
                 lines.append(f"Key drivers: {cfg['fundamentals'][:300]}")
+
             # Macro sensitivity
             sensitivities = cfg.get("macro_sensitivity") or []
             if sensitivities:
                 sens_str = " | ".join(
-                    f"{s.get('regime','?')} → {s.get('effect','?')}"
+                    f"{s.get('regime','?')} -> {s.get('effect','?')}"
                     for s in sensitivities[:4]
                 )
                 lines.append(f"Regime sensitivity: {sens_str}")
+
             # Price thresholds
             thresh = cfg.get("price_delta_thresholds") or {}
             if thresh:
@@ -405,7 +419,30 @@ def _build_specialist_context_block(asset_classes: set) -> str:
                     f"Price thresholds: significant_week={thresh.get('significant_week','?')}% "
                     f"extreme_week={thresh.get('extreme_week','?')}%"
                 )
-            # Upcoming reports
+
+            # COT positioning for this asset class
+            cot_ac = [c for c in all_cot if c.get("asset_class") == ac]
+            if cot_ac:
+                lines.append("COT Positioning (Money Managers net, % of OI):")
+                for c in cot_ac[:5]:
+                    net_pct = c.get("net_pct_oi", 0)
+                    chg = c.get("change_net", 0)
+                    direction = "LONG" if net_pct > 5 else "SHORT" if net_pct < -5 else "NEUTRAL"
+                    chg_str = f" (chg {'+' if chg >= 0 else ''}{chg:,} wk)" if chg else ""
+                    lines.append(f"  - {c['commodity']}: net={net_pct:+.1f}% OI [{direction}]{chg_str} -- {c.get('report_date','?')}")
+
+            # Forward curves for this asset class
+            curves_ac = [c for c in all_curves if c.get("asset_class") == ac]
+            if curves_ac:
+                lines.append("Forward Curve Structure (spot vs +3M):")
+                for curve in curves_ac[:5]:
+                    struct = curve.get("structure", "unknown")
+                    slope = curve.get("slope_pct")
+                    slope_str = f" ({slope:+.1f}%)" if slope is not None else ""
+                    struct_label = struct.upper()
+                    lines.append(f"  - {curve['asset']}: {struct_label}{slope_str}")
+
+            # Recent reports with surprise scores
             today = _dt.utcnow().strftime("%Y-%m-%d")
             upcoming = sorted(
                 [r for r in reports if r.get("next_date") and r["next_date"] >= today],
@@ -414,16 +451,85 @@ def _build_specialist_context_block(asset_classes: set) -> str:
             if upcoming:
                 lines.append("Upcoming reports:")
                 for r in upcoming:
-                    lines.append(
-                        f"  • {r['name']} ({r['source']}) — {r['next_date']} [{r['cadence']}]"
-                        + (f" ⭐×{r['importance']}" if r.get("importance", 1) >= 3 else "")
-                    )
+                    line = f"  - {r['name']} ({r['source']}) -- {r['next_date']} [{r['cadence']}]"
+                    if r.get("importance", 1) >= 3:
+                        line += " ***"
+                    if r.get("consensus_estimate") is not None:
+                        line += f" [consensus: {r['consensus_estimate']}]"
+                    lines.append(line)
+
+            # Recent releases with surprise scores
+            recent_releases = sorted(
+                [r for r in reports if r.get("last_date") and r.get("surprise_score") is not None],
+                key=lambda r: r.get("last_date", ""),
+                reverse=True
+            )[:3]
+            if recent_releases:
+                lines.append("Recent releases (surprise index = actual - consensus):")
+                for r in recent_releases:
+                    surprise = r["surprise_score"]
+                    label = r.get("text_sentiment_label", "")
+                    label_str = f" [{label}]" if label else ""
+                    sign = "+" if surprise >= 0 else ""
+                    lines.append(f"  - {r['name']}: surprise={sign}{surprise:.2f}{label_str} -- {r.get('last_date','?')}")
+
             blocks.append("\n".join(lines))
         return "\n".join(blocks) + "\n" if blocks else ""
     except Exception:
         return ""
 
 
+def score_report_text(text: str, report_name: str = "", desk: str = "forex") -> Dict:
+    """Score a report/statement excerpt. Returns score (-1 to +1), label, summary, key_phrases."""
+    if desk in ("forex", "bonds"):
+        dimension = "monetary policy stance: -1 = very dovish (rate cuts expected), 0 = neutral, +1 = very hawkish (rate hikes expected)"
+        examples = "Hawkish: 'inflation remains persistent', 'further tightening may be appropriate'. Dovish: 'inflation returning to target', 'downside risks dominate', 'easing cycle beginning'."
+    elif desk in ("energy", "metals", "agri"):
+        dimension = "commodity market outlook: -1 = very bearish (oversupply, demand weakness), 0 = neutral, +1 = very bullish (deficit, demand surge)"
+        examples = "Bullish: 'output cuts extended', 'crop failure', 'stockpiles at multi-year low'. Bearish: 'surplus expected', 'demand revision lower', 'record production'."
+    else:
+        dimension = "market sentiment: -1 = very bearish, 0 = neutral, +1 = very bullish"
+        examples = ""
+
+    prompt = f"""You are a macro trading analyst. Analyze the following text and return ONLY valid JSON.
+
+Report: {report_name or 'Unknown'}
+Desk: {desk}
+Score dimension: {dimension}
+{examples}
+
+TEXT TO ANALYZE:
+{text[:3000]}
+
+Return JSON with exactly these fields:
+{{
+  "score": <float from -1.0 to +1.0>,
+  "label": <"very_hawkish" | "hawkish" | "neutral" | "dovish" | "very_dovish"> (for forex/bonds)
+         or <"very_bullish" | "bullish" | "neutral" | "bearish" | "very_bearish"> (for others),
+  "summary": <1-2 sentence summary of the key signal>,
+  "key_phrases": [<up to 5 most significant phrases that drove the score>],
+  "confidence": <"high" | "medium" | "low">
+}}"""
+
+    try:
+        import openai as _oai
+        from services.database import get_config as _gc
+        api_key = _gc("openai_api_key") or ""
+        if not api_key:
+            return {"score": 0, "label": "neutral", "summary": "No API key configured", "key_phrases": [], "confidence": "low"}
+        client = _oai.OpenAI(api_key=api_key)
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.1,
+            response_format={"type": "json_object"},
+        )
+        import json
+        return json.loads(response.choices[0].message.content)
+    except Exception as e:
+        return {"score": 0, "label": "neutral", "summary": f"Error: {str(e)}", "key_phrases": [], "confidence": "low"}
+
+
 def score_patterns_with_context(
     patterns: List[Dict],
     recent_news: List[Dict],

backend/services/auto_cycle.py

@@ -594,6 +594,21 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
             except Exception as _ibe:
                 logger.warning(f"[Cycle] Institutional block failed (non-blocking): {_ibe}")
 
+            # ── COT + Forward Curves (background refresh, non-blocking) ───────────────
+            try:
+                from services.cot_fetcher import fetch_all_cot
+                from services.forward_curve import fetch_forward_curves
+                from services.database import save_cot_data, save_forward_curves as _sfc
+                cot_data = fetch_all_cot()
+                if cot_data:
+                    save_cot_data(cot_data)
+                curve_data = fetch_forward_curves()
+                if curve_data:
+                    _sfc(curve_data)
+            except Exception as _e:
+                import logging as _logging
+                _logging.getLogger(__name__).warning(f"COT/Curve refresh failed (non-blocking): {_e}")
+
             suggestions = suggest_patterns_from_market_context(
                 news, quotes, calendar, macro_regime=macro_regime, geo_score=geo_score_obj,
                 portfolio_lessons=portfolio_lessons,

backend/services/cot_fetcher.py

@@ -1,6 +1,10 @@
 """
 CFTC Commitment of Traders (COT) weekly fetcher.
 Data from CFTC Socrata public API — no API key required.
+
+Two modes:
+  - fetch_cot_report(): legacy institutional_reports format (used by institutional_scheduler)
+  - fetch_all_cot(): flat list of per-commodity/asset COT entries (used by specialist desks / cot_data table)
 """
 import logging
 import math
@@ -192,3 +196,162 @@ def fetch_cot_report() -> Optional[Dict]:
         **signals,
         "ai_summary": ai_summary,
     }
+
+
+# ── Specialist Desks flat COT feed (cot_data table) ──────────────────────────
+
+_DISAGG_URL = "https://publicreporting.cftc.gov/resource/72hh-3qpy.json"   # disaggregated commodities
+_LEGACY_FIN_URL = "https://publicreporting.cftc.gov/resource/gpe5-46if.json"  # financial/forex
+
+_DISAGG_MARKETS = [
+    ("CRUDE OIL, LIGHT SWEET - NYMEX",    "WTI Crude",    "energy"),
+    ("NATURAL GAS - NYMEX",                "Natural Gas",  "energy"),
+    ("GOLD - COMEX",                        "Gold",         "metals"),
+    ("SILVER - COMEX",                      "Silver",       "metals"),
+    ("COPPER- #1 - COMEX",                  "Copper",       "metals"),
+    ("CORN - CBOT",                         "Corn",         "agri"),
+    ("WHEAT - CBOT",                        "Wheat",        "agri"),
+    ("SOYBEANS - CBOT",                     "Soybeans",     "agri"),
+    ("SOYBEAN OIL - CBOT",                  "Soybean Oil",  "agri"),
+    ("COCOA - ICE",                         "Cocoa",        "agri"),
+    ("COFFEE C - ICE",                      "Coffee",       "agri"),
+]
+
+_FIN_MARKETS = [
+    ("EURO FX - CME",                        "Euro (EUR/USD)",  "forex"),
+    ("JAPANESE YEN - CME",                   "Japanese Yen",    "forex"),
+    ("BRITISH POUND STERLING - CME",         "British Pound",   "forex"),
+    ("SWISS FRANC - CME",                    "Swiss Franc",     "forex"),
+    ("U.S. DOLLAR INDEX - ICE FUTURES U.S.", "DXY Index",       "forex"),
+    ("30-DAY FEDERAL FUNDS - CBOT",          "Fed Funds",       "bonds"),
+    ("U.S. TREASURY BONDS - CBOT",           "US T-Bonds",      "bonds"),
+    ("10-YEAR U.S. TREASURY NOTES - CBOT",   "10Y T-Notes",     "bonds"),
+]
+
+
+def _parse_int_flat(v) -> int:
+    try:
+        return int(str(v).replace(",", "").strip())
+    except Exception:
+        return 0
+
+
+def _fetch_disaggregated_flat() -> List[Dict[str, Any]]:
+    """Fetch commodity COT (disaggregated) — MM long/short positions."""
+    results = []
+    market_names = [m[0] for m in _DISAGG_MARKETS]
+    name_map = {m[0]: (m[1], m[2]) for m in _DISAGG_MARKETS}
+
+    quoted = ", ".join(f"'{n}'" for n in market_names)
+    params = {
+        "$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,m_money_positions_long_all,m_money_positions_short_all,open_interest_all,change_in_m_money_long_all,change_in_m_money_short_all",
+        "$where": f"market_and_exchange_names in ({quoted})",
+        "$order": "report_date_as_yyyy_mm_dd DESC",
+        "$limit": str(len(market_names) * 2),
+    }
+    try:
+        resp = requests.get(_DISAGG_URL, params=params, timeout=20)
+        resp.raise_for_status()
+        rows = resp.json()
+    except Exception as e:
+        logger.error(f"COT disaggregated fetch failed: {e}")
+        return []
+
+    seen: set = set()
+    for row in rows:
+        mkt = row.get("market_and_exchange_names", "")
+        if mkt not in name_map or mkt in seen:
+            continue
+        seen.add(mkt)
+        label, ac = name_map[mkt]
+
+        mm_long   = _parse_int_flat(row.get("m_money_positions_long_all", 0))
+        mm_short  = _parse_int_flat(row.get("m_money_positions_short_all", 0))
+        oi        = _parse_int_flat(row.get("open_interest_all", 0))
+        net       = mm_long - mm_short
+        chg_long  = _parse_int_flat(row.get("change_in_m_money_long_all", 0))
+        chg_short = _parse_int_flat(row.get("change_in_m_money_short_all", 0))
+        change_net = chg_long - chg_short
+        net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
+
+        results.append({
+            "market_name": mkt,
+            "commodity": label,
+            "asset_class": ac,
+            "report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10],
+            "mm_long": mm_long,
+            "mm_short": mm_short,
+            "open_interest": oi,
+            "net_position": net,
+            "net_pct_oi": net_pct_oi,
+            "change_net": change_net,
+        })
+
+    return results
+
+
+def _fetch_financial_flat() -> List[Dict[str, Any]]:
+    """Fetch financial/forex COT (legacy) — Non-commercial long/short."""
+    results = []
+    market_names = [m[0] for m in _FIN_MARKETS]
+    name_map = {m[0]: (m[1], m[2]) for m in _FIN_MARKETS}
+
+    quoted = ", ".join(f"'{n}'" for n in market_names)
+    params = {
+        "$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,noncomm_positions_long_all,noncomm_positions_short_all,open_interest_all,change_in_noncomm_long_all,change_in_noncomm_short_all",
+        "$where": f"market_and_exchange_names in ({quoted})",
+        "$order": "report_date_as_yyyy_mm_dd DESC",
+        "$limit": str(len(market_names) * 2),
+    }
+    try:
+        resp = requests.get(_LEGACY_FIN_URL, params=params, timeout=20)
+        resp.raise_for_status()
+        rows = resp.json()
+    except Exception as e:
+        logger.error(f"COT financial fetch failed: {e}")
+        return []
+
+    seen: set = set()
+    for row in rows:
+        mkt = row.get("market_and_exchange_names", "")
+        if mkt not in name_map or mkt in seen:
+            continue
+        seen.add(mkt)
+        label, ac = name_map[mkt]
+
+        nc_long   = _parse_int_flat(row.get("noncomm_positions_long_all", 0))
+        nc_short  = _parse_int_flat(row.get("noncomm_positions_short_all", 0))
+        oi        = _parse_int_flat(row.get("open_interest_all", 0))
+        net       = nc_long - nc_short
+        chg_long  = _parse_int_flat(row.get("change_in_noncomm_long_all", 0))
+        chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0))
+        change_net = chg_long - chg_short
+        net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
+
+        results.append({
+            "market_name": mkt,
+            "commodity": label,
+            "asset_class": ac,
+            "report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10],
+            "mm_long": nc_long,
+            "mm_short": nc_short,
+            "open_interest": oi,
+            "net_position": net,
+            "net_pct_oi": net_pct_oi,
+            "change_net": change_net,
+        })
+
+    return results
+
+
+def fetch_all_cot() -> List[Dict[str, Any]]:
+    """Fetch both disaggregated (commodities) and financial (forex/bonds) COT data.
+
+    Returns a flat list of per-market dicts suitable for save_cot_data().
+    """
+    logger.info("Fetching COT data from CFTC (specialist desks feed)...")
+    all_data: List[Dict[str, Any]] = []
+    all_data.extend(_fetch_disaggregated_flat())
+    all_data.extend(_fetch_financial_flat())
+    logger.info(f"COT: fetched {len(all_data)} markets")
+    return all_data

backend/services/database.py

@@ -108,6 +108,32 @@ def init_db():
         except Exception:
             pass
 
+    # Specialist Reports — surprise index + text sentiment columns
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN consensus_estimate REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN actual_value REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN surprise_score REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_score REAL")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_label TEXT")
+    except Exception:
+        pass
+    try:
+        c.execute("ALTER TABLE specialist_reports ADD COLUMN text_sentiment_summary TEXT")
+    except Exception:
+        pass
+
     # Pattern Lab — historical backtest runs
     c.execute("""CREATE TABLE IF NOT EXISTS backtest_lab_runs (
         id TEXT PRIMARY KEY,
@@ -677,6 +703,36 @@ def init_db():
         PRIMARY KEY (report_id, asset_class)
     )""")
 
+    # ── COT & Forward Curve data ──────────────────────────────────────────────
+    c.execute("""CREATE TABLE IF NOT EXISTS cot_data (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        market_name TEXT NOT NULL,
+        commodity TEXT NOT NULL,
+        asset_class TEXT NOT NULL,
+        report_date TEXT NOT NULL,
+        mm_long INTEGER DEFAULT 0,
+        mm_short INTEGER DEFAULT 0,
+        open_interest INTEGER DEFAULT 0,
+        net_position INTEGER DEFAULT 0,
+        net_pct_oi REAL DEFAULT 0.0,
+        change_net INTEGER DEFAULT 0,
+        fetched_at TEXT NOT NULL DEFAULT (datetime('now')),
+        UNIQUE(commodity, report_date)
+    )""")
+
+    c.execute("""CREATE TABLE IF NOT EXISTS forward_curve_data (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        asset TEXT NOT NULL,
+        asset_class TEXT NOT NULL,
+        front_price REAL,
+        far_price REAL,
+        slope_pct REAL,
+        structure TEXT NOT NULL DEFAULT 'unknown',
+        months_spread INTEGER DEFAULT 3,
+        fetched_at TEXT NOT NULL DEFAULT (datetime('now')),
+        UNIQUE(asset, DATE(fetched_at))
+    )""")
+
     # Seed desk defaults (idempotent)
     _desk_count = c.execute("SELECT COUNT(*) FROM asset_class_configs").fetchone()[0]
     if _desk_count == 0:
@@ -4268,3 +4324,109 @@ def upsert_asset_class_config(
         return True
     finally:
         conn.close()
+
+
+# ── COT Data ──────────────────────────────────────────────────────────────────
+
+def save_cot_data(entries: List[Dict[str, Any]]) -> int:
+    conn = get_conn()
+    try:
+        saved = 0
+        for e in entries:
+            conn.execute("""INSERT INTO cot_data
+                (market_name, commodity, asset_class, report_date, mm_long, mm_short,
+                 open_interest, net_position, net_pct_oi, change_net)
+                VALUES (?,?,?,?,?,?,?,?,?,?)
+                ON CONFLICT(commodity, report_date) DO UPDATE SET
+                    mm_long=excluded.mm_long, mm_short=excluded.mm_short,
+                    open_interest=excluded.open_interest, net_position=excluded.net_position,
+                    net_pct_oi=excluded.net_pct_oi, change_net=excluded.change_net,
+                    fetched_at=datetime('now')""",
+                (e["market_name"], e["commodity"], e["asset_class"], e["report_date"],
+                 e.get("mm_long", 0), e.get("mm_short", 0), e.get("open_interest", 0),
+                 e.get("net_position", 0), e.get("net_pct_oi", 0.0), e.get("change_net", 0)))
+            saved += 1
+        conn.commit()
+        return saved
+    finally:
+        conn.close()
+
+
+def get_latest_cot_data() -> List[Dict[str, Any]]:
+    conn = get_conn()
+    try:
+        rows = conn.execute("""
+            SELECT * FROM cot_data
+            WHERE (commodity, report_date) IN (
+                SELECT commodity, MAX(report_date) FROM cot_data GROUP BY commodity
+            )
+            ORDER BY asset_class, net_pct_oi DESC""").fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+# ── Forward Curve Data ────────────────────────────────────────────────────────
+
+def save_forward_curves(entries: List[Dict[str, Any]]) -> int:
+    conn = get_conn()
+    try:
+        saved = 0
+        for e in entries:
+            conn.execute("""INSERT INTO forward_curve_data
+                (asset, asset_class, front_price, far_price, slope_pct, structure, months_spread)
+                VALUES (?,?,?,?,?,?,?)
+                ON CONFLICT(asset, DATE(fetched_at)) DO UPDATE SET
+                    front_price=excluded.front_price, far_price=excluded.far_price,
+                    slope_pct=excluded.slope_pct, structure=excluded.structure,
+                    fetched_at=datetime('now')""",
+                (e["asset"], e["asset_class"], e.get("front_price"), e.get("far_price"),
+                 e.get("slope_pct"), e.get("structure", "unknown"), e.get("months_spread", 3)))
+            saved += 1
+        conn.commit()
+        return saved
+    finally:
+        conn.close()
+
+
+def get_latest_forward_curves() -> List[Dict[str, Any]]:
+    conn = get_conn()
+    try:
+        rows = conn.execute("""
+            SELECT * FROM forward_curve_data
+            WHERE (asset, DATE(fetched_at)) IN (
+                SELECT asset, MAX(DATE(fetched_at)) FROM forward_curve_data GROUP BY asset
+            )
+            ORDER BY asset_class, asset""").fetchall()
+        return [dict(r) for r in rows]
+    finally:
+        conn.close()
+
+
+# ── Surprise Index ────────────────────────────────────────────────────────────
+
+def update_report_result(
+    report_id: str,
+    actual_value: Optional[float],
+    consensus_estimate: Optional[float],
+    text_sentiment_score: Optional[float] = None,
+    text_sentiment_label: Optional[str] = None,
+    text_sentiment_summary: Optional[str] = None,
+) -> bool:
+    conn = get_conn()
+    try:
+        surprise = None
+        if actual_value is not None and consensus_estimate is not None:
+            surprise = round(actual_value - consensus_estimate, 4)
+        conn.execute("""UPDATE specialist_reports SET
+            actual_value=?, consensus_estimate=?, surprise_score=?,
+            text_sentiment_score=?, text_sentiment_label=?, text_sentiment_summary=?,
+            updated_at=datetime('now')
+            WHERE id=?""",
+            (actual_value, consensus_estimate, surprise,
+             text_sentiment_score, text_sentiment_label, text_sentiment_summary,
+             report_id))
+        conn.commit()
+        return True
+    finally:
+        conn.close()

backend/services/forward_curve.py

@@ -0,0 +1,95 @@
+"""
+Forward Curve Service — contango / backwardation per commodity.
+Uses yfinance to compare front-month vs near-dated futures contracts.
+"""
+import logging
+from datetime import datetime, timedelta
+from typing import List, Dict, Any, Optional
+
+logger = logging.getLogger(__name__)
+
+# Month codes for futures tickers (standard CME/NYMEX/CBOT convention)
+_MONTH_CODES = {1:'F',2:'G',3:'H',4:'J',5:'K',6:'M',7:'N',8:'Q',9:'U',10:'V',11:'X',12:'Z'}
+
+# (base_ticker, label, asset_class, months_spread)
+_CURVE_SPECS = [
+    ("CL",  "WTI Crude",    "energy",  3),
+    ("NG",  "Natural Gas",  "energy",  3),
+    ("GC",  "Gold",         "metals",  3),
+    ("SI",  "Silver",       "metals",  3),
+    ("HG",  "Copper",       "metals",  3),
+    ("ZC",  "Corn",         "agri",    3),
+    ("ZW",  "Wheat",        "agri",    3),
+    ("ZS",  "Soybeans",     "agri",    3),
+]
+
+
+def _ticker_for_month(base: str, months_ahead: int) -> str:
+    target = datetime.now() + timedelta(days=months_ahead * 30)
+    code = _MONTH_CODES[target.month]
+    year = str(target.year)[-2:]
+    return f"{base}{code}{year}"
+
+
+def _get_price(ticker: str) -> Optional[float]:
+    try:
+        import yfinance as yf
+        data = yf.download(ticker, period="3d", progress=False, auto_adjust=True)
+        if data.empty:
+            return None
+        close = data["Close"].dropna()
+        if close.empty:
+            return None
+        val = float(close.iloc[-1])
+        # Handle multi-index DataFrames
+        if hasattr(val, '__iter__'):
+            val = list(val)[0]
+        return round(val, 4) if val and val > 0 else None
+    except Exception:
+        return None
+
+
+def fetch_forward_curves() -> List[Dict[str, Any]]:
+    """Fetch front-month and far-month prices to compute curve slope."""
+    results = []
+    for base, label, ac, spread in _CURVE_SPECS:
+        front_ticker = f"{base}=F"
+        front_price = _get_price(front_ticker)
+        if not front_price:
+            logger.warning(f"Forward curve: no front-month price for {label} ({front_ticker})")
+            continue
+
+        # Try M+spread, M+spread-1, M+spread+1 (some contract months are illiquid)
+        far_price = None
+        for offset in [0, -1, 1, -2, 2]:
+            far_ticker = _ticker_for_month(base, spread + offset)
+            far_price = _get_price(far_ticker)
+            if far_price:
+                break
+
+        slope_pct = None
+        structure = "unknown"
+        if far_price:
+            slope_pct = round((far_price - front_price) / front_price * 100, 2)
+            if slope_pct > 0.15:
+                structure = "contango"
+            elif slope_pct < -0.15:
+                structure = "backwardation"
+            else:
+                structure = "flat"
+
+        results.append({
+            "asset": label,
+            "asset_class": ac,
+            "front_price": round(front_price, 2),
+            "far_price": round(far_price, 2) if far_price else None,
+            "slope_pct": slope_pct,
+            "structure": structure,
+            "months_spread": spread,
+        })
+        logger.info(
+            f"Forward curve {label}: front={front_price:.2f} "
+            f"far={far_price} slope={slope_pct}% -> {structure}"
+        )
+
+    return results