feat: Specialist Desks v2 — COT, Forward Curves, Surprise Index, Hawk/Dove scorer
- COT Positioning: CFTC disaggregated + financial futures (19 markets) via Socrata free API net MM position % OI + weekly change stored in cot_data table - Forward Curves: yfinance front-month vs +3M slope (8 commodities) contango/backwardation/flat stored in forward_curve_data table - Surprise Index: consensus_estimate + actual_value on specialist_reports auto-computes surprise_score = actual - consensus on save - Hawk/Dove Text Scorer: GPT-4o-mini endpoint for CB statements score -1..+1, label, summary, key_phrases (forex/bonds: hawk/dove; commodities: bull/bear) - AI context injection: COT net positioning, forward curve structure, surprise scores, upcoming consensus estimates injected into all desk blocks - Frontend: COT panel (net% bars), Forward Curves panel, SurpriseInput on report cards, Hawk/Dove scorer in forex/bonds config tab - auto_cycle.py: non-blocking COT + curve refresh before each cycle Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
@@ -1,6 +1,10 @@
|
||||
"""
|
||||
CFTC Commitment of Traders (COT) weekly fetcher.
|
||||
Data from CFTC Socrata public API — no API key required.
|
||||
|
||||
Two modes:
|
||||
- fetch_cot_report(): legacy institutional_reports format (used by institutional_scheduler)
|
||||
- fetch_all_cot(): flat list of per-commodity/asset COT entries (used by specialist desks / cot_data table)
|
||||
"""
|
||||
import logging
|
||||
import math
|
||||
@@ -192,3 +196,162 @@ def fetch_cot_report() -> Optional[Dict]:
|
||||
**signals,
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
|
||||
|
||||
# ── Specialist Desks flat COT feed (cot_data table) ──────────────────────────
|
||||
|
||||
_DISAGG_URL = "https://publicreporting.cftc.gov/resource/72hh-3qpy.json" # disaggregated commodities
|
||||
_LEGACY_FIN_URL = "https://publicreporting.cftc.gov/resource/gpe5-46if.json" # financial/forex
|
||||
|
||||
_DISAGG_MARKETS = [
|
||||
("CRUDE OIL, LIGHT SWEET - NYMEX", "WTI Crude", "energy"),
|
||||
("NATURAL GAS - NYMEX", "Natural Gas", "energy"),
|
||||
("GOLD - COMEX", "Gold", "metals"),
|
||||
("SILVER - COMEX", "Silver", "metals"),
|
||||
("COPPER- #1 - COMEX", "Copper", "metals"),
|
||||
("CORN - CBOT", "Corn", "agri"),
|
||||
("WHEAT - CBOT", "Wheat", "agri"),
|
||||
("SOYBEANS - CBOT", "Soybeans", "agri"),
|
||||
("SOYBEAN OIL - CBOT", "Soybean Oil", "agri"),
|
||||
("COCOA - ICE", "Cocoa", "agri"),
|
||||
("COFFEE C - ICE", "Coffee", "agri"),
|
||||
]
|
||||
|
||||
_FIN_MARKETS = [
|
||||
("EURO FX - CME", "Euro (EUR/USD)", "forex"),
|
||||
("JAPANESE YEN - CME", "Japanese Yen", "forex"),
|
||||
("BRITISH POUND STERLING - CME", "British Pound", "forex"),
|
||||
("SWISS FRANC - CME", "Swiss Franc", "forex"),
|
||||
("U.S. DOLLAR INDEX - ICE FUTURES U.S.", "DXY Index", "forex"),
|
||||
("30-DAY FEDERAL FUNDS - CBOT", "Fed Funds", "bonds"),
|
||||
("U.S. TREASURY BONDS - CBOT", "US T-Bonds", "bonds"),
|
||||
("10-YEAR U.S. TREASURY NOTES - CBOT", "10Y T-Notes", "bonds"),
|
||||
]
|
||||
|
||||
|
||||
def _parse_int_flat(v) -> int:
|
||||
try:
|
||||
return int(str(v).replace(",", "").strip())
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
|
||||
def _fetch_disaggregated_flat() -> List[Dict[str, Any]]:
|
||||
"""Fetch commodity COT (disaggregated) — MM long/short positions."""
|
||||
results = []
|
||||
market_names = [m[0] for m in _DISAGG_MARKETS]
|
||||
name_map = {m[0]: (m[1], m[2]) for m in _DISAGG_MARKETS}
|
||||
|
||||
quoted = ", ".join(f"'{n}'" for n in market_names)
|
||||
params = {
|
||||
"$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,m_money_positions_long_all,m_money_positions_short_all,open_interest_all,change_in_m_money_long_all,change_in_m_money_short_all",
|
||||
"$where": f"market_and_exchange_names in ({quoted})",
|
||||
"$order": "report_date_as_yyyy_mm_dd DESC",
|
||||
"$limit": str(len(market_names) * 2),
|
||||
}
|
||||
try:
|
||||
resp = requests.get(_DISAGG_URL, params=params, timeout=20)
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
except Exception as e:
|
||||
logger.error(f"COT disaggregated fetch failed: {e}")
|
||||
return []
|
||||
|
||||
seen: set = set()
|
||||
for row in rows:
|
||||
mkt = row.get("market_and_exchange_names", "")
|
||||
if mkt not in name_map or mkt in seen:
|
||||
continue
|
||||
seen.add(mkt)
|
||||
label, ac = name_map[mkt]
|
||||
|
||||
mm_long = _parse_int_flat(row.get("m_money_positions_long_all", 0))
|
||||
mm_short = _parse_int_flat(row.get("m_money_positions_short_all", 0))
|
||||
oi = _parse_int_flat(row.get("open_interest_all", 0))
|
||||
net = mm_long - mm_short
|
||||
chg_long = _parse_int_flat(row.get("change_in_m_money_long_all", 0))
|
||||
chg_short = _parse_int_flat(row.get("change_in_m_money_short_all", 0))
|
||||
change_net = chg_long - chg_short
|
||||
net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
|
||||
|
||||
results.append({
|
||||
"market_name": mkt,
|
||||
"commodity": label,
|
||||
"asset_class": ac,
|
||||
"report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10],
|
||||
"mm_long": mm_long,
|
||||
"mm_short": mm_short,
|
||||
"open_interest": oi,
|
||||
"net_position": net,
|
||||
"net_pct_oi": net_pct_oi,
|
||||
"change_net": change_net,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_financial_flat() -> List[Dict[str, Any]]:
|
||||
"""Fetch financial/forex COT (legacy) — Non-commercial long/short."""
|
||||
results = []
|
||||
market_names = [m[0] for m in _FIN_MARKETS]
|
||||
name_map = {m[0]: (m[1], m[2]) for m in _FIN_MARKETS}
|
||||
|
||||
quoted = ", ".join(f"'{n}'" for n in market_names)
|
||||
params = {
|
||||
"$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,noncomm_positions_long_all,noncomm_positions_short_all,open_interest_all,change_in_noncomm_long_all,change_in_noncomm_short_all",
|
||||
"$where": f"market_and_exchange_names in ({quoted})",
|
||||
"$order": "report_date_as_yyyy_mm_dd DESC",
|
||||
"$limit": str(len(market_names) * 2),
|
||||
}
|
||||
try:
|
||||
resp = requests.get(_LEGACY_FIN_URL, params=params, timeout=20)
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
except Exception as e:
|
||||
logger.error(f"COT financial fetch failed: {e}")
|
||||
return []
|
||||
|
||||
seen: set = set()
|
||||
for row in rows:
|
||||
mkt = row.get("market_and_exchange_names", "")
|
||||
if mkt not in name_map or mkt in seen:
|
||||
continue
|
||||
seen.add(mkt)
|
||||
label, ac = name_map[mkt]
|
||||
|
||||
nc_long = _parse_int_flat(row.get("noncomm_positions_long_all", 0))
|
||||
nc_short = _parse_int_flat(row.get("noncomm_positions_short_all", 0))
|
||||
oi = _parse_int_flat(row.get("open_interest_all", 0))
|
||||
net = nc_long - nc_short
|
||||
chg_long = _parse_int_flat(row.get("change_in_noncomm_long_all", 0))
|
||||
chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0))
|
||||
change_net = chg_long - chg_short
|
||||
net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
|
||||
|
||||
results.append({
|
||||
"market_name": mkt,
|
||||
"commodity": label,
|
||||
"asset_class": ac,
|
||||
"report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10],
|
||||
"mm_long": nc_long,
|
||||
"mm_short": nc_short,
|
||||
"open_interest": oi,
|
||||
"net_position": net,
|
||||
"net_pct_oi": net_pct_oi,
|
||||
"change_net": change_net,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def fetch_all_cot() -> List[Dict[str, Any]]:
|
||||
"""Fetch both disaggregated (commodities) and financial (forex/bonds) COT data.
|
||||
|
||||
Returns a flat list of per-market dicts suitable for save_cot_data().
|
||||
"""
|
||||
logger.info("Fetching COT data from CFTC (specialist desks feed)...")
|
||||
all_data: List[Dict[str, Any]] = []
|
||||
all_data.extend(_fetch_disaggregated_flat())
|
||||
all_data.extend(_fetch_financial_flat())
|
||||
logger.info(f"COT: fetched {len(all_data)} markets")
|
||||
return all_data
|
||||
|
||||
Reference in New Issue
Block a user