fix: COT fetch returns 0 — CFTC exchange names changed since 2022
Replace per-endpoint market name matching with unified contract code lookup on the legacy Socrata endpoint (6dca-aqww.json). COMEX became "COMMODITY EXCHANGE INC.", CBOT became "CHICAGO BOARD OF TRADE", and the disaggregated endpoints stopped receiving Natural Gas / financial instruments after Feb 2022. Contract codes (067651, 023651, etc.) are stable across rebranding. Fetch now returns 19/19 markets dated 2026-06-16. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
@@ -199,34 +199,30 @@ def fetch_cot_report() -> Optional[Dict]:
|
||||
|
||||
|
||||
# ── Specialist Desks flat COT feed (cot_data table) ──────────────────────────
|
||||
# All markets fetched from legacy endpoint via stable contract codes.
|
||||
# noncomm_positions = non-commercial (speculative) positioning.
|
||||
|
||||
_DISAGG_URL = "https://publicreporting.cftc.gov/resource/72hh-3qpy.json" # disaggregated commodities
|
||||
_LEGACY_FIN_URL = "https://publicreporting.cftc.gov/resource/gpe5-46if.json" # financial/forex
|
||||
|
||||
_DISAGG_MARKETS = [
|
||||
("CRUDE OIL, LIGHT SWEET - NYMEX", "WTI Crude", "energy"),
|
||||
("NATURAL GAS - NYMEX", "Natural Gas", "energy"),
|
||||
("GOLD - COMEX", "Gold", "metals"),
|
||||
("SILVER - COMEX", "Silver", "metals"),
|
||||
("COPPER- #1 - COMEX", "Copper", "metals"),
|
||||
("CORN - CBOT", "Corn", "agri"),
|
||||
("WHEAT - CBOT", "Wheat", "agri"),
|
||||
("SOYBEANS - CBOT", "Soybeans", "agri"),
|
||||
("SOYBEAN OIL - CBOT", "Soybean Oil", "agri"),
|
||||
("COCOA - ICE", "Cocoa", "agri"),
|
||||
("COFFEE C - ICE", "Coffee", "agri"),
|
||||
]
|
||||
|
||||
_FIN_MARKETS = [
|
||||
("EURO FX - CME", "Euro (EUR/USD)", "forex"),
|
||||
("JAPANESE YEN - CME", "Japanese Yen", "forex"),
|
||||
("BRITISH POUND STERLING - CME", "British Pound", "forex"),
|
||||
("SWISS FRANC - CME", "Swiss Franc", "forex"),
|
||||
("U.S. DOLLAR INDEX - ICE FUTURES U.S.", "DXY Index", "forex"),
|
||||
("30-DAY FEDERAL FUNDS - CBOT", "Fed Funds", "bonds"),
|
||||
("U.S. TREASURY BONDS - CBOT", "US T-Bonds", "bonds"),
|
||||
("10-YEAR U.S. TREASURY NOTES - CBOT", "10Y T-Notes", "bonds"),
|
||||
]
|
||||
_COT_MARKETS = {
|
||||
"067651": ("WTI Crude", "energy"),
|
||||
"023651": ("Natural Gas", "energy"),
|
||||
"088691": ("Gold", "metals"),
|
||||
"084691": ("Silver", "metals"),
|
||||
"085692": ("Copper", "metals"),
|
||||
"002602": ("Corn", "agri"),
|
||||
"001602": ("Wheat", "agri"),
|
||||
"005602": ("Soybeans", "agri"),
|
||||
"007601": ("Soybean Oil", "agri"),
|
||||
"073732": ("Cocoa", "agri"),
|
||||
"083731": ("Coffee", "agri"),
|
||||
"099741": ("Euro (EUR/USD)", "forex"),
|
||||
"097741": ("Japanese Yen", "forex"),
|
||||
"096742": ("British Pound", "forex"),
|
||||
"092741": ("Swiss Franc", "forex"),
|
||||
"098662": ("DXY Index", "forex"),
|
||||
"045601": ("Fed Funds", "bonds"),
|
||||
"020601": ("US T-Bonds", "bonds"),
|
||||
"043602": ("10Y T-Notes", "bonds"),
|
||||
}
|
||||
|
||||
|
||||
def _parse_int_flat(v) -> int:
|
||||
@@ -236,95 +232,52 @@ def _parse_int_flat(v) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _fetch_disaggregated_flat() -> List[Dict[str, Any]]:
|
||||
"""Fetch commodity COT (disaggregated) — MM long/short positions."""
|
||||
results = []
|
||||
market_names = [m[0] for m in _DISAGG_MARKETS]
|
||||
name_map = {m[0]: (m[1], m[2]) for m in _DISAGG_MARKETS}
|
||||
def fetch_all_cot() -> List[Dict[str, Any]]:
|
||||
"""Fetch COT non-commercial positioning for all tracked markets via contract codes.
|
||||
|
||||
Uses the legacy CFTC Socrata endpoint which covers all asset classes with
|
||||
stable contract codes and noncomm_positions_long/short_all fields.
|
||||
Returns a flat list of per-market dicts suitable for save_cot_data().
|
||||
"""
|
||||
logger.info("Fetching COT data from CFTC (specialist desks feed)...")
|
||||
codes = list(_COT_MARKETS.keys())
|
||||
codes_clause = ", ".join(f"'{c}'" for c in codes)
|
||||
|
||||
quoted = ", ".join(f"'{n}'" for n in market_names)
|
||||
params = {
|
||||
"$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,m_money_positions_long_all,m_money_positions_short_all,open_interest_all,change_in_m_money_long_all,change_in_m_money_short_all",
|
||||
"$where": f"market_and_exchange_names in ({quoted})",
|
||||
"$select": (
|
||||
"cftc_contract_market_code,market_and_exchange_names,"
|
||||
"report_date_as_yyyy_mm_dd,open_interest_all,"
|
||||
"noncomm_positions_long_all,noncomm_positions_short_all,"
|
||||
"change_in_noncomm_long_all,change_in_noncomm_short_all"
|
||||
),
|
||||
"$where": f"cftc_contract_market_code in ({codes_clause})",
|
||||
"$order": "report_date_as_yyyy_mm_dd DESC",
|
||||
"$limit": str(len(market_names) * 2),
|
||||
"$limit": str(len(codes) * 3),
|
||||
}
|
||||
try:
|
||||
resp = requests.get(_DISAGG_URL, params=params, timeout=20)
|
||||
resp = requests.get(SOCRATA_URL, params=params, timeout=25)
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
except Exception as e:
|
||||
logger.error(f"COT disaggregated fetch failed: {e}")
|
||||
logger.error(f"COT fetch_all_cot failed: {e}")
|
||||
return []
|
||||
|
||||
seen: set = set()
|
||||
results: List[Dict[str, Any]] = []
|
||||
for row in rows:
|
||||
mkt = row.get("market_and_exchange_names", "")
|
||||
if mkt not in name_map or mkt in seen:
|
||||
code = row.get("cftc_contract_market_code", "")
|
||||
if code not in _COT_MARKETS or code in seen:
|
||||
continue
|
||||
seen.add(mkt)
|
||||
label, ac = name_map[mkt]
|
||||
|
||||
mm_long = _parse_int_flat(row.get("m_money_positions_long_all", 0))
|
||||
mm_short = _parse_int_flat(row.get("m_money_positions_short_all", 0))
|
||||
oi = _parse_int_flat(row.get("open_interest_all", 0))
|
||||
net = mm_long - mm_short
|
||||
chg_long = _parse_int_flat(row.get("change_in_m_money_long_all", 0))
|
||||
chg_short = _parse_int_flat(row.get("change_in_m_money_short_all", 0))
|
||||
change_net = chg_long - chg_short
|
||||
net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
|
||||
|
||||
results.append({
|
||||
"market_name": mkt,
|
||||
"commodity": label,
|
||||
"asset_class": ac,
|
||||
"report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10],
|
||||
"mm_long": mm_long,
|
||||
"mm_short": mm_short,
|
||||
"open_interest": oi,
|
||||
"net_position": net,
|
||||
"net_pct_oi": net_pct_oi,
|
||||
"change_net": change_net,
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def _fetch_financial_flat() -> List[Dict[str, Any]]:
|
||||
"""Fetch financial/forex COT (legacy) — Non-commercial long/short."""
|
||||
results = []
|
||||
market_names = [m[0] for m in _FIN_MARKETS]
|
||||
name_map = {m[0]: (m[1], m[2]) for m in _FIN_MARKETS}
|
||||
|
||||
quoted = ", ".join(f"'{n}'" for n in market_names)
|
||||
params = {
|
||||
"$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,noncomm_positions_long_all,noncomm_positions_short_all,open_interest_all,change_in_noncomm_long_all,change_in_noncomm_short_all",
|
||||
"$where": f"market_and_exchange_names in ({quoted})",
|
||||
"$order": "report_date_as_yyyy_mm_dd DESC",
|
||||
"$limit": str(len(market_names) * 2),
|
||||
}
|
||||
try:
|
||||
resp = requests.get(_LEGACY_FIN_URL, params=params, timeout=20)
|
||||
resp.raise_for_status()
|
||||
rows = resp.json()
|
||||
except Exception as e:
|
||||
logger.error(f"COT financial fetch failed: {e}")
|
||||
return []
|
||||
|
||||
seen: set = set()
|
||||
for row in rows:
|
||||
seen.add(code)
|
||||
label, ac = _COT_MARKETS[code]
|
||||
mkt = row.get("market_and_exchange_names", "")
|
||||
if mkt not in name_map or mkt in seen:
|
||||
continue
|
||||
seen.add(mkt)
|
||||
label, ac = name_map[mkt]
|
||||
|
||||
nc_long = _parse_int_flat(row.get("noncomm_positions_long_all", 0))
|
||||
nc_short = _parse_int_flat(row.get("noncomm_positions_short_all", 0))
|
||||
oi = _parse_int_flat(row.get("open_interest_all", 0))
|
||||
net = nc_long - nc_short
|
||||
chg_long = _parse_int_flat(row.get("change_in_noncomm_long_all", 0))
|
||||
chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0))
|
||||
nc_long = _parse_int_flat(row.get("noncomm_positions_long_all", 0))
|
||||
nc_short = _parse_int_flat(row.get("noncomm_positions_short_all", 0))
|
||||
oi = _parse_int_flat(row.get("open_interest_all", 0))
|
||||
net = nc_long - nc_short
|
||||
chg_long = _parse_int_flat(row.get("change_in_noncomm_long_all", 0))
|
||||
chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0))
|
||||
change_net = chg_long - chg_short
|
||||
net_pct_oi = round(net / oi * 100, 2) if oi else 0.0
|
||||
|
||||
@@ -341,17 +294,5 @@ def _fetch_financial_flat() -> List[Dict[str, Any]]:
|
||||
"change_net": change_net,
|
||||
})
|
||||
|
||||
logger.info(f"COT: fetched {len(results)}/{len(codes)} markets")
|
||||
return results
|
||||
|
||||
|
||||
def fetch_all_cot() -> List[Dict[str, Any]]:
|
||||
"""Fetch both disaggregated (commodities) and financial (forex/bonds) COT data.
|
||||
|
||||
Returns a flat list of per-market dicts suitable for save_cot_data().
|
||||
"""
|
||||
logger.info("Fetching COT data from CFTC (specialist desks feed)...")
|
||||
all_data: List[Dict[str, Any]] = []
|
||||
all_data.extend(_fetch_disaggregated_flat())
|
||||
all_data.extend(_fetch_financial_flat())
|
||||
logger.info(f"COT: fetched {len(all_data)} markets")
|
||||
return all_data
|
||||
|
||||
Reference in New Issue
Block a user