From c9f7757a861247c618656253191563b935ede9c1 Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Tue, 23 Jun 2026 18:53:51 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20COT=20fetch=20returns=200=20=E2=80=94=20?= =?UTF-8?q?CFTC=20exchange=20names=20changed=20since=202022?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace per-endpoint market name matching with unified contract code lookup on the legacy Socrata endpoint (6dca-aqww.json). COMEX became "COMMODITY EXCHANGE INC.", CBOT became "CHICAGO BOARD OF TRADE", and the disaggregated endpoints stopped receiving Natural Gas / financial instruments after Feb 2022. Contract codes (067651, 023651, etc.) are stable across rebranding. Fetch now returns 19/19 markets dated 2026-06-16. Co-Authored-By: Claude Sonnet 4.6 --- backend/services/cot_fetcher.py | 169 +++++++++++--------------------- 1 file changed, 55 insertions(+), 114 deletions(-) diff --git a/backend/services/cot_fetcher.py b/backend/services/cot_fetcher.py index 237a724..a006046 100644 --- a/backend/services/cot_fetcher.py +++ b/backend/services/cot_fetcher.py @@ -199,34 +199,30 @@ def fetch_cot_report() -> Optional[Dict]: # ── Specialist Desks flat COT feed (cot_data table) ────────────────────────── +# All markets fetched from legacy endpoint via stable contract codes. +# noncomm_positions = non-commercial (speculative) positioning. -_DISAGG_URL = "https://publicreporting.cftc.gov/resource/72hh-3qpy.json" # disaggregated commodities -_LEGACY_FIN_URL = "https://publicreporting.cftc.gov/resource/gpe5-46if.json" # financial/forex - -_DISAGG_MARKETS = [ - ("CRUDE OIL, LIGHT SWEET - NYMEX", "WTI Crude", "energy"), - ("NATURAL GAS - NYMEX", "Natural Gas", "energy"), - ("GOLD - COMEX", "Gold", "metals"), - ("SILVER - COMEX", "Silver", "metals"), - ("COPPER- #1 - COMEX", "Copper", "metals"), - ("CORN - CBOT", "Corn", "agri"), - ("WHEAT - CBOT", "Wheat", "agri"), - ("SOYBEANS - CBOT", "Soybeans", "agri"), - ("SOYBEAN OIL - CBOT", "Soybean Oil", "agri"), - ("COCOA - ICE", "Cocoa", "agri"), - ("COFFEE C - ICE", "Coffee", "agri"), -] - -_FIN_MARKETS = [ - ("EURO FX - CME", "Euro (EUR/USD)", "forex"), - ("JAPANESE YEN - CME", "Japanese Yen", "forex"), - ("BRITISH POUND STERLING - CME", "British Pound", "forex"), - ("SWISS FRANC - CME", "Swiss Franc", "forex"), - ("U.S. DOLLAR INDEX - ICE FUTURES U.S.", "DXY Index", "forex"), - ("30-DAY FEDERAL FUNDS - CBOT", "Fed Funds", "bonds"), - ("U.S. TREASURY BONDS - CBOT", "US T-Bonds", "bonds"), - ("10-YEAR U.S. TREASURY NOTES - CBOT", "10Y T-Notes", "bonds"), -] +_COT_MARKETS = { + "067651": ("WTI Crude", "energy"), + "023651": ("Natural Gas", "energy"), + "088691": ("Gold", "metals"), + "084691": ("Silver", "metals"), + "085692": ("Copper", "metals"), + "002602": ("Corn", "agri"), + "001602": ("Wheat", "agri"), + "005602": ("Soybeans", "agri"), + "007601": ("Soybean Oil", "agri"), + "073732": ("Cocoa", "agri"), + "083731": ("Coffee", "agri"), + "099741": ("Euro (EUR/USD)", "forex"), + "097741": ("Japanese Yen", "forex"), + "096742": ("British Pound", "forex"), + "092741": ("Swiss Franc", "forex"), + "098662": ("DXY Index", "forex"), + "045601": ("Fed Funds", "bonds"), + "020601": ("US T-Bonds", "bonds"), + "043602": ("10Y T-Notes", "bonds"), +} def _parse_int_flat(v) -> int: @@ -236,95 +232,52 @@ def _parse_int_flat(v) -> int: return 0 -def _fetch_disaggregated_flat() -> List[Dict[str, Any]]: - """Fetch commodity COT (disaggregated) — MM long/short positions.""" - results = [] - market_names = [m[0] for m in _DISAGG_MARKETS] - name_map = {m[0]: (m[1], m[2]) for m in _DISAGG_MARKETS} +def fetch_all_cot() -> List[Dict[str, Any]]: + """Fetch COT non-commercial positioning for all tracked markets via contract codes. + + Uses the legacy CFTC Socrata endpoint which covers all asset classes with + stable contract codes and noncomm_positions_long/short_all fields. + Returns a flat list of per-market dicts suitable for save_cot_data(). + """ + logger.info("Fetching COT data from CFTC (specialist desks feed)...") + codes = list(_COT_MARKETS.keys()) + codes_clause = ", ".join(f"'{c}'" for c in codes) - quoted = ", ".join(f"'{n}'" for n in market_names) params = { - "$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,m_money_positions_long_all,m_money_positions_short_all,open_interest_all,change_in_m_money_long_all,change_in_m_money_short_all", - "$where": f"market_and_exchange_names in ({quoted})", + "$select": ( + "cftc_contract_market_code,market_and_exchange_names," + "report_date_as_yyyy_mm_dd,open_interest_all," + "noncomm_positions_long_all,noncomm_positions_short_all," + "change_in_noncomm_long_all,change_in_noncomm_short_all" + ), + "$where": f"cftc_contract_market_code in ({codes_clause})", "$order": "report_date_as_yyyy_mm_dd DESC", - "$limit": str(len(market_names) * 2), + "$limit": str(len(codes) * 3), } try: - resp = requests.get(_DISAGG_URL, params=params, timeout=20) + resp = requests.get(SOCRATA_URL, params=params, timeout=25) resp.raise_for_status() rows = resp.json() except Exception as e: - logger.error(f"COT disaggregated fetch failed: {e}") + logger.error(f"COT fetch_all_cot failed: {e}") return [] seen: set = set() + results: List[Dict[str, Any]] = [] for row in rows: - mkt = row.get("market_and_exchange_names", "") - if mkt not in name_map or mkt in seen: + code = row.get("cftc_contract_market_code", "") + if code not in _COT_MARKETS or code in seen: continue - seen.add(mkt) - label, ac = name_map[mkt] - - mm_long = _parse_int_flat(row.get("m_money_positions_long_all", 0)) - mm_short = _parse_int_flat(row.get("m_money_positions_short_all", 0)) - oi = _parse_int_flat(row.get("open_interest_all", 0)) - net = mm_long - mm_short - chg_long = _parse_int_flat(row.get("change_in_m_money_long_all", 0)) - chg_short = _parse_int_flat(row.get("change_in_m_money_short_all", 0)) - change_net = chg_long - chg_short - net_pct_oi = round(net / oi * 100, 2) if oi else 0.0 - - results.append({ - "market_name": mkt, - "commodity": label, - "asset_class": ac, - "report_date": row.get("report_date_as_yyyy_mm_dd", "")[:10], - "mm_long": mm_long, - "mm_short": mm_short, - "open_interest": oi, - "net_position": net, - "net_pct_oi": net_pct_oi, - "change_net": change_net, - }) - - return results - - -def _fetch_financial_flat() -> List[Dict[str, Any]]: - """Fetch financial/forex COT (legacy) — Non-commercial long/short.""" - results = [] - market_names = [m[0] for m in _FIN_MARKETS] - name_map = {m[0]: (m[1], m[2]) for m in _FIN_MARKETS} - - quoted = ", ".join(f"'{n}'" for n in market_names) - params = { - "$select": "market_and_exchange_names,report_date_as_yyyy_mm_dd,noncomm_positions_long_all,noncomm_positions_short_all,open_interest_all,change_in_noncomm_long_all,change_in_noncomm_short_all", - "$where": f"market_and_exchange_names in ({quoted})", - "$order": "report_date_as_yyyy_mm_dd DESC", - "$limit": str(len(market_names) * 2), - } - try: - resp = requests.get(_LEGACY_FIN_URL, params=params, timeout=20) - resp.raise_for_status() - rows = resp.json() - except Exception as e: - logger.error(f"COT financial fetch failed: {e}") - return [] - - seen: set = set() - for row in rows: + seen.add(code) + label, ac = _COT_MARKETS[code] mkt = row.get("market_and_exchange_names", "") - if mkt not in name_map or mkt in seen: - continue - seen.add(mkt) - label, ac = name_map[mkt] - nc_long = _parse_int_flat(row.get("noncomm_positions_long_all", 0)) - nc_short = _parse_int_flat(row.get("noncomm_positions_short_all", 0)) - oi = _parse_int_flat(row.get("open_interest_all", 0)) - net = nc_long - nc_short - chg_long = _parse_int_flat(row.get("change_in_noncomm_long_all", 0)) - chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0)) + nc_long = _parse_int_flat(row.get("noncomm_positions_long_all", 0)) + nc_short = _parse_int_flat(row.get("noncomm_positions_short_all", 0)) + oi = _parse_int_flat(row.get("open_interest_all", 0)) + net = nc_long - nc_short + chg_long = _parse_int_flat(row.get("change_in_noncomm_long_all", 0)) + chg_short = _parse_int_flat(row.get("change_in_noncomm_short_all", 0)) change_net = chg_long - chg_short net_pct_oi = round(net / oi * 100, 2) if oi else 0.0 @@ -341,17 +294,5 @@ def _fetch_financial_flat() -> List[Dict[str, Any]]: "change_net": change_net, }) + logger.info(f"COT: fetched {len(results)}/{len(codes)} markets") return results - - -def fetch_all_cot() -> List[Dict[str, Any]]: - """Fetch both disaggregated (commodities) and financial (forex/bonds) COT data. - - Returns a flat list of per-market dicts suitable for save_cot_data(). - """ - logger.info("Fetching COT data from CFTC (specialist desks feed)...") - all_data: List[Dict[str, Any]] = [] - all_data.extend(_fetch_disaggregated_flat()) - all_data.extend(_fetch_financial_flat()) - logger.info(f"COT: fetched {len(all_data)} markets") - return all_data