From 5d3ff1939314d129907b0b19f45eedc653288918 Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Sat, 20 Jun 2026 18:58:25 +0200 Subject: [PATCH] fix: ticker-based asset_class fallback + backfill migration for NULL rows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - _normalize_asset_class() now accepts ticker param and infers class from a full ticker→class lookup table (energy/metals/agri/indices/equities/forex) - init_db() runs one-time UPDATE to backfill all NULL asset_class rows in trade_entry_prices and skipped_trades using known ticker lists - log_trade_entries and log_skipped_trade pass ticker to normalizer - Frontend _normalizeAssetClass() gets same ticker lookup + pattern fallbacks for =F futures, NSE: prefixed equities, =X currency pairs - All 3 filter calls now pass t.underlying as second argument Co-Authored-By: Claude Sonnet 4.6 --- backend/services/database.py | 112 ++++++++++++++++++++++----- frontend/src/pages/JournalDeBord.tsx | 71 +++++++++++++---- 2 files changed, 152 insertions(+), 31 deletions(-) diff --git a/backend/services/database.py b/backend/services/database.py index a1c0677..5ab3a0b 100644 --- a/backend/services/database.py +++ b/backend/services/database.py @@ -230,6 +230,28 @@ def init_db(): except Exception: pass + # One-time migration: fix existing NULL asset_class rows using ticker lookup + _backfill_cases = [ + ("energy", "'CL=F','BZ=F','NG=F','RB=F','HO=F','XLE','XOP','USO','UCO','BOIL','UNG'"), + ("metals", "'GC=F','SI=F','HG=F','PA=F','PL=F','GLD','IAU','SLV','GDX','GDXJ','GLDM','PPLT'"), + ("agriculture", "'ZC=F','ZS=F','ZW=F','CC=F','KC=F','CT=F','OJ=F','CORN','WEAT','SOYB','DBA'"), + ("indices", "'^GSPC','^NDX','^DJI','^RUT','SPY','QQQ','IWM','DIA','RSP','VGK','EEM','EWZ','FXI','EWJ','EFA','ACWI','INDA','^NSEI','^HSI','EWG','EWU','EWI'"), + ("equities", "'XLF','XLK','XLV','XLI','XLP','XLU','XLY','XLRE','XLB','XLC'"), + ("forex", "'DX-Y.NYB','UUP','FXE','FXY','EUO','YCS','FXA','FXB','FXF'"), + ] + for _cls, _tickers in _backfill_cases: + try: + c.execute( + f"UPDATE trade_entry_prices SET asset_class=? WHERE (asset_class IS NULL OR asset_class='') AND underlying IN ({_tickers})", + (_cls,) + ) + c.execute( + f"UPDATE skipped_trades SET asset_class=? WHERE (asset_class IS NULL OR asset_class='') AND underlying IN ({_tickers})", + (_cls,) + ) + except Exception: + pass + c.execute("""CREATE TABLE IF NOT EXISTS risk_profiles ( id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, @@ -1027,24 +1049,77 @@ def _normalize_yf_ticker(ticker: str) -> str: return t -def _normalize_asset_class(cls: str) -> str: - """Map AI-returned asset_class variants to canonical keys used in the UI.""" - if not cls: +_TICKER_ASSET_CLASS: dict = { + # Energy + "CL=F": "energy", "BZ=F": "energy", "NG=F": "energy", "RB=F": "energy", "HO=F": "energy", + "XLE": "energy", "XOP": "energy", "USO": "energy", "UCO": "energy", "BOIL": "energy", "UNG": "energy", + # Metals + "GC=F": "metals", "SI=F": "metals", "HG=F": "metals", "PA=F": "metals", "PL=F": "metals", + "GLD": "metals", "IAU": "metals", "SLV": "metals", "GDX": "metals", "GDXJ": "metals", + "PPLT": "metals", "DBP": "metals", "GLDM": "metals", + # Agriculture + "ZC=F": "agriculture", "ZS=F": "agriculture", "ZW=F": "agriculture", + "CC=F": "agriculture", "KC=F": "agriculture", "CT=F": "agriculture", "OJ=F": "agriculture", + "CORN": "agriculture", "WEAT": "agriculture", "SOYB": "agriculture", "DBA": "agriculture", + # Indices + "^GSPC": "indices", "^NDX": "indices", "^DJI": "indices", "^RUT": "indices", + "SPY": "indices", "QQQ": "indices", "IWM": "indices", "DIA": "indices", "RSP": "indices", + "VGK": "indices", "EEM": "indices", "EWZ": "indices", "FXI": "indices", + "EWJ": "indices", "EFA": "indices", "ACWI": "indices", "INDA": "indices", + "^NSEI": "indices", "^HSI": "indices", "EWG": "indices", "EWU": "indices", "EWI": "indices", + # Equities (sector ETFs & individual stocks treated as equities) + "XLF": "equities", "XLK": "equities", "XLV": "equities", "XLI": "equities", + "XLP": "equities", "XLU": "equities", "XLY": "equities", "XLRE": "equities", + "XLB": "equities", "XLC": "equities", + # Forex + "DX-Y.NYB": "forex", "UUP": "forex", "FXE": "forex", "FXY": "forex", + "EUO": "forex", "YCS": "forex", "FXA": "forex", "FXB": "forex", "FXF": "forex", +} + + +def _asset_class_from_ticker(ticker: str) -> str: + """Infer asset class from ticker symbol when no explicit class is available.""" + if not ticker: return "" - c = cls.lower().strip() - if any(k in c for k in ("energy", "oil", "gas", "petrol", "brent", "wti")): - return "energy" - if any(k in c for k in ("metal", "gold", "silver", "copper", "mining", "precious")): - return "metals" - if any(k in c for k in ("agri", "grain", "corn", "wheat", "soy", "crop", "coton", "coffee", "cocoa")): - return "agriculture" - if any(k in c for k in ("index", "indic", "indices", "spx", "nasdaq", "dow", "s&p", "russell", "cac", "dax")): - return "indices" - if any(k in c for k in ("equit", "stock", "action", "share", "sector", "xle", "xlf", "xlk")): - return "equities" - if any(k in c for k in ("forex", "currency", "fx", "devise", "change", "eur", "usd", "jpy", "dxy")): + t = ticker.upper().strip() + if t in _TICKER_ASSET_CLASS: + return _TICKER_ASSET_CLASS[t] + # Futures suffix patterns + if t.endswith("=F"): + stem = t[:-2] + if stem[:2] in ("CL", "RB", "HO", "NG", "BZ"): + return "energy" + if stem[:2] in ("GC", "SI", "HG", "PA", "PL"): + return "metals" + if stem[:2] in ("ZC", "ZS", "ZW", "CC", "KC", "CT"): + return "agriculture" + # Currency pairs + if "=X" in t or "/" in t: return "forex" - return c + return "" + + +def _normalize_asset_class(cls: str, ticker: str = "") -> str: + """Map AI-returned asset_class variants to canonical keys, with ticker fallback.""" + if cls: + c = cls.lower().strip() + if any(k in c for k in ("energy", "oil", "gas", "petrol", "brent", "wti")): + return "energy" + if any(k in c for k in ("metal", "gold", "silver", "copper", "mining", "precious")): + return "metals" + if any(k in c for k in ("agri", "grain", "corn", "wheat", "soy", "crop", "coton", "coffee", "cocoa")): + return "agriculture" + if any(k in c for k in ("index", "indic", "indices", "spx", "nasdaq", "dow", "s&p", "russell", "cac", "dax")): + return "indices" + if any(k in c for k in ("equit", "stock", "action", "share", "sector")): + return "equities" + if any(k in c for k in ("forex", "currency", "fx", "devise", "change", "eur", "usd", "jpy", "dxy")): + return "forex" + # Already a canonical value + if c in ("energy", "metals", "agriculture", "indices", "equities", "forex"): + return c + # Fallback: infer from ticker + return _asset_class_from_ticker(ticker) def log_trade_entries(run_id: str, scored_patterns: List[Dict[str, Any]], quotes: Dict[str, Any]): @@ -1215,7 +1290,8 @@ def log_trade_entries(run_id: str, scored_patterns: List[Dict[str, Any]], quotes trade.get("asset_class") or sp.get("asset_class") or _orig.get("asset_class") or - "" + "", + ticker=ticker_key ) conn.execute("""INSERT INTO trade_entry_prices (run_id, pattern_id, pattern_name, underlying, strategy, @@ -1407,7 +1483,7 @@ def log_skipped_trade(run_id: str, pattern_id: str, pattern_name: str, expected_move_pct, skip_reason, skip_detail, asset_class) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""", (run_id, pattern_id, pattern_name, underlying, strategy, score, - expected_move_pct, skip_reason, skip_detail, _normalize_asset_class(asset_class)) + expected_move_pct, skip_reason, skip_detail, _normalize_asset_class(asset_class, ticker=underlying)) ) conn.commit() conn.close() diff --git a/frontend/src/pages/JournalDeBord.tsx b/frontend/src/pages/JournalDeBord.tsx index c8e42e5..6a6299a 100644 --- a/frontend/src/pages/JournalDeBord.tsx +++ b/frontend/src/pages/JournalDeBord.tsx @@ -240,16 +240,61 @@ function _isBearishStr(strategy: string) { } // Normalize AI-returned asset_class variants to canonical CATEGORIES keys -function _normalizeAssetClass(cls: string | null | undefined): string { - if (!cls) return '' - const c = cls.toLowerCase().trim() - if (/energy|oil|gas|petrol|brent|wti/.test(c)) return 'energy' - if (/metal|gold|silver|copper|mining|precious/.test(c)) return 'metals' - if (/agri|grain|corn|wheat|soy|crop|coton|coffee|cocoa/.test(c)) return 'agriculture' - if (/index|indic|indices|spx|nasdaq|dow|s&p|russell|cac|dax/.test(c)) return 'indices' - if (/equit|stock|action|share|sector|xle|xlf|xlk/.test(c)) return 'equities' - if (/forex|currency|fx|devise|change|eur|usd|jpy|dxy/.test(c)) return 'forex' - return c +const _TICKER_TO_CLASS: Record = { + // Energy + 'CL=F': 'energy', 'BZ=F': 'energy', 'NG=F': 'energy', 'RB=F': 'energy', 'HO=F': 'energy', + 'XLE': 'energy', 'XOP': 'energy', 'USO': 'energy', 'UCO': 'energy', 'BOIL': 'energy', 'UNG': 'energy', + // Metals + 'GC=F': 'metals', 'SI=F': 'metals', 'HG=F': 'metals', 'PA=F': 'metals', 'PL=F': 'metals', + 'GLD': 'metals', 'IAU': 'metals', 'SLV': 'metals', 'GDX': 'metals', 'GDXJ': 'metals', + 'PPLT': 'metals', 'DBP': 'metals', 'GLDM': 'metals', + // Agriculture + 'ZC=F': 'agriculture', 'ZS=F': 'agriculture', 'ZW=F': 'agriculture', + 'CC=F': 'agriculture', 'KC=F': 'agriculture', 'CT=F': 'agriculture', 'OJ=F': 'agriculture', + 'CORN': 'agriculture', 'WEAT': 'agriculture', 'SOYB': 'agriculture', 'DBA': 'agriculture', + // Indices + '^GSPC': 'indices', '^NDX': 'indices', '^DJI': 'indices', '^RUT': 'indices', + 'SPY': 'indices', 'QQQ': 'indices', 'IWM': 'indices', 'DIA': 'indices', 'RSP': 'indices', + 'VGK': 'indices', 'EEM': 'indices', 'EWZ': 'indices', 'FXI': 'indices', + 'EWJ': 'indices', 'EFA': 'indices', 'ACWI': 'indices', 'INDA': 'indices', + '^NSEI': 'indices', '^HSI': 'indices', 'EWG': 'indices', 'EWU': 'indices', 'EWI': 'indices', + // Equities (sector ETFs) + 'XLF': 'equities', 'XLK': 'equities', 'XLV': 'equities', 'XLI': 'equities', + 'XLP': 'equities', 'XLU': 'equities', 'XLY': 'equities', 'XLRE': 'equities', + 'XLB': 'equities', 'XLC': 'equities', + // Forex + 'DX-Y.NYB': 'forex', 'UUP': 'forex', 'FXE': 'forex', 'FXY': 'forex', + 'EUO': 'forex', 'YCS': 'forex', 'FXA': 'forex', 'FXB': 'forex', 'FXF': 'forex', +} + +function _normalizeAssetClass(cls: string | null | undefined, ticker?: string | null): string { + if (cls) { + const c = cls.toLowerCase().trim() + if (/energy|oil|gas|petrol|brent|wti/.test(c)) return 'energy' + if (/metal|gold|silver|copper|mining|precious/.test(c)) return 'metals' + if (/agri|grain|corn|wheat|soy|crop|coton|coffee|cocoa/.test(c)) return 'agriculture' + if (/index|indic|indices|spx|nasdaq|dow|s&p|russell|cac|dax/.test(c)) return 'indices' + if (/equit|stock|action|share|sector/.test(c)) return 'equities' + if (/forex|currency|fx|devise|change|eur|usd|jpy|dxy/.test(c)) return 'forex' + if (['energy', 'metals', 'agriculture', 'indices', 'equities', 'forex'].includes(c)) return c + } + // Fallback: infer from ticker symbol + if (ticker) { + const t = ticker.toUpperCase().trim() + if (_TICKER_TO_CLASS[t]) return _TICKER_TO_CLASS[t] + // NSE: or other exchange-prefixed equities + if (t.includes(':')) return 'equities' + // Remaining =F futures: crude/energy or precious/metals heuristic + if (t.endsWith('=F')) { + const stem = t.slice(0, 2) + if (['CL', 'RB', 'HO', 'NG', 'BZ'].includes(stem)) return 'energy' + if (['GC', 'SI', 'HG', 'PA', 'PL'].includes(stem)) return 'metals' + if (['ZC', 'ZS', 'ZW', 'CC', 'KC', 'CT'].includes(stem)) return 'agriculture' + } + // Currency pairs + if (t.includes('=X') || t.includes('/')) return 'forex' + } + return cls?.toLowerCase().trim() ?? '' } interface FilterBarProps { @@ -343,7 +388,7 @@ function ClosedTradesSection({ days }: { days: number }) { const trades = allTrades.filter((t: any) => { const q = search.toLowerCase() if (q && !`${t.underlying} ${t.strategy ?? ''} ${t.pattern_name ?? ''}`.toLowerCase().includes(q)) return false - if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class) !== assetClass) return false + if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class, t.underlying) !== assetClass) return false const bearish = _isBearishStr(t.strategy ?? '') if (direction === 'bullish' && bearish) return false if (direction === 'bearish' && !bearish) return false @@ -913,7 +958,7 @@ function TradeMtmSection({ days }: { days: number }) { if ((t.latest_score ?? t.score_at_entry ?? 0) < minScoreFilter) return false const q = search.toLowerCase() if (q && !`${t.underlying} ${t.strategy ?? ''} ${t.pattern_name ?? ''}`.toLowerCase().includes(q)) return false - if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class) !== assetClass) return false + if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class, t.underlying) !== assetClass) return false const bearish = _isBearishStr(t.strategy ?? '') if (direction === 'bullish' && bearish) return false if (direction === 'bearish' && !bearish) return false @@ -1645,7 +1690,7 @@ function SkippedTradesSection({ days }: { days: number }) { const trades = allTrades.filter((t: any) => { const q = search.toLowerCase() if (q && !`${t.underlying} ${t.strategy ?? ''} ${t.pattern_name ?? ''}`.toLowerCase().includes(q)) return false - if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class) !== assetClass) return false + if (assetClass !== 'all' && _normalizeAssetClass(t.asset_class, t.underlying) !== assetClass) return false if (reasonFilter !== 'all' && t.skip_reason !== reasonFilter) return false return true })