admin/OpenFin
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: ticker normalization + GPT-4o 429 retry

Browse Source 
Ticker normalization (_normalize_ticker):
- EUR/USD slash-format → EURUSD=X (was passed raw to yfinance → 500/404 spam)
- bare 6-char forex pairs EURUSD/USDJPY etc → append =X
- commodity alias table: WHEAT→ZW=F, CORN→ZC=F, WTI→CL=F, BRENT→BZ=F,
  GOLD→GC=F, SILVER→SI=F, NATGAS→NG=F, SUGAR→SB=F, + 15 others
- also normalize underlying at log_trade_entries time so stored tickers
  are already canonical before MtM lookups

GPT-4o 429 rate limit:
- _chat() retries up to 3× on rate_limit errors, respects retry-after hint
  from error message (e.g. "try again in 12.37s"), falls back to 2^n×5s
- batch scorer: parallel workers 4→2 to halve the token burst per cycle
  (2 concurrent batches × ~6K tokens vs 4 × ~6K = 24K burst at 30K limit)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-19 14:29:39 +02:00
parent 8122bd019c
commit fda6b6a297
2 changed files with 63 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
backend/services/ai_analyzer.py
View File

@@ -21,6 +21,7 @@ def get_client() -> Optional[OpenAI]:
def _chat(system: str, user: str, model: str = "gpt-4o-mini", json_mode: bool = True, max_tokens: int = 1500) -> Optional[Dict]:
import time as _time
client = get_client()
if not client:
return None
@@ -32,11 +33,26 @@ def _chat(system: str, user: str, model: str = "gpt-4o-mini", json_mode: bool =
}
if json_mode:
kwargs["response_format"] = {"type": "json_object"}
resp = client.chat.completions.create(**kwargs)
content = resp.choices[0].message.content
if json_mode:
return json.loads(content)
return {"text": content}
last_exc = None
for attempt in range(4): # up to 3 retries
try:
resp = client.chat.completions.create(**kwargs)
content = resp.choices[0].message.content
if json_mode:
return json.loads(content)
return {"text": content}
except Exception as e:
last_exc = e
err_str = str(e)
# 429 rate-limit: respect the retry-after hint, then back off
if "429" in err_str or "rate_limit" in err_str:
import re as _re
m = _re.search(r"try again in ([\d.]+)s", err_str)
wait = float(m.group(1)) + 1.0 if m else 2 ** (attempt + 1) * 5.0
_time.sleep(min(wait, 60.0))
continue
raise # non-429 errors propagate immediately
raise last_exc
# ── News / Article Analysis ───────────────────────────────────────────────────
@@ -744,12 +760,13 @@ TEMPLATE DE NOTATION:
BATCH_SIZE = 4 # 4 patterns × ~800 tokens output = ~3200 tokens, safely within gpt-4o limits
# Score all batches in parallel
# Score batches with limited parallelism to stay under TPM limits.
# 2 concurrent batches × ~6K tokens = ~12K tokens burst, well under 30K TPM.
from concurrent.futures import ThreadPoolExecutor, as_completed
batches = [pattern_blocks[i:i+BATCH_SIZE] for i in range(0, len(pattern_blocks), BATCH_SIZE)]
_scorer_log.info(f"[Scorer] Scoring {len(pattern_blocks)} patterns in {len(batches)} batches of max {BATCH_SIZE}")
all_scored = []
with ThreadPoolExecutor(max_workers=min(len(batches), 4)) as executor:
with ThreadPoolExecutor(max_workers=min(len(batches), 2)) as executor:
futures = [executor.submit(_score_batch, b) for b in batches]
for future in as_completed(futures):
try:

44
backend/services/database.py
View File

@@ -1017,7 +1017,7 @@ def log_trade_entries(run_id: str, scored_patterns: List[Dict[str, Any]], quotes
ev_gross, ev_net, trade_score = _compute_trade_score(eff_score, exp_move)
ticker_key = underlying.upper()
ticker_key = _normalize_ticker(underlying.upper())
entry_price = price_map.get(ticker_key)
horizon = int(
trade.get("horizon_days") or
@@ -1331,13 +1331,47 @@ _EXCHANGE_PREFIX_MAP = {
"HKG": ".HK", "SHA": ".SS", "SHE": ".SZ", "TYO": ".T",
}
# Commodity names that GPT-4o sometimes suggests as tickers
_COMMODITY_ALIAS: Dict[str, str] = {
"WHEAT": "ZW=F", "CORN": "ZC=F", "SOYBEANS": "ZS=F", "SOYBEAN": "ZS=F",
"SUGAR": "SB=F", "COFFEE": "KC=F", "COTTON": "CT=F",
"OIL": "CL=F", "CRUDE": "CL=F", "CRUDEOIL": "CL=F", "WTI": "CL=F",
"BRENT": "BZ=F", "NATGAS": "NG=F", "GAS": "NG=F",
"GOLD": "GC=F", "SILVER": "SI=F", "COPPER": "HG=F",
"PLATINUM": "PL=F", "PALLADIUM": "PA=F",
"NASDAQ": "QQQ", "SP500": "SPY", "DOW": "DIA",
"VIX": "^VIX", "DOLLAR": "DX-Y.NYB", "DXY": "DX-Y.NYB",
"EURUSD": "EURUSD=X", "GBPUSD": "GBPUSD=X", "USDJPY": "USDJPY=X",
"USDCHF": "USDCHF=X", "AUDUSD": "AUDUSD=X", "USDCAD": "USDCAD=X",
}
def _normalize_ticker(ticker: str) -> str:
"""Convert exchange:symbol formats (from GPT-4o) to yfinance-compatible tickers."""
if ":" in ticker:
exchange, symbol = ticker.split(":", 1)
"""Convert various ticker formats (from GPT-4o) to yfinance-compatible tickers."""
t = ticker.strip()
if not t:
return t
# exchange:symbol format (e.g. NSE:RELIANCE → RELIANCE.NS)
if ":" in t:
exchange, symbol = t.split(":", 1)
suffix = _EXCHANGE_PREFIX_MAP.get(exchange.upper(), "")
return symbol + suffix if suffix else symbol
return ticker
# slash forex format (e.g. EUR/USD → EURUSD=X)
if "/" in t:
parts = t.upper().split("/")
if len(parts) == 2 and all(p.isalpha() and len(p) >= 2 for p in parts):
return parts[0] + parts[1] + "=X"
return t
u = t.upper()
# commodity / index aliases (checked first so EURUSD etc. in table win)
if u in _COMMODITY_ALIAS:
return _COMMODITY_ALIAS[u]
# bare 6-char alphabetic forex pairs not already in alias table (e.g. USDSEK → USDSEK=X)
if len(u) == 6 and u.isalpha():
_MAJORS = {"USD", "EUR", "GBP", "JPY", "CHF", "AUD", "CAD", "NZD", "SEK", "NOK"}
if u[3:] in _MAJORS or u[:3] in _MAJORS:
return u + "=X"
return t
def _fetch_live_prices(tickers: List[str], timeout: int = 20) -> Dict[str, Optional[float]]: