"""
AI analysis engine using OpenAI GPT-4o.
Tasks: news scoring, speech analysis, pattern evaluation, trade idea ranking.
"""
from openai import OpenAI
from typing import Optional, List, Dict, Any
import json
import os
import math
from datetime import datetime, timezone

_client: Optional[OpenAI] = None


def get_client() -> Optional[OpenAI]:
    global _client
    key = os.environ.get("OPENAI_API_KEY", "")
    if not key:
        return None
    if _client is None or _client.api_key != key:
        _client = OpenAI(api_key=key)
    return _client


def _chat(
    system: str,
    user: str,
    model: str = "gpt-4o-mini",
    json_mode: bool = True,
    max_tokens: int = 1500,
    log_meta: Optional[Dict] = None,
) -> Optional[Dict]:
    import time as _time
    client = get_client()
    if not client:
        return None
    kwargs: Dict[str, Any] = {
        "model": model,
        "messages": [{"role": "system", "content": system}, {"role": "user", "content": user}],
        "temperature": 0.2,
        "max_tokens": max_tokens,
    }
    if json_mode:
        kwargs["response_format"] = {"type": "json_object"}

    last_exc = None
    result: Optional[Dict] = None
    usage = None
    t0 = _time.time()

    for attempt in range(4):
        try:
            resp = client.chat.completions.create(**kwargs)
            content = resp.choices[0].message.content
            usage = resp.usage
            result = json.loads(content) if json_mode else {"text": content}
            break  # success
        except Exception as e:
            last_exc = e
            err_str = str(e)
            if "429" in err_str or "rate_limit" in err_str:
                import re as _re
                m = _re.search(r"try again in ([\d.]+)s", err_str)
                wait = float(m.group(1)) + 1.0 if m else 2 ** (attempt + 1) * 5.0
                _time.sleep(min(wait, 60.0))
                continue
            raise  # non-429 errors propagate immediately

    if result is None:
        raise last_exc

    # Persist AI call log when requested (non-blocking)
    if log_meta and log_meta.get("run_id"):
        try:
            from services.database import save_ai_call_log
            duration_ms = int((_time.time() - t0) * 1000)
            save_ai_call_log(
                run_id=log_meta["run_id"],
                call_type=log_meta.get("call_type", "unknown"),
                system_prompt=system,
                user_prompt=user,
                response_json=json.dumps(result, ensure_ascii=False),
                model=model,
                tokens_prompt=usage.prompt_tokens if usage else 0,
                tokens_completion=usage.completion_tokens if usage else 0,
                duration_ms=duration_ms,
                pattern_id=log_meta.get("pattern_id"),
                pattern_name=log_meta.get("pattern_name"),
            )
        except Exception:
            pass

    return result


# ── News / Article Analysis ───────────────────────────────────────────────────

SYSTEM_NEWS = """You are a senior geopolitical financial analyst specializing in options.
You analyze news and identify their potential impact on financial markets.
Respond ONLY in JSON according to the requested schema. Be precise and concise."""

def analyze_news_item(title: str, summary: str) -> Dict[str, Any]:
    """Classify and score a single news item with GPT."""
    user = f"""Analyze this geopolitical/economic article:
Title: {title}
Summary: {summary}

Return this JSON:
{{
  "category": "military|sanctions|elections|natural_disaster|health_crisis|resource_scarcity|trade_war|energy|political_speech|financial_crisis|general",
  "impact_score": <float 0.0-1.0>,
  "direction": "bullish|bearish|neutral|volatile",
  "affected_assets": {{
    "energy": <float -1.0 to 1.0 or null>,
    "metals": <float -1.0 to 1.0 or null>,
    "agriculture": <float -1.0 to 1.0 or null>,
    "indices": <float -1.0 to 1.0 or null>,
    "forex": <float -1.0 to 1.0 or null>
  }},
  "key_entities": [<max 5 key entities: countries, people, organizations>],
  "horizon": "immediate|days|weeks|months",
  "reasoning": "<1 sentence explaining the impact>"
}}"""
    result = _chat(SYSTEM_NEWS, user)
    if not result:
        return {"category": "general", "impact_score": 0.1, "direction": "neutral",
                "affected_assets": {}, "key_entities": [], "horizon": "days", "reasoning": "AI unavailable"}
    return result


# ── Speech / Text Analysis (Trump, Powell, etc.) ─────────────────────────────

SYSTEM_SPEECH = """You are a quantitative geopolitical analyst. You decode speeches and statements
from political/economic figures to identify options trading opportunities.
You specialize in: Trump speeches (tariffs, energy, dollar), Powell/Fed (rates),
geopolitical leaders (sanctions, wars, resources). Respond in JSON only."""

def analyze_speech(text: str, speaker: str = "") -> Dict[str, Any]:
    """Deep analysis of a speech/statement for trading signals."""
    user = f"""Analyze this statement{'from ' + speaker if speaker else ''} for trading signals:

---
{text[:3000]}
---

Return this JSON:
{{
  "speaker_identified": "<name if detected>",
  "tone": "hawkish|dovish|aggressive|conciliatory|ambiguous",
  "key_statements": [<list of 3-5 most impactful phrases/points>],
  "market_signals": [
    {{
      "asset": "<symbol or class>",
      "direction": "up|down|volatile",
      "magnitude": "low|medium|high|extreme",
      "reasoning": "<why>",
      "timeframe": "<immediate|1 week|1 month|3 months>"
    }}
  ],
  "options_opportunities": [
    {{
      "underlying": "<ETF or futures symbol>",
      "strategy": "Long Call|Long Put|Bull Call Spread|Bear Put Spread|Long Straddle",
      "strike_guidance": "<ATM|5% OTM|10% OTM>",
      "expiry_guidance": "<30d|60d|90d>",
      "rationale": "<why this strategy>",
      "confidence": <int 0-100>,
      "capital_1000eur": "<how to allocate €1000>"
    }}
  ],
  "risk_level": "low|medium|high|extreme",
  "geo_pattern_triggered": "<pattern name if applicable or null>",
  "summary": "<2-3 sentence summary for a trader>"
}}"""
    result = _chat(SYSTEM_SPEECH, user, model="gpt-4o")
    if not result:
        return {"error": "OpenAI unavailable — check API key"}
    return result


# ── Pattern Evaluation & Creation ────────────────────────────────────────────

SYSTEM_PATTERN = """You are an expert in quantitative geopolitical analysis and options trading.
You evaluate and improve geopolitical patterns for an algorithmic trading system.
Your evaluations are based on verifiable historical facts. Respond in JSON."""

def evaluate_pattern(pattern: Dict[str, Any]) -> Dict[str, Any]:
    """AI evaluation of a user-defined pattern."""
    user = f"""Evaluate this geopolitical trading pattern:

{json.dumps(pattern, ensure_ascii=False, indent=2)}

Return this JSON:
{{
  "quality_score": <int 0-100>,
  "validity": "excellent|good|fair|poor",
  "strengths": [<list of strengths>],
  "weaknesses": [<list of weaknesses or gaps>],
  "suggested_improvements": {{
    "additional_keywords": [<relevant missing keywords>],
    "additional_triggers": [<missing categories>],
    "probability_estimate": <float 0.0-1.0, your estimate>,
    "expected_move_revision": <float % or null if ok>,
    "horizon_revision": <int days or null if ok>
  }},
  "historical_validation": [
    {{
      "date": "<YYYY-MM-DD>",
      "event": "<real event that confirms the pattern>",
      "outcome": "<what happened in the markets>"
    }}
  ],
  "counter_scenarios": [<2-3 scenarios that would invalidate this pattern>],
  "overall_recommendation": "<general advice in 2-3 sentences>",
  "risk_warnings": [<specific risks for this pattern>]
}}"""
    result = _chat(SYSTEM_PATTERN, user, model="gpt-4o")
    if not result:
        return {"error": "OpenAI unavailable", "quality_score": 0}
    return result


def suggest_pattern_from_context(context: str) -> Dict[str, Any]:
    """AI creates a pattern structure from a free-text context description."""
    user = f"""A trader describes this geopolitical context and wants to create a trading pattern:

"{context}"

Generate a complete pattern in JSON:
{{
  "id": "P_USER_<3 random letters>",
  "name": "<concise pattern name>",
  "description": "<precise description of the mechanism>",
  "triggers": [<categories from: military, sanctions, elections, natural_disaster, health_crisis, resource_scarcity, trade_war, energy, political_speech, financial_crisis>],
  "keywords": [<10-15 English keywords to detect this pattern in news>],
  "historical_instances": [
    {{"date": "<YYYY-MM-DD>", "event": "<real event>", "outcome": "<observed market move>"}}
  ],
  "suggested_trades": [
    {{"strategy": "<strategy>", "underlying": "<symbol>", "rationale": "<why>"}}
  ],
  "asset_class": "<main class>",
  "expected_move_pct": <float>,
  "probability": <float 0.0-1.0>,
  "horizon_days": <int>,
  "confidence_in_pattern": <int 0-100>,
  "caveats": [<important caveats>]
}}"""
    result = _chat(SYSTEM_PATTERN, user, model="gpt-4o")
    if not result:
        return {"error": "OpenAI unavailable"}
    return result


# ── Top 10 Trade Ideas Ranking ────────────────────────────────────────────────

SYSTEM_RANKING = """You are a portfolio manager specializing in options.
You must select and rank the 10 best options trading opportunities
for a capital of ~€1000 with a 3-month horizon, integrating the current geopolitical context.
Prioritize: optimal risk/reward, options liquidity, clarity of catalyst. Respond in JSON."""

def rank_trade_ideas(
    pattern_matches: List[Dict],
    geo_score: Dict,
    recent_news: List[Dict],
    market_quotes: Dict,
) -> List[Dict[str, Any]]:
    """Generate and rank top 10 trade ideas using GPT-4o."""

    context = {
        "geo_risk_score": geo_score.get("score", 50),
        "geo_risk_level": geo_score.get("level", "medium"),
        "top_risks": geo_score.get("top_risks", []),
        "active_patterns": [
            {"name": p["name"], "similarity": p["similarity"],
             "asset_class": p["asset_class"], "expected_move": p["expected_move_pct"]}
            for p in pattern_matches[:5]
        ],
        "top_news": [
            {"title": n["title"], "category": n["category"], "impact": n["impact_score"]}
            for n in recent_news[:10]
        ],
    }

    user = f"""Current geopolitical and market context:
{json.dumps(context, ensure_ascii=False, indent=2)}

Generate the 10 best options trade ideas for €1000 / 3-month horizon.
Diversify across asset classes. Include at least: 2 energy, 1 metal, 1 agri, 2 indices/equities, 1 forex.

Return this JSON:
{{
  "ideas": [
    {{
      "rank": <1-10>,
      "title": "<short title>",
      "underlying": "<liquid ETF/futures symbol>",
      "strategy": "Long Call|Long Put|Bull Call Spread|Bear Put Spread|Long Straddle|Long Strangle",
      "asset_class": "<class>",
      "rationale": "<geopolitical reasoning in 2 sentences>",
      "geo_trigger": "<pattern or trigger event>",
      "strike_guidance": "<ATM|5% OTM|10% OTM>",
      "expiry_days": <int>,
      "expected_move_pct": <float>,
      "max_loss_eur": <float, max 1000>,
      "target_gain_eur": <float>,
      "confidence": <int 0-100>,
      "risk_level": "low|medium|high|extreme",
      "timing": "<enter now|wait for catalyst|after date X>",
      "invalidation": "<condition that invalidates the trade>"
    }}
  ],
  "portfolio_note": "<general note on allocating €1000 across these ideas>",
  "current_bias": "bullish|bearish|neutral|volatile",
  "key_risk": "<main risk to monitor>"
}}"""

    result = _chat(SYSTEM_RANKING, user, model="gpt-4o")
    if not result:
        return []
    return result.get("ideas", [])


# ── Pattern Scoring with Rich Context ────────────────────────────────────────

DEFAULT_ANALYSIS_TEMPLATE = """For each pattern, score each sub-pillar AND provide a 1-2 sentence comment in English.
Total score = exact sum of the 4 pillars (0-100).

PILLAR 1 — NEWS & GEO-CONTEXT  (30 pts max)
  1a. Geopolitical news        (0-12): relevance of recent events vs pattern keywords/triggers
  1b. Macro/economic news      (0-10): macro data, economic releases, monetary/fiscal policies
  1c. Signal volume & recency  (0-8) : number of independent sources, freshness (<48h = max), consistency

PILLAR 2 — ECONOMIC CALENDAR  (20 pts max)
  2a. Central banks            (0-10): upcoming FOMC/ECB/BoJ/BoE decisions, minutes, member speeches
  2b. Macro releases           (0-10): CPI, NFP, GDP, PMI, OPEC report — alignment with pattern

PILLAR 3 — PRICE SIGNALS  (35 pts max)
  3a. Rates & Bonds            (0-7): yield moves, yield curve, credit spreads
  3b. Energy & Commodities     (0-7): gold, oil, gas, copper, wheat — direction and momentum
  3c. Forex                    (0-7): USD index, EUR/USD, emerging pairs — consistency with pattern
  3d. Equities & Indices       (0-7): SPX, NDX, sector rotation, breadth, sentiment
  3e. Volatility & Vol Surface (0-7):
      - VIX regime (absolute level and trend)
      - SKEW Index: if >135 → expensive tails → PREFER spreads (budget = limited max loss)
        if <115 → cheap vol → consider straddles/strangles if binary catalyst
      - VVIX: if >100 → straddles/strangles too expensive → prefer directionals
      - OVX (>40) / GVZ (>22): elevated sector vol on the underlying → inflated premiums, use spreads
      - Surface regime: contango_calm (ideal to sell vol) vs backwardation_panic (exploded premiums)

PILLAR 4 — RISK / REWARD  (15 pts max)
  4a. R/R Asymmetry            (0-10): potential gain / premium paid / max loss ratio for ~€1000
  4b. Entry timing             (0-5) : quality of entry point vs historical analogues of the pattern

Rules: total score = 1a+1b+1c+2a+2b+3a+3b+3c+3d+3e+4a+4b; do not exceed maxima; comment each sub-pillar.

⚠️ ANTI-DIRECTIONAL BIAS RULE (MANDATORY):
- "expected_direction" indicates whether the pattern expects a rise or fall.
- "contra_signals" lists AI-scored news that CONTRADICTS this direction.
- "has_strong_contra": true = current context CANCELS or REVERSES the pattern thesis.
→ If has_strong_contra=true: total score ≤ 40/100; sub-pillar 1a geo ≤ 3/12.
→ If resolution=true in contra_signals (agreement/ceasefire resolving the trigger conflict): 1a geo = 0-2/12.
→ Always indicate in "summary" whether the signal is [SUPPORTING], [NEUTRAL] or [CONTRA]."""

SYSTEM_SCORER = """You are a senior portfolio manager specializing in geopolitical options.
You analyze geopolitical patterns with their enriched market context (news, prices, IV) to identify
the best options trading opportunities (~€1000, 3-month horizon).
You are rigorous, quantitative and pragmatic. Respond ONLY in valid JSON.

SCORE DISTRIBUTION RULE (MANDATORY):
- Scores must reflect a real distribution: some patterns are strong (70+), others weak (30-)
- It is FORBIDDEN to score all patterns the same or close to 50
- Base your scores on: relevant_news_count (active catalysts), macro_regime.asset_class_bias, has_strong_contra, horizon vs imminent catalysts
- A pattern with no relevant recent news (relevant_news_count=0) cannot exceed 35/100
- A pattern with has_strong_contra=true cannot exceed 40/100
- A pattern with 3+ relevant news AND favorable macro can reach 70-85/100"""


def _build_specialist_context_block(asset_classes: set) -> str:
    """Build SPECIALIST DESK CONTEXT block including COT, forward curves, surprise scores."""
    try:
        from services.database import get_asset_class_config, get_desk_reports, get_latest_cot_data, get_latest_forward_curves
        from datetime import datetime as _dt

        # Fetch COT and forward curve data once
        try:
            all_cot = get_latest_cot_data()
        except Exception:
            all_cot = []
        try:
            all_curves = get_latest_forward_curves()
        except Exception:
            all_curves = []

        blocks = []
        for ac in sorted(asset_classes):
            cfg = get_asset_class_config(ac)
            if not cfg:
                continue
            reports = get_desk_reports(ac)
            lines = [f"\n## SPECIALIST DESK — {cfg['display_name'].upper()} {cfg.get('icon','')}"]

            if cfg.get("fundamentals"):
                lines.append(f"Key drivers: {cfg['fundamentals'][:300]}")

            # Macro sensitivity
            sensitivities = cfg.get("macro_sensitivity") or []
            if sensitivities:
                sens_str = " | ".join(
                    f"{s.get('regime','?')} -> {s.get('effect','?')}"
                    for s in sensitivities[:4]
                )
                lines.append(f"Regime sensitivity: {sens_str}")

            # Price thresholds
            thresh = cfg.get("price_delta_thresholds") or {}
            if thresh:
                lines.append(
                    f"Price thresholds: significant_week={thresh.get('significant_week','?')}% "
                    f"extreme_week={thresh.get('extreme_week','?')}%"
                )

            # COT positioning for this asset class
            cot_ac = [c for c in all_cot if c.get("asset_class") == ac]
            if cot_ac:
                lines.append("COT Positioning (Money Managers net, % of OI):")
                for c in cot_ac[:5]:
                    net_pct = c.get("net_pct_oi", 0)
                    chg = c.get("change_net", 0)
                    direction = "LONG" if net_pct > 5 else "SHORT" if net_pct < -5 else "NEUTRAL"
                    chg_str = f" (chg {'+' if chg >= 0 else ''}{chg:,} wk)" if chg else ""
                    lines.append(f"  - {c['commodity']}: net={net_pct:+.1f}% OI [{direction}]{chg_str} -- {c.get('report_date','?')}")

            # Forward curves for this asset class
            curves_ac = [c for c in all_curves if c.get("asset_class") == ac]
            if curves_ac:
                lines.append("Forward Curve Structure (spot vs +3M):")
                for curve in curves_ac[:5]:
                    struct = curve.get("structure", "unknown")
                    slope = curve.get("slope_pct")
                    slope_str = f" ({slope:+.1f}%)" if slope is not None else ""
                    struct_label = struct.upper()
                    lines.append(f"  - {curve['asset']}: {struct_label}{slope_str}")

            # Recent reports with surprise scores
            today = _dt.utcnow().strftime("%Y-%m-%d")
            upcoming = sorted(
                [r for r in reports if r.get("next_date") and r["next_date"] >= today],
                key=lambda r: r["next_date"]
            )[:5]
            if upcoming:
                lines.append("Upcoming reports:")
                for r in upcoming:
                    line = f"  - {r['name']} ({r['source']}) -- {r['next_date']} [{r['cadence']}]"
                    if r.get("importance", 1) >= 3:
                        line += " ***"
                    if r.get("consensus_estimate") is not None:
                        line += f" [consensus: {r['consensus_estimate']}]"
                    lines.append(line)

            # Recent releases with surprise scores
            recent_releases = sorted(
                [r for r in reports if r.get("last_date") and r.get("surprise_score") is not None],
                key=lambda r: r.get("last_date", ""),
                reverse=True
            )[:3]
            if recent_releases:
                lines.append("Recent releases (surprise index = actual - consensus):")
                for r in recent_releases:
                    surprise = r["surprise_score"]
                    label = r.get("text_sentiment_label", "")
                    label_str = f" [{label}]" if label else ""
                    sign = "+" if surprise >= 0 else ""
                    lines.append(f"  - {r['name']}: surprise={sign}{surprise:.2f}{label_str} -- {r.get('last_date','?')}")

            blocks.append("\n".join(lines))
        return "\n".join(blocks) + "\n" if blocks else ""
    except Exception:
        return ""


def score_report_text(text: str, report_name: str = "", desk: str = "forex") -> Dict:
    """Score a report/statement excerpt. Returns score (-1 to +1), label, summary, key_phrases."""
    if desk in ("forex", "bonds"):
        dimension = "monetary policy stance: -1 = very dovish (rate cuts expected), 0 = neutral, +1 = very hawkish (rate hikes expected)"
        examples = "Hawkish: 'inflation remains persistent', 'further tightening may be appropriate'. Dovish: 'inflation returning to target', 'downside risks dominate', 'easing cycle beginning'."
    elif desk in ("energy", "metals", "agri"):
        dimension = "commodity market outlook: -1 = very bearish (oversupply, demand weakness), 0 = neutral, +1 = very bullish (deficit, demand surge)"
        examples = "Bullish: 'output cuts extended', 'crop failure', 'stockpiles at multi-year low'. Bearish: 'surplus expected', 'demand revision lower', 'record production'."
    else:
        dimension = "market sentiment: -1 = very bearish, 0 = neutral, +1 = very bullish"
        examples = ""

    prompt = f"""You are a macro trading analyst. Analyze the following text and return ONLY valid JSON.

Report: {report_name or 'Unknown'}
Desk: {desk}
Score dimension: {dimension}
{examples}

TEXT TO ANALYZE:
{text[:3000]}

Return JSON with exactly these fields:
{{
  "score": <float from -1.0 to +1.0>,
  "label": <"very_hawkish" | "hawkish" | "neutral" | "dovish" | "very_dovish"> (for forex/bonds)
         or <"very_bullish" | "bullish" | "neutral" | "bearish" | "very_bearish"> (for others),
  "summary": <1-2 sentence summary of the key signal>,
  "key_phrases": [<up to 5 most significant phrases that drove the score>],
  "confidence": <"high" | "medium" | "low">
}}"""

    try:
        import openai as _oai
        from services.database import get_config as _gc
        api_key = _gc("openai_api_key") or ""
        if not api_key:
            return {"score": 0, "label": "neutral", "summary": "No API key configured", "key_phrases": [], "confidence": "low"}
        client = _oai.OpenAI(api_key=api_key)
        response = client.chat.completions.create(
            model="gpt-4o-mini",
            messages=[{"role": "user", "content": prompt}],
            temperature=0.1,
            response_format={"type": "json_object"},
        )
        import json
        return json.loads(response.choices[0].message.content)
    except Exception as e:
        return {"score": 0, "label": "neutral", "summary": f"Error: {str(e)}", "key_phrases": [], "confidence": "low"}


def score_patterns_with_context(
    patterns: List[Dict],
    recent_news: List[Dict],
    quotes_by_class: Dict,
    geo_score: Dict,
    template: str = None,
    top_n: int = 10,
    category_filter: str = None,
    macro_regime: Optional[Dict] = None,
    portfolio_lessons: Optional[Dict] = None,
    iv_context: str = "",
    risk_context: str = "",
    cycle_meta: Optional[Dict] = None,
    tech_indicators_block: str = "",
    fred_block: str = "",
    price_discovery_block: str = "",
    portfolio_context_block: str = "",
    institutional_block: str = "",
    run_id: str = "",
) -> List[Dict[str, Any]]:
    """Score all patterns with rich context (news, prices, IV, risk clusters) using GPT-4o."""
    if not get_client():
        return []

    scoring_template = template or DEFAULT_ANALYSIS_TEMPLATE

    # Flatten quotes to symbol -> data dict for fast lookup
    quotes_flat: Dict[str, Dict] = {}
    for cls_quotes in quotes_by_class.values():
        for q in cls_quotes:
            quotes_flat[q.get("symbol", "")] = q

    # Build per-pattern context blocks
    pattern_blocks = []
    for pat in patterns:
        if category_filter and category_filter != "all":
            if pat.get("asset_class") != category_filter:
                # also check suggested trades
                trade_classes = [t.get("asset_class", "") for t in pat.get("suggested_trades", [])]
                if category_filter not in trade_classes:
                    continue

        # Filter news relevant to this pattern
        keywords = [kw.lower() for kw in pat.get("keywords", [])]
        relevant_news = []
        for n in recent_news[:50]:
            text = (n.get("title", "") + " " + n.get("summary", "")).lower()
            if any(kw in text for kw in keywords):
                relevant_news.append({
                    "title": n.get("title", ""),
                    "date": n.get("published", "")[:10],
                    "source": n.get("source", ""),
                    "impact": n.get("impact_score", 0),
                })
                if len(relevant_news) >= 4:
                    break

        # Market data for each suggested underlying
        market_data = {}
        for trade in pat.get("suggested_trades", []):
            sym = trade.get("underlying", "")
            if sym and sym in quotes_flat:
                q = quotes_flat[sym]
                from services.data_fetcher import compute_historical_iv
                try:
                    iv = compute_historical_iv(sym)
                except Exception:
                    iv = None
                market_data[sym] = {
                    "price": q.get("price"),
                    "change_1d_pct": q.get("change_pct"),
                    "iv_pct": round(iv * 100, 1) if iv else None,
                    "name": q.get("name", sym),
                }

        # Detect contra-signals: AI-scored news that contradicts this pattern's direction
        expected_up = pat.get("expected_move_pct", 0) > 0
        asset_cls = pat.get("asset_class", "")
        _dir_field = {"energy": "ai_dir_energy", "metals": "ai_dir_metals"}.get(asset_cls, "ai_dir_indices")

        contra_signals = []
        for n in recent_news[:25]:
            if not n.get("ai_scored"):
                continue
            ai_dir = n.get(_dir_field, "neutral")
            impact = float(n.get("impact_score") or 0)
            is_contra = (expected_up and ai_dir == "bearish") or (not expected_up and ai_dir == "bullish")
            if is_contra and impact >= 0.35:
                contra_signals.append({
                    "title": (n.get("title") or "")[:100],
                    "impact": round(impact, 2),
                    "direction": ai_dir,
                    "resolution": n.get("ai_resolution", False),
                    "insight": n.get("ai_insight", ""),
                })
        has_strong_contra = any(c["impact"] >= 0.55 or c.get("resolution") for c in contra_signals)

        # Macro regime context for this pattern
        macro_ctx = None
        if macro_regime:
            scenarios = macro_regime.get("scenarios", {})
            gauges = macro_regime.get("gauges", {})
            dominant = scenarios.get("dominant", "incertain")
            asset_bias = scenarios.get("asset_bias", {})
            pat_cls = pat.get("asset_class", "")
            bias_for_class = asset_bias.get(dominant, {}).get(pat_cls, "neutral") if dominant != "incertain" else "neutral"
            macro_ctx = {
                "dominant_scenario": dominant,
                "scenario_scores": scenarios.get("scores", {}),
                "asset_class_bias": bias_for_class,
                # ── Signaux historiques (phase 1) ─────────────────────────────
                "vix": gauges.get("vix", {}).get("value"),
                "yield_slope_pct": gauges.get("slope_10y3m", {}).get("value"),
                "gold_copper_ratio": gauges.get("gold_copper_ratio", {}).get("value"),
                "brent_1d_pct": gauges.get("brent", {}).get("change_pct"),
                "spx_vs_200d_pct": gauges.get("spx_vs_200d", {}).get("value"),
                # ── Surface de volatilité (phase 2) ──────────────────────────
                # SKEW >135 = queues chères → spreads; <115 = vol bon marché → straddles
                "skew_index": gauges.get("skew", {}).get("value"),
                # VVIX >100 = straddles/strangles trop chers → préférer directionnels
                "vvix": gauges.get("vvix", {}).get("value"),
                # Vol sectorielle spécifique au sous-jacent
                "oil_vol_ovx": gauges.get("ovx", {}).get("value"),
                "gold_vol_gvz": gauges.get("gvz", {}).get("value"),
                # Régime composite: contango_calm|normal|tail_risk_elevated|backwardation_panic|complacency_hedged
                "vol_surface_regime": gauges.get("vol_surface_regime", {}).get("note"),
                # ── Rotation sectorielle (phase 2) ────────────────────────────
                # Positif = tech > défensifs = risk-on; négatif = rotation défensive
                "tech_vs_staples_pct": gauges.get("xlk_xlp_momentum", {}).get("value"),
                # Négatif = banques < marché = stress économique anticipé
                "financials_vs_spx_pct": gauges.get("xlf_spx_ratio", {}).get("value"),
                # ── Global / Marchés émergents (phase 2) ─────────────────────
                "em_equity_1d_pct": gauges.get("eem", {}).get("change_pct"),
                "em_bonds_1d_pct": gauges.get("emb", {}).get("change_pct"),
                "china_equity_1d_pct": gauges.get("fxi", {}).get("change_pct"),
                # Ratio EM vs SPX: positif = croissance globale; négatif = fuite vers US
                "em_vs_us_divergence_pct": gauges.get("eem_spx_ratio", {}).get("value"),
                # ── Carry / Risk-off (phase 2) ────────────────────────────────
                # Négatif = JPY s'apprécie = carry unwind = risk-off global
                "usdjpy_1d_pct": gauges.get("usdjpy", {}).get("change_pct"),
                # ── Long bonds / Qualité (phase 2) ────────────────────────────
                # Positif = flight to quality = risk-off; négatif = taux longs remontent
                "long_bond_tlt_1d_pct": gauges.get("tlt", {}).get("change_pct"),
                # ── Métaux (phase 2) ──────────────────────────────────────────
                # Ratio Ag/Au: >0.016 = argent > or = risk-on industriel; <0.012 = defensive metals
                "silver_gold_ratio": gauges.get("silver_gold_ratio", {}).get("value"),
            }

        pattern_blocks.append({
            "id": pat.get("id", pat.get("pattern_id", "")),
            "name": pat.get("name", ""),
            "description": pat.get("description", ""),
            "asset_class": pat.get("asset_class", ""),
            "triggers": pat.get("triggers", []),
            "historical_instances": pat.get("historical_instances", [])[:2],
            "suggested_trades": pat.get("suggested_trades", []),
            "expected_move_pct": pat.get("expected_move_pct", 0),
            "expected_direction": "up" if expected_up else "down",
            "horizon_days": pat.get("horizon_days", 90),
            "relevant_news_count": len(relevant_news),
            "relevant_news": relevant_news,
            "market_data": market_data,
            "contra_signals": contra_signals[:3],
            "has_strong_contra": has_strong_contra,
            "macro_regime": macro_ctx,
        })

    if not pattern_blocks:
        return []

    macro_section = ""
    if macro_regime:
        sc = macro_regime.get("scenarios", {})
        gauges_g = macro_regime.get("gauges", {})
        dom = sc.get("dominant", "incertain")
        sc_scores = sc.get("scores", {})
        # Extract key new signals for global macro summary
        _skew  = gauges_g.get("skew", {}).get("value")
        _vvix  = gauges_g.get("vvix", {}).get("value")
        _ovx   = gauges_g.get("ovx", {}).get("value")
        _vol_r = gauges_g.get("vol_surface_regime", {}).get("note", "normal")
        _tech_st = gauges_g.get("xlk_xlp_momentum", {}).get("value")
        _xlf_spx = gauges_g.get("xlf_spx_ratio", {}).get("value")
        _eem_c = gauges_g.get("eem", {}).get("change_pct")
        _usdjpy_c = gauges_g.get("usdjpy", {}).get("change_pct")
        _tlt_c = gauges_g.get("tlt", {}).get("change_pct")
        _sgr   = gauges_g.get("silver_gold_ratio", {}).get("value")

        def _fmt(v, unit="", decimals=1):
            return f"{v:.{decimals}f}{unit}" if v is not None else "n/d"

        macro_section = f"""
CURRENT MACRO REGIME (50 counters — 29 tickers + 9 derived metrics):
- Dominant scenario: {dom.upper()} | Scores: {json.dumps(sc_scores, ensure_ascii=False)}

VOLATILITY SURFACE (pillar 3e — direct impact on options strategy):
- SKEW Index: {_fmt(_skew, '', 0)} | Vol regime: {_vol_r} | VVIX: {_fmt(_vvix, '', 0)} | OVX: {_fmt(_ovx, '', 0)}
  → SKEW >135 = expensive tails → SPREADS (no straddles). VVIX >100 = inflated premiums → directionals.
  → {_vol_r} = {"contango calm: ideal for cheap spreads" if _vol_r == "contango_calm" else "backwardation/panic: very expensive options, premiums to sell" if _vol_r == "backwardation_panic" else "elevated tail risk: tail protection = favor defined spreads" if _vol_r == "tail_risk_elevated" else "normal environment"}

SECTOR ROTATION & RISK (pillars 3d + 3e):
- Tech vs Defensives (XLK-XLP): {_fmt(_tech_st, '%pts')} | Financials vs S&P: {_fmt(_xlf_spx, '%pts')}
  → {"Strong sector risk-on" if _tech_st and _tech_st > 0.5 else "Defensive rotation ⚠️" if _tech_st and _tech_st < -0.5 else "Sector neutral"}
  → {"Healthy banks = no recession" if _xlf_spx and _xlf_spx > 0.3 else "Anticipated banking stress ⚠️" if _xlf_spx and _xlf_spx < -0.3 else ""}

GLOBAL / CARRY / QUALITY (pillars 3a + 3d):
- EM Equities D+1: {_fmt(_eem_c, '%')} | USD/JPY D+1: {_fmt(_usdjpy_c, '%')} | TLT (20Y bonds) D+1: {_fmt(_tlt_c, '%')}
- Ag/Au Ratio: {_fmt(_sgr, '', 5)}
  → {"EM outperforms = global growth" if _eem_c and _eem_c > 0.5 else "EM stress = flight to US ⚠️" if _eem_c and _eem_c < -1.0 else ""}
  → {"JPY appreciating = carry unwind = risk-off ⚠️" if _usdjpy_c and _usdjpy_c < -0.8 else "Weak JPY = risk-on carry active" if _usdjpy_c and _usdjpy_c > 0.5 else ""}
  → {"TLT rising = flight to quality = long bonds in demand" if _tlt_c and _tlt_c > 0.3 else "TLT falling = long rates rising = inflation/risk-on" if _tlt_c and _tlt_c < -0.3 else ""}

Scoring instructions:
- Integrate regime into 3a (rates: slope+TLT), 3b (energy+OVX), 3d (indices+XLF+EM), 3e (SKEW+VVIX+regime)
- "macro_regime.asset_class_bias" in each pattern → increase/decrease 3b or 3d
- "macro_regime.skew_index" + "vol_surface_regime" → direct influence on strategy choice in "recommended_trade"
- Indicate in "summary": [GOLDILOCKS|STAGFLATION|RECESSION|DISINFLATION|CRISIS] + [SUPPORTING|NEUTRAL|CONTRA]
"""

    # Temporal context block for scoring
    _cm = cycle_meta or {}
    _delta_min_sc = _cm.get("delta_minutes", 180)
    _calib_sc = _cm.get("calibration_label", "")
    temporal_section_sc = ""
    if _cm:
        # Apply decay to news used in scoring
        news_decayed_sc = apply_news_decay(recent_news)
        _partitioned_sc = partition_news_by_age(news_decayed_sc, _delta_min_sc)
        _inter_sc = _partitioned_sc["inter_cycle"]
        temporal_section_sc = (
            f"\nCYCLE TEMPORAL CONTEXT:\n"
            f"- Last cycle: {_delta_min_sc:.0f} minutes ago | {_calib_sc}\n"
            f"- INTER-CYCLE NEWS (not yet priced, {len(_inter_sc)} articles): "
            + " | ".join(n.get("title", "")[:60] for n in _inter_sc[:3]) + "\n"
            f"⚠️ Account for the delay since the last cycle to assess whether news is already priced in.\n"
        )

    _tech_sc_section = f"\n{tech_indicators_block}\n" if tech_indicators_block else ""
    _fred_sc_section = f"\n{fred_block}\n" if fred_block else ""
    _pd_sc_section = f"\n{price_discovery_block}\n" if price_discovery_block else ""
    _portfolio_sc_section = f"\n{portfolio_context_block}\n" if portfolio_context_block else ""
    _inst_sc_section = f"\n{institutional_block}\n" if institutional_block else ""

    # Trade mandate block from cycle_meta
    _budget = _cm.get("trade_budget_eur", 5000)
    _h_min  = _cm.get("preferred_horizon_min", 30)
    _h_max  = _cm.get("preferred_horizon_max", 180)
    _max_pos = round(_budget * 0.25)
    _trade_mandate_sc = (
        f"\n## INVESTOR TRADE MANDATE\n"
        f"- Available capital budget: €{_budget:,.0f}\n"
        f"- Preferred horizon: {_h_min}–{_h_max} days\n"
        f"- Max capital per position: €{_max_pos:,.0f} (25% of budget)\n"
        f"⚠️ Patterns whose horizon_days is outside [{_h_min}, {_h_max}] should receive a penalty"
        f" in the R/R pillar (poor timing fit). Size trade suggestions within the budget cap.\n"
    )

    user = f"""GLOBAL CONTEXT:
- Geopolitical risk score: {geo_score.get('score', 50)}/100 ({geo_score.get('level', 'medium')})
- Top risks: {geo_score.get('top_risks', [])}
{temporal_section_sc}
{_trade_mandate_sc}
{macro_section}
{_fred_sc_section}
{_pd_sc_section}
{_tech_sc_section}
{_inst_sc_section}
{_portfolio_sc_section}
SCORING TEMPLATE:
{scoring_template}

PATTERNS TO SCORE ({len(pattern_blocks)} patterns):
{json.dumps(pattern_blocks, ensure_ascii=False, indent=2)}

For each of the {len(pattern_blocks)} patterns, score each sub-pillar + add a comment in English.
The "score" field = exact sum of all sub-pillars.

⚠️ TRADE RANKINGS (MANDATORY): A pattern can have multiple suggested_trades (e.g.: Long Call WTI + Bull Spread XLE).
These trades do NOT all deserve the same score. For each pattern, fill "trade_rankings" by:
- ranking trades from best (rank 1) to worst
- assigning a "score_delta" between -20 and +20 (e.g.: +10 for the best, 0 for average, -8 for the worst)
- explaining in 1 sentence why each trade is above/below the pattern average
- the sum of score_delta should be ≈ 0 (trades offset each other relative to the pattern score)

Return ONLY this valid JSON:
{{
  "scored_patterns": [
    {{
      "pattern_id": "<id>",
      "score": <int 0-100, exact sum of the 4 pillars>,
      "confidence": <int 0-100>,
      "buckets": [
        {{
          "id": "actualites",
          "label": "News & Geo-context",
          "score": <0-30>,
          "max": 30,
          "comment": "<1-2 sentence summary>",
          "subs": [
            {{"id": "geo",  "label": "Geopolitical news",  "score": <0-12>, "max": 12, "comment": "<1-2 sentences>"}},
            {{"id": "eco",  "label": "Macro/economic news", "score": <0-10>, "max": 10, "comment": "<1-2 sentences>"}},
            {{"id": "flux", "label": "Volume & recency",    "score": <0-8>,  "max": 8,  "comment": "<1-2 sentences>"}}
          ]
        }},
        {{
          "id": "calendrier",
          "label": "Economic Calendar",
          "score": <0-20>,
          "max": 20,
          "comment": "<summary>",
          "subs": [
            {{"id": "banques",    "label": "Central banks",     "score": <0-10>, "max": 10, "comment": "<1-2 sentences>"}},
            {{"id": "macro_cal",  "label": "Macro releases",    "score": <0-10>, "max": 10, "comment": "<1-2 sentences>"}}
          ]
        }},
        {{
          "id": "prix",
          "label": "Price Signals",
          "score": <0-35>,
          "max": 35,
          "comment": "<summary>",
          "subs": [
            {{"id": "taux",      "label": "Rates & Bonds",       "score": <0-7>, "max": 7, "comment": "<1-2 sentences>"}},
            {{"id": "energie",   "label": "Energy & Commodities", "score": <0-7>, "max": 7, "comment": "<1-2 sentences>"}},
            {{"id": "forex_sig", "label": "Forex",                "score": <0-7>, "max": 7, "comment": "<1-2 sentences>"}},
            {{"id": "actions",   "label": "Equities & Indices",   "score": <0-7>, "max": 7, "comment": "<1-2 sentences>"}},
            {{"id": "vix",       "label": "Volatility (VIX/IV)",  "score": <0-7>, "max": 7, "comment": "<1-2 sentences>"}}
          ]
        }},
        {{
          "id": "rr",
          "label": "Risk / Reward",
          "score": <0-15>,
          "max": 15,
          "comment": "<summary>",
          "subs": [
            {{"id": "asymetrie",  "label": "R/R Asymmetry",   "score": <0-10>, "max": 10, "comment": "<1-2 sentences>"}},
            {{"id": "timing_rr", "label": "Entry timing",     "score": <0-5>,  "max": 5,  "comment": "<1-2 sentences>"}}
          ]
        }}
      ],
      "key_catalyst": "<main catalyst in 1 sentence>",
      "recommended_trade": {{
        "underlying": "<symbol>",
        "strategy": "<Long Call|Long Put|Bull Call Spread|Bear Put Spread|Long Straddle>",
        "strike_guidance": "<ATM|5% OTM|...>",
        "expiry_days": <int>,
        "rationale": "<why this trade now, max 2 sentences>",
        "target_gain_eur": <float>,
        "max_loss_eur": <float max 1000>,
        "timing_note": "<enter now|wait for X|monitor Y>",
        "invalidation": "<condition that invalidates>"
      }},
      "asset_class": "<class>",
      "geo_trigger": "<pattern name>",
      "summary": "<1 sentence summary>",
      "trade_rankings": [
        {{
          "underlying": "<ticker>",
          "strategy": "<strategy>",
          "rank": <1-N>,
          "score_delta": <int -20 to +20, positive if this trade is above the pattern average>,
          "rationale": "<1 sentence: why this trade deserves more/less than others in the same pattern>",
          "expected_move_pct": <float, EXPECTED OPTION RETURN in % if thesis confirmed, leverage included. Long Call ATM: 80-200%, Spread: 40-120%, Straddle: 60-180%. Re-evaluate against current context.>
        }}
      ]
    }}
  ],
  "analysis_meta": {{
    "patterns_analyzed": <int>,
    "top_bias": "bullish|bearish|neutral|volatile",
    "key_risk": "<main risk>"
  }}
}}"""

    # Extract the return-schema portion from `user` so batches use the identical full schema
    # (includes bucket id/label/max definitions that GPT-4o needs to populate correctly)
    _split_key = "Return ONLY this valid JSON:\n"
    _parts = user.split(_split_key, 1)
    _return_schema = _parts[1] if len(_parts) == 2 else user

    # Build lessons feedback block for the scorer
    lessons_header = ""
    if portfolio_lessons:
        lessons = portfolio_lessons.get("key_lessons") or []
        super_ctx = portfolio_lessons.get("super_context", "")
        priorities = portfolio_lessons.get("strategic_priorities", [])
        mistakes = portfolio_lessons.get("recurring_mistakes", [])
        super_scoring_block = ""
        if super_ctx:
            super_scoring_block = f"""
🧠 SUPER CONTEXTE (base de raisonnement accumulée) :
{super_ctx[:400]}
Priorités: {' | '.join(str(p) for p in priorities[:2])}
Erreurs à éviter: {' | '.join(str(m) for m in mistakes[:2])}
"""
        lessons_header = f"""
{super_scoring_block}
PERFORMANCE FEEDBACK (report from {portfolio_lessons.get('created_at','?')[:10]}):
Overall summary: {portfolio_lessons.get('headline', '')[:150]}
Blind spots detected: {portfolio_lessons.get('blind_spots', '')[:150]}
Priorities: {portfolio_lessons.get('next_cycle_priorities', '')[:150]}
Lessons: {' | '.join(str(l)[:80] for l in lessons[:3])}
⚠️ Take the Super Context and these lessons into account to adjust scores and pillar comments.

"""

    # Build specialist-desk blocks for asset classes present in this pattern set
    _all_pattern_asset_classes = {
        p.get("asset_class", "") for p in patterns if p.get("asset_class")
    }
    specialist_block = _build_specialist_context_block(_all_pattern_asset_classes)

    # Build the per-batch prompt template (static parts)
    prompt_header = f"""GLOBAL CONTEXT:
- Geopolitical risk score: {geo_score.get('score', 50)}/100 ({geo_score.get('level', 'medium')})
- Top risks: {geo_score.get('top_risks', [])}
{macro_section}{lessons_header}{iv_context}
{risk_context}{specialist_block}
SCORING TEMPLATE:
{scoring_template}

"""

    import logging as _logging
    _scorer_log = _logging.getLogger(__name__)

    def _score_batch(batch: list) -> list:
        ids = [p.get("id", "?") for p in batch]
        _scorer_log.info(f"[Scorer] Batch of {len(batch)} patterns: {ids}")
        batch_user = (
            prompt_header
            + f"PATTERNS TO SCORE ({len(batch)} patterns):\n"
            + json.dumps(batch, ensure_ascii=False, indent=2)
            + f"\n\n⚠️ MANDATORY: You must return EXACTLY {len(batch)} objects in scored_patterns — one for EACH pattern in the list, WITHOUT EXCEPTION. Even if a pattern has score=0 (currently irrelevant), it must appear in the list.\n\n"
            + f"For each of the {len(batch)} patterns, score each sub-pillar + add a comment in English.\n"
            + "The \"score\" field = exact sum of all sub-pillars.\n\n"
            + "🎯 MANDATORY CALIBRATION — Scores MUST be differentiated, never all the same:\n"
            + "  score 75-100 = strong active catalyst, clear signal, excellent timing\n"
            + "  score 55-74  = moderate signal, favorable but uncertain context\n"
            + "  score 35-54  = neutral, little signal, waiting\n"
            + "  score 15-34  = contra-signal, unfavorable regime, high risk\n"
            + "  score 0-14   = no relevance in current context\n"
            + "⛔ Do not score all patterns at 50 — that is a calibration failure.\n"
            + "   Use relevant_news_count, macro_regime.asset_class_bias, and contra_signals to truly differentiate.\n\n"
            + "⚠️ TRADE RANKINGS (MANDATORY): A pattern can have multiple suggested_trades (e.g.: Long Call WTI + Bull Spread XLE).\n"
            + "These trades do NOT all deserve the same score. For each pattern, fill \"trade_rankings\" by:\n"
            + "- ranking trades from best (rank 1) to worst\n"
            + "- assigning a \"score_delta\" between -20 and +20 (e.g.: +10 for the best, 0 for average, -8 for the worst)\n"
            + "- explaining in 1 sentence why each trade is above/below the pattern average\n"
            + "- the sum of score_delta should be ≈ 0 (trades offset each other relative to the pattern score)\n\n"
            + "Return ONLY this valid JSON:\n"
            + _return_schema
        )
        try:
            res = _chat(
                SYSTEM_SCORER, batch_user, model="gpt-4o", json_mode=True, max_tokens=12000,
                log_meta={"run_id": run_id, "call_type": "score_batch", "pattern_name": f"batch:{','.join(ids[:3])}"} if run_id else None,
            )
        except Exception as e:
            _scorer_log.error(f"[Scorer] GPT-4o call failed for batch {ids}: {e}")
            res = None
        scored = res.get("scored_patterns", []) if res else []
        _scorer_log.info(f"[Scorer] Batch returned {len(scored)} scored_patterns (expected {len(batch)})")
        # Guarantee every pattern in the batch has an entry — prevents silent drops on truncation
        scored_ids = {str(s.get("pattern_id", "")) for s in scored}
        for p in batch:
            if str(p.get("id", "")) not in scored_ids:
                _scorer_log.warning(f"[Scorer] Pattern id='{p.get('id')}' name='{p.get('name')}' missing from GPT-4o response — adding stub score=0")
                scored.append({
                    "pattern_id": p["id"],
                    "score": 0,
                    "confidence": 0,
                    "buckets": [],
                    "key_catalyst": "Not relevant in current context",
                    "recommended_trade": {},
                    "asset_class": p.get("asset_class", ""),
                    "geo_trigger": p.get("name", ""),
                    "summary": "[CONTRA] Pattern not relevant in current context.",
                    "trade_rankings": [],
                })
        return scored

    BATCH_SIZE = 4  # 4 patterns × ~800 tokens output = ~3200 tokens, safely within gpt-4o limits

    # Score batches sequentially (max_workers=1) — parallel workers both retry on 429
    # simultaneously, defeating the sleep backoff. Sequential ensures the retry window
    # is fully cleared before the next batch fires.
    from concurrent.futures import ThreadPoolExecutor, as_completed
    batches = [pattern_blocks[i:i+BATCH_SIZE] for i in range(0, len(pattern_blocks), BATCH_SIZE)]
    _scorer_log.info(f"[Scorer] Scoring {len(pattern_blocks)} patterns in {len(batches)} batches of max {BATCH_SIZE}")
    all_scored = []
    with ThreadPoolExecutor(max_workers=1) as executor:
        futures = [executor.submit(_score_batch, b) for b in batches]
        for future in as_completed(futures):
            try:
                results = future.result()
                all_scored.extend(results)
            except Exception as e:
                _scorer_log.error(f"[Scorer] Batch future raised: {e}")

    # Hardcoded max values per bucket id — used as fallback if GPT-4o omits the max field
    _BUCKET_MAX = {"actualites": 30, "calendrier": 20, "prix": 35, "rr": 15}
    _SUB_MAX = {
        "geo": 12, "eco": 10, "flux": 8,
        "banques": 10, "macro_cal": 10,
        "taux": 7, "energie": 7, "forex_sig": 7, "actions": 7, "vix": 7,
        "asymetrie": 10, "timing_rr": 5,
    }

    # Normalize bucket scores and recompute total from sub-buckets
    for p in all_scored:
        if p.get("buckets"):
            total = 0
            for b in p["buckets"]:
                bid = b.get("id", "")
                b_max = int(b.get("max") or _BUCKET_MAX.get(bid, 30))
                sub_sum = 0
                for sub in b.get("subs", []):
                    sid = sub.get("id", "")
                    s_max = int(sub.get("max") or _SUB_MAX.get(sid, 10))
                    sub["score"] = max(0, min(int(sub.get("score") or 0), s_max))
                    sub["max"] = s_max  # ensure max is always set for frontend display
                    sub_sum += sub["score"]
                b["score"] = max(0, min(int(b.get("score") or sub_sum), b_max))
                b["max"] = b_max  # ensure max is always set for frontend display
                total += b["score"]
            p["score"] = min(total, 100)

    all_scored.sort(key=lambda x: x.get("score", 0), reverse=True)

    # ── Phase convergence: compute cross-pattern conviction bonuses ────────────
    all_scored, _conv_block = _compute_convergence(all_scored, patterns)
    all_scored.sort(key=lambda x: x.get("conviction_score", x.get("score", 0)), reverse=True)

    return all_scored[:top_n]


# ── Pattern convergence ───────────────────────────────────────────────────────

PATTERN_CATEGORIES = {
    "géopolitique":     "Armed conflicts, sanctions, elections, military alliances, regional tensions",
    "macro_monétaire":  "Inflation, Fed/ECB policy rates, dollar index, yield curve, QE/QT",
    "technique":        "RSI, Bollinger, momentum, support/resistance breakouts, chart patterns",
    "commodités_supply":"OPEC, oil/gas inventories, harvests, mine disruptions, supply chain logistics",
    "risk_off":         "Natural disasters, banking crises, pandemics, volatility shocks",
    "flux_saisonnier":  "COT flows, seasonality, end-of-quarter rebalancing, options expiry",
    "géo_économique":   "Supply chains, protectionism, de-dollarization, reshoring",
    "crédit_stress":    "HY spreads, sovereign CDS, banking stress, default risk",
}


def classify_patterns_batch(patterns: List[Dict]) -> List[Dict]:
    """Classify unclassified patterns via GPT-4o-mini → [{id, category, signal_direction}]."""
    if not get_client() or not patterns:
        return []

    compact = [
        {
            "id": p.get("id"),
            "name": p.get("name", ""),
            "description": (p.get("description") or "")[:200],
            "triggers": (p.get("triggers") or [])[:3],
            "asset_class": p.get("asset_class", ""),
        }
        for p in patterns
    ]
    cats_desc = "\n".join(f"- {c}: {d}" for c, d in PATTERN_CATEGORIES.items())
    system = "You are a geopolitical financial pattern classifier. Respond ONLY in valid JSON."
    user = f"""Classify each pattern according to ONE of these categories:
{cats_desc}

signal_direction:
- "bullish": anticipates a rise in the underlying
- "bearish": anticipates a fall in the underlying
- "volatility": anticipates a move without clear direction (straddle, vol play)
- "neutral": uncertain direction / range strategy

Patterns to classify:
{json.dumps(compact, ensure_ascii=False)}

Return ONLY this JSON:
{{"classifications": [{{"id": "<id>", "category": "<category>", "signal_direction": "<direction>"}}]}}"""

    try:
        res = _chat(system, user, model="gpt-4o-mini", json_mode=True, max_tokens=2000)
        return res.get("classifications", []) if res else []
    except Exception as _e:
        import logging as _log2
        _log2.getLogger(__name__).warning(f"[Classify] Batch classification failed: {_e}")
        return []


def _compute_convergence(
    scored_patterns: List[Dict],
    original_patterns: List[Dict],
) -> tuple:
    """
    Group scored patterns by (underlying, signal_direction).
    Apply conviction_bonus = min(20, +5 per additional agreeing pattern).
    Returns (enriched_list, convergence_summary_block_str).
    """
    from collections import defaultdict

    pat_meta = {str(p.get("id", "")): p for p in original_patterns}

    # Collect underlyings + direction per pattern
    def _pat_underlyings(sp: Dict):
        underlyings = set()
        rec = sp.get("recommended_trade") or {}
        if rec.get("underlying"):
            underlyings.add(rec["underlying"])
        for tr in (sp.get("trade_rankings") or []):
            if tr.get("underlying"):
                underlyings.add(tr["underlying"])
        return underlyings

    # Build (underlying, direction) → list of pattern entries
    groups: Dict = defaultdict(list)
    for sp in scored_patterns:
        pid = str(sp.get("pattern_id", ""))
        orig = pat_meta.get(pid, {})
        direction = (orig.get("signal_direction") or "neutral").lower()
        if direction not in ("bullish", "bearish", "volatility", "neutral"):
            direction = "neutral"
        category = orig.get("category") or ""
        name = (orig.get("name") or sp.get("geo_trigger") or "")[:50]
        score = sp.get("score", 0)

        for und in _pat_underlyings(sp):
            groups[(und, direction)].append({
                "pid": pid, "name": name, "category": category, "score": score,
            })

    # Enrich each scored pattern
    enriched = []
    for sp in scored_patterns:
        pid = str(sp.get("pattern_id", ""))
        orig = pat_meta.get(pid, {})
        direction = (orig.get("signal_direction") or "neutral").lower()
        if direction not in ("bullish", "bearish", "volatility", "neutral"):
            direction = "neutral"

        best_count, best_und, best_partners = 0, "", []
        for und in _pat_underlyings(sp):
            group = groups.get((und, direction), [])
            others = [x for x in group if x["pid"] != pid]
            if len(others) > best_count:
                best_count, best_und, best_partners = len(others), und, others

        conviction_bonus = min(20, 5 * best_count)
        conviction_score = min(100, sp.get("score", 0) + conviction_bonus)
        enriched.append({
            **sp,
            "conviction_score": conviction_score,
            "conviction_bonus": conviction_bonus,
            "convergence_count": best_count,
            "convergence_underlying": best_und,
            "convergence_partners": [
                {"name": p["name"], "category": p["category"], "score": p["score"]}
                for p in best_partners[:5]
            ],
        })

    # Build convergence block for injection into NEXT suggestion prompt
    conv_lines = []
    for (und, direction), pats in groups.items():
        if len(pats) >= 2:
            avg_sc = sum(p["score"] for p in pats) / len(pats)
            cats = ", ".join(sorted({p["category"] for p in pats if p["category"]}))
            conv_lines.append((len(pats), avg_sc, und, direction, pats, cats))
    conv_lines.sort(key=lambda x: (-x[0], -x[1]))

    if not conv_lines:
        conv_block = ""
    else:
        _dir_sym = {"bullish": "▲", "bearish": "▼", "volatility": "⟷", "neutral": "↔"}
        lines = [
            "\n## 🎯 SIGNAL CONVERGENCE — High-conviction underlyings across patterns",
            "These underlyings are targeted by multiple active patterns in the SAME direction:",
        ]
        for count, avg_sc, und, direction, pats, cats in conv_lines[:8]:
            sym = _dir_sym.get(direction, "↔")
            pat_names = " | ".join(p["name"][:30] for p in pats[:4])
            lines.append(f"  {sym} {und} {direction.upper()}: {count} converging patterns (avg score {avg_sc:.0f}/100)")
            lines.append(f"     Categories: {cats or 'N/A'} | Patterns: {pat_names}")
        lines += [
            "",
            "⚠️ These convergences signal STRONG collective conviction — favor complementary patterns",
            "on these underlyings or flag a contra-signal if context reverses.",
        ]
        conv_block = "\n".join(lines)

    return enriched, conv_block


# ── Suggest new patterns from live market context ─────────────────────────────

# ── Temporal news utilities ───────────────────────────────────────────────────

# Halflife by news category (hours) — after how long the impact is halved
_NEWS_HALFLIFE_HOURS: Dict[str, float] = {
    "economic":    4.0,   # CPI, NFP, GDP, FOMC → priced in very fast
    "geopolitical": 48.0, # conflict, sanction, deal → slow digestion
    "corporate":   12.0,  # earnings, deal, merger → medium digestion
    "default":     24.0,
}

_ECONOMIC_KEYWORDS = {"cpi", "inflation", "nfp", "jobs", "fomc", "fed", "gdp", "pmi", "ecb", "bce", "earnings", "taux"}
_GEO_KEYWORDS = {"war", "guerre", "sanction", "conflit", "ceasefire", "accord", "treaty", "nuclear", "missile", "invasion", "trump", "israel", "ukraine", "russia", "iran", "china"}


def _news_category(title: str) -> str:
    t = title.lower()
    if any(k in t for k in _ECONOMIC_KEYWORDS):
        return "economic"
    if any(k in t for k in _GEO_KEYWORDS):
        return "geopolitical"
    return "corporate"


def _article_age_hours(article: Dict) -> float:
    """Return age of article in hours from now. Falls back to 6h if no timestamp."""
    for field in ("published", "published_at", "fetched_at"):
        raw = article.get(field, "")
        if not raw:
            continue
        try:
            # Strip timezone info for naive comparison
            raw_clean = str(raw).replace("Z", "+00:00")
            dt = datetime.fromisoformat(raw_clean)
            if dt.tzinfo is not None:
                dt = dt.replace(tzinfo=None) - dt.utcoffset()
            age = (datetime.utcnow() - dt).total_seconds() / 3600
            return max(0.0, age)
        except Exception:
            continue
    return 6.0  # default: assume 6h old


def apply_news_decay(news: List[Dict]) -> List[Dict]:
    """
    Add `decayed_score` to each article: impact_score × exp(-age / halflife).
    Does NOT modify original list — returns new list with added field.
    """
    result = []
    for n in news:
        cat = _news_category(n.get("title", ""))
        halflife = _NEWS_HALFLIFE_HOURS.get(cat, _NEWS_HALFLIFE_HOURS["default"])
        age_h = _article_age_hours(n)
        raw_score = float(n.get("impact_score") or 0)
        decayed = raw_score * math.exp(-age_h / halflife)
        result.append({**n, "decayed_score": round(decayed, 4), "age_hours": round(age_h, 1), "news_category": cat})
    return result


def partition_news_by_age(news: List[Dict], delta_minutes: float) -> Dict[str, List[Dict]]:
    """
    Split news into temporal buckets relative to cycle delta:
      - inter_cycle  : published within delta_minutes → potentially NOT YET priced in
      - recent_24h   : published 24h → delta_minutes ago → partially priced in
      - older        : > 24h → considered already priced in by the market
    """
    delta_hours = delta_minutes / 60.0
    inter, recent, older = [], [], []
    for n in news:
        age_h = n.get("age_hours", _article_age_hours(n))
        if age_h <= delta_hours:
            inter.append(n)
        elif age_h <= 24.0:
            recent.append(n)
        else:
            older.append(n)
    # Sort each bucket by decayed_score desc
    key = lambda x: -(x.get("decayed_score") or x.get("impact_score") or 0)
    return {
        "inter_cycle": sorted(inter, key=key),
        "recent_24h":  sorted(recent, key=key),
        "older":       sorted(older, key=key),
    }


def _build_temporal_news_block(partitioned: Dict[str, List], cycle_meta: Dict) -> str:
    """Build the news section of the IA prompt with temporal framing + market session context."""
    delta_min = cycle_meta.get("delta_minutes", 180)
    calib = cycle_meta.get("calibration_label", "")
    day_of_week = cycle_meta.get("day_of_week", "")
    is_weekend = cycle_meta.get("is_weekend", False)
    market_session = cycle_meta.get("market_session", "")
    market_note = cycle_meta.get("market_note", "")

    def _fmt_news(articles: List[Dict], limit: int = 6) -> str:
        lines = []
        for n in articles[:limit]:
            score = n.get("decayed_score") or n.get("impact_score") or 0
            age = n.get("age_hours", "?")
            lines.append(
                f"  • [{n.get('source','')}] {n.get('title','')[:110]}"
                f" (impact={score:.2f}, age={age:.0f}h)"
            )
        return "\n".join(lines) if lines else "  (none)"

    inter = partitioned.get("inter_cycle", [])
    recent = partitioned.get("recent_24h", [])
    older  = partitioned.get("older", [])

    # Market session banner
    if is_weekend:
        session_banner = (
            f"\n## 📅 MARKET STATUS — {day_of_week.upper()}\n"
            f"⚠️ WEEKEND — Markets CLOSED. {market_note}\n"
            "→ Patterns can be identified but CANNOT be executed before Monday morning.\n"
            "→ Use this cycle to prepare limit orders for Monday's open.\n"
            "→ Prices shown are FRIDAY CLOSE prices — do not interpret intraday moves.\n"
            "→ Implied volatility is artificially elevated this weekend (market maker weekend premium) — IVR displayed is overstated.\n"
        )
    elif market_session in ("pre_market", "after_hours", "overnight"):
        session_banner = (
            f"\n## 📅 MARKET STATUS — {day_of_week} {market_session.upper()}\n"
            f"{market_note}\n"
            "→ Prices may not reflect the regular session — reduced liquidity.\n"
        )
    else:
        session_banner = f"\n## 📅 MARKET STATUS — {day_of_week} | {market_session.upper()}\n{market_note}\n" if day_of_week else ""

    block = f"""
{session_banner}
## ⏱ CYCLE TEMPORAL CONTEXT
- Last cycle: {delta_min:.0f} minutes ago
- Calibration: {calib}

## 📍 INTER-CYCLE NEWS — last {delta_min:.0f}min (POTENTIALLY NOT YET priced in)
{_fmt_news(inter, 6)}
*→ These are the news most likely to create unpriced opportunities.*

## 📰 RECENT NEWS — 24h (partially priced, decay applied)
{_fmt_news(recent, 5)}

## 📦 OLDER NEWS — >24h (considered already priced in by the market)
{_fmt_news(older, 3)}
"""
    return block


def build_institutional_block(days: int = 7) -> str:
    """Build a concise institutional reports block for injection into AI prompts."""
    try:
        from services.database import get_conn
        conn = get_conn()
        try:
            from datetime import datetime as _dt, timedelta as _td
            cutoff = (_dt.utcnow() - _td(days=days)).strftime("%Y-%m-%d")
            rows = conn.execute(
                "SELECT report_type, report_date, key_points_json, trading_implications, "
                "signal_energy, signal_metals, signal_indices, signal_forex, importance "
                "FROM institutional_reports WHERE report_date >= ? ORDER BY importance DESC, report_date DESC LIMIT 12",
                (cutoff,),
            ).fetchall()
        finally:
            conn.close()

        if not rows:
            return ""

        import json as _json
        lines = [f"## INSTITUTIONAL REPORTS (COT · EIA · Earnings · VX · Central Banks · Sentiment — last {days}d)"]
        for r in rows:
            rtype = r["report_type"].upper()
            rdate = r["report_date"]
            importance_str = "★★★" if r["importance"] == 3 else "★★" if r["importance"] == 2 else "★"
            signals = (
                f"Energy={r['signal_energy']} | Metals={r['signal_metals']} | "
                f"Indices={r['signal_indices']} | Forex={r['signal_forex']}"
            )
            lines.append(f"\n### {rtype} {rdate} {importance_str} — {signals}")
            try:
                kps = _json.loads(r["key_points_json"] or "[]")
                for kp in kps[:4]:
                    lines.append(f"  • {kp}")
            except Exception:
                pass
            if r["trading_implications"]:
                lines.append(f"  → Implications: {r['trading_implications'][:200]}")

        return "\n".join(lines)
    except Exception as _e:
        return ""


def suggest_patterns_from_market_context(
    news: List[Dict],
    quotes_by_class: Dict[str, List[Dict]],
    calendar: List[Dict],
    macro_regime: Optional[Dict] = None,
    geo_score: Optional[Dict] = None,
    portfolio_lessons: Optional[Dict] = None,
    reliability_map: Optional[Dict] = None,
    iv_context: str = "",
    cycle_meta: Optional[Dict] = None,
    tech_indicators_block: str = "",
    fred_block: str = "",
    price_discovery_block: str = "",
    portfolio_context_block: str = "",
    convergence_block: str = "",
    run_id: str = "",
) -> List[Dict]:
    """Ask GPT-4o to propose new patterns based on current geo/market + macro regime context."""
    _cycle_meta = cycle_meta or {}
    _delta_min = _cycle_meta.get("delta_minutes", 180.0)

    # Apply temporal decay + partition news by age relative to cycle delta
    news_decayed = apply_news_decay(news)
    _partitioned = partition_news_by_age(news_decayed, _delta_min)

    # For backward-compat blocks that still use a flat list: use decayed inter+recent
    top_news = sorted(
        _partitioned["inter_cycle"] + _partitioned["recent_24h"],
        key=lambda x: -(x.get("decayed_score") or 0)
    )[:12]
    news_block = "\n".join([
        f"- [{n.get('source','')}] {n.get('title','')} (impact_decay {n.get('decayed_score',0):.2f})"
        for n in top_news
    ])

    market_lines = []
    for cls, qs in quotes_by_class.items():
        for q in qs[:3]:
            if q.get("price"):
                market_lines.append(f"  {cls} | {q.get('name', q['symbol'])}: {q['price']} ({q.get('change_pct', 0):+.1f}%)")
    market_block = "\n".join(market_lines)

    cal_block = "\n".join([
        f"- {e.get('date','')} [{e.get('importance','')}] {e.get('title','')}"
        for e in (calendar or [])[:8]
    ])

    # Macro regime block
    macro_block = ""
    if macro_regime:
        sc = macro_regime.get("scenarios", {})
        gauges = macro_regime.get("gauges", {})
        dominant = sc.get("dominant", "incertain")
        scores = sc.get("scores", {})
        asset_bias = sc.get("asset_bias", {}).get(dominant, {})
        reasons = sc.get("reasons", {}).get(dominant, [])
        vix = gauges.get("vix", {}).get("value")
        slope = gauges.get("slope_10y3m", {}).get("value")
        gold_cu = gauges.get("gold_copper_ratio", {}).get("value")
        spx_200 = gauges.get("spx_vs_200d", {}).get("value")
        brent_chg = gauges.get("brent", {}).get("change_pct")
        bias_lines = "\n".join([f"  - {cls}: {b}" for cls, b in asset_bias.items()])
        brent_str = f"{brent_chg:+.2f}%" if brent_chg is not None else "N/A"
        macro_block = f"""
## Current macro regime (30 institutional counters)
- Dominant scenario: {dominant.upper()} | Scores: {json.dumps(scores, ensure_ascii=False)}
- Key signals: {', '.join(reasons[:4])}
- Counters: VIX={vix} | Slope 10Y-3M={slope}% | Gold/Copper={gold_cu} | SPX vs 200d={spx_200}% | Brent D-1={brent_str}
- Bias by asset class (scenario {dominant.upper()}):
{bias_lines}

⚠️ CONSTRAINT: Proposed patterns must be CONSISTENT with this macro regime.
- Favor patterns whose asset_class has a "bullish" or "bullish+" bias in the current regime.
- Avoid bullish patterns on "bearish" or "bearish+" classes unless an exceptional geopolitical catalyst justifies it.
- Each pattern must explain in "macro_fit" why it is compatible (or in tension) with the {dominant.upper()} regime.
"""

    geo_block = ""
    if geo_score:
        geo_block = f"\n## Global geopolitical risk\n- Score: {geo_score.get('score', 50)}/100 ({geo_score.get('level', 'medium')})\n- Top risks: {', '.join(str(r) for r in geo_score.get('top_risks', [])[:3])}\n"

    lessons_block = ""
    if portfolio_lessons:
        super_ctx = portfolio_lessons.get("super_context", "")
        priorities = portfolio_lessons.get("strategic_priorities", [])
        mistakes = portfolio_lessons.get("recurring_mistakes", [])
        lessons = portfolio_lessons.get("key_lessons") or []
        super_block = ""
        if super_ctx:
            super_block = f"""
## 🧠 SUPER CONTEXT — Accumulated reasoning base
{super_ctx[:600]}
Strategic priorities: {' | '.join(str(p) for p in priorities[:3])}
Recurring mistakes to avoid: {' | '.join(str(m) for m in mistakes[:3])}
"""
        lessons_block = f"""
{super_block}
## ⚡ PERFORMANCE FEEDBACK — previous cycles (report from {portfolio_lessons.get('created_at','?')[:10]})
Overall performance: {portfolio_lessons.get('headline', '')}
Why gains: {portfolio_lessons.get('winners_analysis', '')[:200]}
Why losses: {portfolio_lessons.get('losers_analysis', '')[:200]}
Blind spots detected: {portfolio_lessons.get('blind_spots', '')[:150]}
Priorities identified: {portfolio_lessons.get('next_cycle_priorities', '')[:200]}
Key lessons:
{chr(10).join(f'  - {l}' for l in lessons[:4])}

⚠️ INSTRUCTION: Take this performance feedback and the Super Context into account to propose BETTER TARGETED patterns.
Avoid mistakes identified in losses. Favor types of theses that have worked.
"""

    reliability_block = ""
    if reliability_map:
        top_reliable = sorted(reliability_map.values(), key=lambda r: -r["reliability_score"])[:5]
        bottom_reliable = [r for r in sorted(reliability_map.values(), key=lambda r: r["reliability_score"]) if r["trade_count"] >= 3][:3]
        lines = []
        for r in top_reliable:
            lines.append(
                f"  ✅ {r['pattern_name']}: WR={r['win_rate_pct']}% | {r['trade_count']} trades | "
                f"avgPnL={r['avg_pnl_pct']:+.1f}% | fiabilité={r['reliability_score']:.2f}"
            )
        for r in bottom_reliable:
            lines.append(
                f"  ❌ {r['pattern_name']}: WR={r['win_rate_pct']}% | {r['trade_count']} trades | "
                f"avgPnL={r['avg_pnl_pct']:+.1f}% → À ÉVITER ou reformuler"
            )
        if lines:
            reliability_block = (
                "\n## 📊 HISTORICAL PATTERN RELIABILITY (mature trades only)\n"
                + "\n".join(lines)
                + "\n⚠️ Draw inspiration from reliable patterns. Avoid reproducing the bottom-ranked patterns.\n"
            )

    iv_block = ""
    if iv_context:
        iv_block = f"""
{iv_context}

## ⚠️ STRICT IV → STRATEGY RULES (MANDATORY for each suggested_trade)
Strategy choice must account for the COST of implied volatility (IVR = IV Rank 52 weeks):

| IVR          | ALLOWED strategy                                         | FORBIDDEN strategy           |
|--------------|----------------------------------------------------------|------------------------------|
| < 30% (cheap)| Long Call, Long Put, Long Straddle                       | Iron Condor, Short Strangle  |
| 30–60% (mod) | Bull Call Spread, Bear Put Spread                        | Long Straddle, naked Strangle|
| 60–80% (exp) | Bull/Bear Spread, Cash-Secured Put                       | naked Long Call/Put          |
| > 80% (peak) | Iron Condor, Short Strangle, Covered Call, Cash-Secured Put | ANY naked long option     |

Additional rules:
- High put skew (> 5 pts) → Long Put too expensive → prefer Bear Put Spread
- Term structure backwardation → do not sell vol (seller trapped)
- If IVR unknown → use debit spreads by default (vol cost-neutral)
- Long Straddle ONLY if IVR < 25% AND catalyst clearly identified

⚠️ PROHIBITION: Never suggest "Long Call" or "Long Put" (naked) if IVR > 60% on this ticker.
"""

    # Build temporal news block (partitioned by cycle delta)
    temporal_news_block = _build_temporal_news_block(_partitioned, _cycle_meta) if _cycle_meta else (
        "## Current geopolitical news (sorted by impact)\n" + news_block
    )

    tech_block_section = f"\n{tech_indicators_block}\n" if tech_indicators_block else ""
    fred_section = f"\n{fred_block}\n" if fred_block else ""
    pd_section = f"\n{price_discovery_block}\n" if price_discovery_block else ""
    portfolio_section = f"\n{portfolio_context_block}\n" if portfolio_context_block else ""
    convergence_section = f"\n{convergence_block}\n" if convergence_block else ""

    # Inject all specialist desks — helps AI generate patterns across all asset classes
    _all_desks = {"forex", "metals", "agri", "energy", "indices", "crypto", "bonds"}
    specialist_block_suggest = _build_specialist_context_block(_all_desks)
    specialist_section = f"\n{specialist_block_suggest}\n" if specialist_block_suggest else ""

    _sg_budget  = _cycle_meta.get("trade_budget_eur", 5000)
    _sg_h_min   = _cycle_meta.get("preferred_horizon_min", 30)
    _sg_h_max   = _cycle_meta.get("preferred_horizon_max", 180)
    _sg_max_pos = round(_sg_budget * 0.25)
    _suggest_mandate = (
        f"\n## INVESTOR TRADE MANDATE\n"
        f"- Available capital budget: €{_sg_budget:,.0f} | Max per position: €{_sg_max_pos:,.0f}\n"
        f"- Target horizon: {_sg_h_min}–{_sg_h_max} days\n"
        f"⚠️ Only propose patterns whose horizon_days is within [{_sg_h_min}, {_sg_h_max}]."
        f" Patterns outside this window will be rejected. Size each trade so capital ≤ €{_sg_max_pos:,.0f}.\n"
    )

    user = f"""You are a senior geopolitical and financial strategist, expert in options.
{_suggest_mandate}
{macro_block}{geo_block}{lessons_block}{reliability_block}{iv_block}{specialist_section}
{temporal_news_block}
## Market prices (D-1 change)
{market_block}
{fred_section}
{pd_section}
{tech_block_section}
{portfolio_section}
{convergence_section}
## Upcoming economic calendar
{cal_block}

Analyzing this landscape, propose 4 to 6 NEW geopolitical patterns that are emerging RIGHT NOW and deserve monitoring for options opportunities.

Do not repeat well-known classic patterns (Middle East Oil Spike, Gold Flight to Safety, etc.) — propose patterns SPECIFIC to the current context, consistent with the macro regime.

IMPORTANT — STRATEGY: Strictly follow the IV→strategy rules defined above if IVR is known.

IMPORTANT — expected_move_pct FIELD:
This field represents the EXPECTED OPTION RETURN in % (leverage included), NOT the move in the underlying.
Reason: if the underlying moves X% in the expected direction, how much does the option gain in %?
- Long Call ATM (delta ~0.5, 30-90d): underlying +5% → option +60 to +150%
- Long Call OTM (delta ~0.25): underlying +8% → option +100 to +300%
- Bull Call Spread: underlying +5% → spread +50 to +120% (capped)
- Long Straddle: ±10% move → option +80 to +200%
Realistic examples: Long Call energy on strong catalyst → 80-200%. Defensive spread → 40-100%.

Return ONLY this JSON:
{{
  "patterns": [
    {{
      "name": "<short and impactful name>",
      "description": "<geopolitical mechanism → market impact, 2-3 sentences>",
      "macro_fit": "<1-2 sentences: why this pattern is consistent or in tension with the current macro regime, and what geopolitical catalyst justifies it>",
      "triggers": ["<trigger1>", "<trigger2>"],
      "keywords": ["<kw1>", "<kw2>", "<kw3>"],
      "asset_class": "<energy|metals|agriculture|indices|equities|forex>",
      "category": "<géopolitique|macro_monétaire|technique|commodités_supply|risk_off|flux_saisonnier|géo_économique|crédit_stress>",
      "signal_direction": "<bullish|bearish|volatility|neutral>",
      "expected_move_pct": <float, AVERAGE OPTION RETURN in % for this pattern, leverage included. Typically 50-300%.>,
      "probability": <float 0-1>,
      "horizon_days": <int>,
      "counter_thesis": "<1-2 sentences: main adverse scenario that would invalidate this pattern — be specific (e.g.: unexpected peace deal, CPI data below 3%, etc.)>",
      "invalidation_trigger": "<precise measurable event to monitor — e.g.: 'oil price < $70/b for 3 consecutive days', 'FOMC hawkish surprise', 'Russia-Ukraine ceasefire'>",
      "invalidation_probability": <float 0-1, probability that this invalidation trigger materializes within the horizon>,
      "suggested_trades": [
        {{
          "strategy": "<Long Call|Long Put|Bull Call Spread|Bear Put Spread|Long Straddle|Iron Condor|Short Strangle|Cash-Secured Put|Covered Call — follow IVR rules>",
          "underlying": "<Yahoo Finance ticker>",
          "rationale": "<why this trade in this macro+geo+vol context>",
          "asset_class": "<energy|metals|agriculture|indices|equities|forex>",
          "expected_move_pct": <float, OPTION RETURN in % for THIS trade if thesis confirmed. Long Call: 80-250%, Spread: 40-120%, Straddle: 60-180%.>
        }},
        {{
          "strategy": "<another strategy following IVR rules>",
          "underlying": "<Yahoo Finance ticker>",
          "rationale": "<rationale>",
          "asset_class": "<energy|metals|agriculture|indices|equities|forex>",
          "expected_move_pct": <float, expected option return in % for this specific trade>
        }}
      ]
    }}
  ]
}}"""

    result = _chat(
        SYSTEM_SCORER, user, model="gpt-4o", json_mode=True, max_tokens=4000,
        log_meta={"run_id": run_id, "call_type": "suggest"} if run_id else None,
    )
    if not result:
        return []
    return result.get("patterns", [])


# ── AI news batch scoring: impact magnitude + directional signals ─────────────

def ai_score_news_batch(news_items: List[Dict]) -> List[Dict]:
    """Score news items with AI: accurate impact + per-asset directional signal.
    Adds ai_dir_energy/metals/indices, ai_resolution, ai_insight, ai_scored fields.
    Called before pattern scoring so contra-signals can be detected.
    """
    if not get_client() or not news_items:
        return news_items

    to_score = [n for n in news_items[:20] if not n.get("ai_scored")]
    if not to_score:
        return news_items

    compact = [
        {"i": idx, "t": n.get("title", ""), "s": (n.get("summary", "") or "")[:150]}
        for idx, n in enumerate(to_score)
    ]

    user = f"""Score these geopolitical news items for TRUE market impact.

CRITICAL: Resolution events (peace deals, ceasefires, truces, agreements ending conflicts)
have HIGH impact (0.7-0.9) but are BEARISH for oil/energy and BEARISH for safe-haven patterns.

Items: {json.dumps(compact, ensure_ascii=False)}

For each item return:
- impact_score: 0.0-1.0 real magnitude (resolution = high, sports/culture = low)
- dir_energy: "bullish"|"bearish"|"neutral" (for oil/gas/energy)
- dir_metals: "bullish"|"bearish"|"neutral" (for gold/silver/copper)
- dir_indices: "bullish"|"bearish"|"neutral" (risk-on vs risk-off)
- resolution: true if this is a de-escalation/peace/deal that REDUCES a prior conflict
- insight: "<1 short English sentence on main market effect>"

JSON: {{"items": [{{"i":<int>,"impact_score":<float>,"dir_energy":"...","dir_metals":"...","dir_indices":"...","resolution":<bool>,"insight":"..."}}]}}"""

    result = _chat(SYSTEM_NEWS, user, model="gpt-4o-mini", json_mode=True, max_tokens=2000)
    if not result:
        return news_items

    scored_map = {s["i"]: s for s in result.get("items", [])}
    for idx, n in enumerate(to_score):
        s = scored_map.get(idx)
        if s:
            n["impact_score"] = max(0.0, min(1.0, float(s.get("impact_score") or n.get("impact_score", 0.1))))
            n["ai_dir_energy"] = s.get("dir_energy", "neutral")
            n["ai_dir_metals"] = s.get("dir_metals", "neutral")
            n["ai_dir_indices"] = s.get("dir_indices", "neutral")
            n["ai_resolution"] = bool(s.get("resolution", False))
            n["ai_insight"] = s.get("insight", "")
            n["ai_scored"] = True

    return news_items


# ── Holistic geopolitical risk score (AI judgment, cycle-frozen) ──────────────

def ai_score_geo_risk(news: List[Dict], algo_score: Dict, log_meta: Optional[Dict] = None) -> Dict:
    """Holistic 0-100 geopolitical risk assessment by the AI. Takes the
    already AI-scored news list plus compute_geo_risk_score()'s deterministic
    category-weighted formula as a reference baseline (not a value to just
    copy) — lets the AI actually judge severity/de-escalation/context instead
    of a rigid weighted sum, while staying anchored enough not to swing
    wildly cycle to cycle. Called once per auto-cycle (services/auto_cycle.py
    Step 1); the result is frozen in DB until the next cycle runs."""
    if not get_client() or not news:
        return {
            "score": algo_score.get("score", 0), "level": algo_score.get("level", "low"),
            "rationale": "AI unavailable — algorithmic score used as-is.",
            "top_risks": [],
        }

    top_news = sorted(news, key=lambda n: -(n.get("impact_score") or 0))[:15]
    compact = [
        {"title": n.get("title", ""), "category": n.get("category", ""), "impact": round(n.get("impact_score") or 0, 2)}
        for n in top_news
    ]

    user = f"""Assess the current overall geopolitical risk level for financial markets, on a scale of 0 to 100.

Reference algorithmic score (category-weighted formula, indicative only — use your own judgment, don't just copy it): {algo_score.get('score')}/100 ({algo_score.get('level')}).
Breakdown by category: {json.dumps(algo_score.get('breakdown', {}), ensure_ascii=False)}

Top news (sorted by impact):
{json.dumps(compact, ensure_ascii=False)}

Instructions:
- A high score must reflect a REAL, current risk to markets (active military escalation, major trade rupture, political crisis...), not just a volume of news.
- A de-escalation/resolution event should LOWER the score even if its raw impact is high.
- Don't mechanically anchor on the algorithmic score if the real context (headlines) justifies a different one.
- rationale: 2-3 sentences in English precisely explaining why this score, citing the most determining events.
- top_risks: the 3 to 5 most determining items for this score (short titles).

JSON: {{"score": <0-100 float>, "level": "low"|"medium"|"high"|"extreme", "rationale": "...", "top_risks": ["...", ...]}}"""

    result = _chat(
        "You are a senior geopolitical analyst assessing overall market risk, not event by event.",
        user, model="gpt-4o", json_mode=True, max_tokens=700, log_meta=log_meta,
    )
    if not result:
        return {
            "score": algo_score.get("score", 0), "level": algo_score.get("level", "low"),
            "rationale": "AI error — algorithmic score used as-is.",
            "top_risks": [],
        }

    score = max(0.0, min(100.0, float(result.get("score", algo_score.get("score", 0)))))
    return {
        "score": round(score, 1),
        "level": result.get("level") or algo_score.get("level", "low"),
        "rationale": result.get("rationale", ""),
        "top_risks": result.get("top_risks", []) or [],
    }


# ── Re-score news batch with AI ───────────────────────────────────────────────

def ai_rescore_news(news_items: List[Dict]) -> List[Dict]:
    """Batch re-score news items using AI for better classification."""
    if not get_client() or not news_items:
        return news_items
    rescored = []
    for item in news_items[:20]:
        try:
            ai = analyze_news_item(item.get("title", ""), item.get("summary", ""))
            item["ai_category"] = ai.get("category", item.get("category"))
            item["ai_impact"] = ai.get("impact_score", item.get("impact_score"))
            item["ai_direction"] = ai.get("direction", "neutral")
            item["ai_reasoning"] = ai.get("reasoning", "")
            item["ai_entities"] = ai.get("key_entities", [])
            if ai.get("affected_assets"):
                item["asset_impacts"] = ai["affected_assets"]
        except Exception:
            pass
        rescored.append(item)
    return rescored
