OpenFin/backend/routers/knowledge.py

from fastapi import APIRouter, HTTPException
from pydantic import BaseModel
from typing import Any, Dict, List, Optional
import json
import os

from services.database import (
    get_kb_entries, get_all_kb_entries, save_kb_entry, update_kb_entry_status,
    get_latest_reasoning_state, get_reasoning_history, get_reasoning_state_by_id,
    save_reasoning_state, list_ai_reports, get_mtm_trades_with_traces, _trade_maturity,
    delete_reasoning_state, delete_kb_entry,
)

router = APIRouter(prefix="/api/knowledge", tags=["knowledge"])


def _build_synthesis_prompt(reports: List[Dict], trades: List[Dict], kb_entries: List[Dict]):
    """Build the GPT-4o synthesis prompt from all accumulated data."""
    from datetime import date as _date
    now_str = __import__("datetime").datetime.utcnow().strftime("%Y-%m-%d %H:%M")

    # Portfolio reports summary
    reports_block = ""
    for r in reports[:10]:
        rpt = r.get("report") or {}
        stats = r.get("stats") or {}
        date_str = r.get("created_at", "")[:16]
        headline = rpt.get("headline", "")
        winners_a = rpt.get("winners_analysis", "")
        losers_a = rpt.get("losers_analysis", "")
        lessons = rpt.get("key_lessons", [])
        blind = rpt.get("blind_spots", "")
        next_p = rpt.get("next_cycle_priorities", "")
        timing = rpt.get("timing_note", "")
        lessons_str = " | ".join(lessons) if isinstance(lessons, list) else str(lessons)
        reports_block += f"""
--- Rapport du {date_str} ---
Headline: {headline}
Stats: {stats}
Gagnants (matures): {winners_a[:300]}
Perdants (matures): {losers_a[:300]}
Leçons clés: {lessons_str[:400]}
Angles morts: {blind[:200]}
Priorités cycle suivant: {next_p[:200]}
Note timing: {timing[:150]}
"""

    # Classify trades by maturity — only draw P&L conclusions from matures
    def _enrich(t: dict) -> dict:
        try:
            dh = (_date.today() - _date.fromisoformat(t["entry_date"])).days
        except Exception:
            dh = 0
        mat = _trade_maturity(dh, t.get("horizon_days") or 90)
        return {**t, "days_held": dh, "maturity": mat}

    enriched = [_enrich(t) for t in trades]
    matures   = [t for t in enriched if t["maturity"]["status"] in ("mature", "fin_horizon") and t.get("pnl_pct") is not None]
    immatures = [t for t in enriched if t["maturity"]["status"] in ("trop_tot", "debut")]

    mature_winners = [t for t in matures if (t.get("pnl_pct") or 0) > 0.5]
    mature_losers  = [t for t in matures if (t.get("pnl_pct") or 0) < -0.5]
    mature_neutral = [t for t in matures if t not in mature_winners and t not in mature_losers]

    def trade_line(t):
        mat = t.get("maturity", {})
        pnl = t.get("pnl_pct") or 0
        return (f"{t.get('underlying','?')} {t.get('strategy','?')} "
                f"P&L={pnl:+.1f}% score={t.get('latest_score','?')} "
                f"regime={t.get('macro_regime','?')} [{mat.get('readable','')}]")

    trades_block = f"""
⚠️ ATTENTION TIMING : nos options durent 30-90 jours. Seuls les trades MATURES (≥35% horizon écoulé) ont un signal P&L fiable.

MATURES — signal fiable ({len(matures)} trades) :
  Gagnants ({len(mature_winners)}): {' | '.join(trade_line(t) for t in mature_winners[:8]) or 'aucun'}
  Perdants ({len(mature_losers)}): {' | '.join(trade_line(t) for t in mature_losers[:8]) or 'aucun'}
  Neutres ({len(mature_neutral)}): {len(mature_neutral)} trades sans signal fort

IMMATURES — trop tôt pour conclure ({len(immatures)} trades) :
  {' | '.join(trade_line(t) for t in immatures[:6]) or 'aucun'}
  → Ces trades sont listés pour transparence uniquement. Ne pas en tirer de conclusions de performance.
"""

    # Existing KB
    kb_block = ""
    if kb_entries:
        by_cat: Dict[str, List] = {}
        for e in kb_entries:
            cat = e.get("category", "général")
            by_cat.setdefault(cat, []).append(e)
        for cat, items in by_cat.items():
            kb_block += f"\n[{cat.upper()}]\n"
            for item in items[:5]:
                kb_block += f"  - [{item['confidence']}%] {item['title']}: {item['content'][:150]}\n"

    system = """Tu es l'intelligence analytique centrale d'un système de trading d'options géopolitiques.
Tu dois synthétiser TOUT l'historique disponible pour produire un document de raisonnement évolutif.
Ce document sera utilisé comme contexte enrichi pour tous les prochains cycles d'analyse.

⚠️ RÈGLE FONDAMENTALE DE TIMING :
Nos trades sont des options de 30 à 90 jours. Un trade ajouté il y a 2 heures ne dit RIEN sur sa performance finale.
Tu NE DOIS PAS réviser les conclusions existantes à cause de trades immatures (< 35% de l'horizon écoulé).
Tire des leçons UNIQUEMENT des trades MATURES. Les trades immatures sont listés pour transparence, pas pour analyse.
Si la synthèse précédente était basée sur des trades matures solides, ne la remets pas en cause à cause de nouveaux trades immatures.

Réponds UNIQUEMENT en JSON valide selon le schéma spécifié."""

    user = f"""Date: {now_str}

=== HISTORIQUE DES RAPPORTS DE PERFORMANCE ({len(reports)} rapports) ===
{reports_block}

=== HISTORIQUE DES TRADES ({len(trades)} trades total — {len(matures)} matures, {len(immatures)} immatures) ===
{trades_block}

=== BASE DE CONNAISSANCES EXISTANTE ===
{kb_block if kb_block else "Aucune entrée existante — première synthèse."}

=== MISSION ===
Produis un JSON avec ces champs.
IMPORTANT : si peu de trades matures sont disponibles, conserve les insights existants de la KB avec une confiance stable plutôt que de tout remettre à zéro.

{{
  "narrative": "Un texte narratif riche (500-800 mots) qui décrit l'état actuel du raisonnement du système, les patterns qui fonctionnent, les erreurs récurrentes, les corrélations géopolitiques/macro identifiées, les régimes qui favorisent nos stratégies, et les priorités d'amélioration. C'est le 'cerveau' du système.",

  "regime_insights": [
    {{"regime": "nom du régime macro", "observation": "ce qu'on sait de ce régime", "confidence": 0-100, "trade_count": N}}
  ],

  "pattern_insights": [
    {{"pattern": "nom du pattern", "observation": "performance et conditions", "confidence": 0-100, "win_rate_pct": 0-100}}
  ],

  "macro_correlations": [
    {{"trigger": "événement géopolitique/macro", "market_reaction": "réaction observée", "reliability": "haute/moyenne/faible"}}
  ],

  "recurring_mistakes": [
    {{"mistake": "description de l'erreur", "frequency": "souvent/parfois", "mitigation": "comment l'éviter"}}
  ],

  "strengths": ["point fort 1", "point fort 2"],

  "blind_spots": ["angle mort 1", "angle mort 2"],

  "strategic_priorities": ["priorité 1", "priorité 2", "priorité 3"],

  "risk_parameters": {{
    "avoid_when": ["condition 1", "condition 2"],
    "prefer_when": ["condition 1", "condition 2"]
  }}
}}"""

    return system, user


@router.get("/state")
def get_state():
    """Latest synthesized reasoning state."""
    state = get_latest_reasoning_state()
    return {"state": state}


@router.get("/history")
def get_history(limit: int = 10):
    """List of reasoning state versions."""
    return {"history": get_reasoning_history(limit)}


@router.get("/history/{state_id}")
def get_state_version(state_id: int):
    state = get_reasoning_state_by_id(state_id)
    if not state:
        raise HTTPException(404, "Version introuvable")
    return {"state": state}


@router.delete("/history/{state_id}")
def delete_state_version(state_id: int):
    """Delete a reasoning state version by ID."""
    deleted = delete_reasoning_state(state_id)
    if not deleted:
        raise HTTPException(404, "Version introuvable")
    return {"deleted": True, "id": state_id}


@router.delete("/entries/{entry_id}")
def delete_entry(entry_id: int):
    """Permanently delete a KB entry."""
    deleted = delete_kb_entry(entry_id)
    if not deleted:
        raise HTTPException(404, "Entrée introuvable")
    return {"deleted": True, "id": entry_id}


@router.get("/entries")
def list_entries(status: str = "all"):
    if status == "all":
        entries = get_all_kb_entries()
    else:
        entries = get_kb_entries(status)
    by_cat: Dict[str, List] = {}
    for e in entries:
        by_cat.setdefault(e.get("category", "général"), []).append(e)
    return {"entries": entries, "by_category": by_cat, "total": len(entries)}


class KbEntryIn(BaseModel):
    category: str
    title: str
    content: str
    confidence: int = 50
    tags: str = ""
    existing_id: Optional[int] = None


@router.post("/entries")
def add_entry(body: KbEntryIn):
    entry_id = save_kb_entry(
        category=body.category,
        title=body.title,
        content=body.content,
        confidence=body.confidence,
        tags=body.tags,
        existing_id=body.existing_id,
    )
    return {"id": entry_id}


@router.patch("/entries/{entry_id}/status")
def patch_entry_status(entry_id: int, body: Dict[str, str]):
    status = body.get("status", "active")
    if status not in ("active", "tentative", "invalidated"):
        raise HTTPException(400, "status must be active | tentative | invalidated")
    update_kb_entry_status(entry_id, status)
    return {"id": entry_id, "status": status}


@router.post("/synthesize")
async def synthesize(force: bool = False):
    """Run GPT-4o synthesis over all historical data and save new reasoning state.

    By default, skips if last synthesis was < 6h ago (use ?force=true to override).
    """
    ai_key = os.environ.get("OPENAI_API_KEY", "")
    if not ai_key:
        raise HTTPException(400, "OpenAI API key not configured")

    # 6h staleness gate (unless force=true)
    if not force:
        last = get_latest_reasoning_state()
        if last and last.get("created_at"):
            import datetime as _dt
            try:
                last_ts = _dt.datetime.fromisoformat(last["created_at"].replace("Z", "+00:00"))
                if last_ts.tzinfo is None:
                    last_ts = last_ts.replace(tzinfo=_dt.timezone.utc)
                age_h = (_dt.datetime.now(_dt.timezone.utc) - last_ts).total_seconds() / 3600
                if age_h < 6:
                    return {
                        "skipped": True,
                        "reason": f"Dernière synthèse il y a {age_h:.1f}h (< 6h). Utilise ?force=true pour forcer.",
                        "last_version": last.get("version"),
                        "last_at": last.get("created_at"),
                        "age_hours": round(age_h, 1),
                    }
            except Exception:
                pass

    import openai
    client = openai.OpenAI(api_key=ai_key)

    reports = list_ai_reports(limit=10)
    mtm_data = get_mtm_trades_with_traces(days=90)
    trades = mtm_data.get("all_trades", []) if isinstance(mtm_data, dict) else []
    kb_entries = get_all_kb_entries()

    system_msg, user_msg = _build_synthesis_prompt(reports, trades, kb_entries)

    try:
        resp = client.chat.completions.create(
            model="gpt-4o",
            messages=[
                {"role": "system", "content": system_msg},
                {"role": "user", "content": user_msg},
            ],
            temperature=0.3,
            max_tokens=2500,
            response_format={"type": "json_object"},
        )
        raw = resp.choices[0].message.content or "{}"
        synthesis = json.loads(raw)
    except Exception as e:
        raise HTTPException(500, f"GPT-4o error: {e}")

    narrative = synthesis.pop("narrative", "Synthèse non disponible.")

    state_id = save_reasoning_state(
        narrative=narrative,
        synthesis=synthesis,
        sources_count=len(reports) + len(trades),
        reports_used=len(reports),
        trades_analyzed=len(trades),
    )

    # Persist KB entries from synthesis
    for regime in synthesis.get("regime_insights", []):
        if regime.get("observation"):
            save_kb_entry(
                category="régimes",
                title=f"Régime: {regime.get('regime', '?')}",
                content=regime.get("observation", ""),
                confidence=regime.get("confidence", 50),
                tags="auto-synth",
            )

    for pattern in synthesis.get("pattern_insights", []):
        if pattern.get("observation"):
            save_kb_entry(
                category="patterns",
                title=f"Pattern: {pattern.get('pattern', '?')}",
                content=pattern.get("observation", ""),
                confidence=pattern.get("confidence", 50),
                tags="auto-synth",
            )

    for mistake in synthesis.get("recurring_mistakes", []):
        if mistake.get("mistake"):
            save_kb_entry(
                category="erreurs",
                title=mistake.get("mistake", "")[:80],
                content=f"{mistake.get('mistake','')} → {mistake.get('mitigation','')}",
                confidence=70,
                tags="auto-synth",
            )

    return {
        "state_id": state_id,
        "narrative_preview": narrative[:200],
        "kb_entries_added": (
            len(synthesis.get("regime_insights", [])) +
            len(synthesis.get("pattern_insights", [])) +
            len(synthesis.get("recurring_mistakes", []))
        ),
        "sources": {"reports": len(reports), "trades": len(trades)},
    }


@router.get("/context-for-cycle")
def context_for_cycle():
    """Compact context to inject into AI cycle prompts."""
    state = get_latest_reasoning_state()
    if not state:
        return {"available": False, "context": ""}

    synthesis = state.get("synthesis") or {}
    narrative = state.get("narrative", "")

    priorities = synthesis.get("strategic_priorities", [])
    avoid = synthesis.get("risk_parameters", {}).get("avoid_when", [])
    prefer = synthesis.get("risk_parameters", {}).get("prefer_when", [])
    mistakes = [m.get("mistake", "") for m in synthesis.get("recurring_mistakes", [])[:3]]
    strengths = synthesis.get("strengths", [])

    context = f"""=== SUPER CONTEXTE — BASE DE RAISONNEMENT ({state.get('created_at','')[:16]}) ===
{narrative[:600]}

PRIORITÉS STRATÉGIQUES: {' | '.join(priorities[:3])}
ERREURS À ÉVITER: {' | '.join(mistakes)}
PRÉFÉRER QUAND: {' | '.join(prefer[:2])}
ÉVITER QUAND: {' | '.join(avoid[:2])}
FORCES: {' | '.join(strengths[:2])}
"""
    return {
        "available": True,
        "context": context,
        "version": state.get("version"),
        "created_at": state.get("created_at"),
        "sources": {
            "reports_used": state.get("reports_used"),
            "trades_analyzed": state.get("trades_analyzed"),
        },
    }