feat: Phase 4 — Moteur Probabiliste & Apprentissage Automatique
Sprint 4.1 — Bayesian Updating - database.py: update_bayesian_posteriors() — Beta(α,β) posteriors sur trades matures - database.py: get_bayesian_posteriors() — posteriors + IC 95% + dérive prior GPT vs posterior - Colonnes Bayésiennes ajoutées : bayesian_alpha, bayesian_beta, bayesian_win_rate, bayesian_sample_size - auto_cycle.py: appel update_bayesian_posteriors() en Step 5.5 (après scoring) Sprint 4.2 — Détection Automatique de Régimes (K-Means numpy pur) - database.py: detect_and_save_regime_clusters() — K-Means sur 7 gauges macro (VIX, slope, DXY…) - database.py: get_regime_cluster_history() — timeline des clusters - database.py: get_regime_transition_matrix() — P(cluster j | cluster i) sur N transitions - Table regime_clusters avec anomaly_flag (points > 3σ) - auto_cycle.py: appel detect_and_save_regime_clusters() en Step 5.6 Sprint 4.3 — Embeddings Sémantiques (remplace Jaccard) - database.py: get_or_create_pattern_embedding() — OpenAI text-embedding-3-small, stocké en DB - database.py: max_cosine_similarity_vs_existing() — similarité cosinus vs patterns existants - Table pattern_embeddings avec vecteur JSON + model_version - auto_cycle.py: _is_duplicate_pattern() — cosinus seuil 0.75 avec fallback Jaccard automatique Sprint 4.4 — Tableau de Bord Analytique Avancé - AnalyticsAdvanced.tsx: nouvelle page /analytics-advanced • BayesianTable : prior GPT vs WR bayésien ± IC 95%, dérive, niveau de confiance • ClusterTimeline : timeline colorée des clusters + anomalies • TransitionMatrix : heatmap P(j|i) avec diagonale auto-transition • EmbeddingsSummary : liste des patterns vectorisés • Boutons "Bayesian update" et "Détecter régime" avec mutation React Query - analytics.py router : 5 nouveaux endpoints (bayesian, regime-clusters, transitions, detect, embeddings) - useApi.ts : 4 nouveaux hooks (useBayesianPosteriors, useRegimeClusters, useRegimeTransitions, usePatternEmbeddings) - App.tsx + Sidebar.tsx : route /analytics-advanced + entrée menu Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
@@ -5,6 +5,13 @@ from services.database import (
|
||||
get_calibration_data,
|
||||
decay_kb_confidence,
|
||||
get_custom_patterns,
|
||||
# Phase 4
|
||||
get_bayesian_posteriors,
|
||||
update_bayesian_posteriors,
|
||||
detect_and_save_regime_clusters,
|
||||
get_regime_cluster_history,
|
||||
get_regime_transition_matrix,
|
||||
get_all_pattern_embeddings_summary,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/analytics", tags=["analytics"])
|
||||
@@ -53,3 +60,43 @@ def invalidation_alerts():
|
||||
if p.get("invalidation_trigger")
|
||||
]
|
||||
return {"alerts": alerts, "count": len(alerts)}
|
||||
|
||||
|
||||
# ── Phase 4 endpoints ────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/bayesian")
|
||||
def bayesian_posteriors():
|
||||
"""Posteriors bayésiens Beta(α,β) par pattern avec IC 95%."""
|
||||
return {"posteriors": get_bayesian_posteriors()}
|
||||
|
||||
|
||||
@router.post("/bayesian/update")
|
||||
def trigger_bayesian_update():
|
||||
"""Force la mise à jour des posteriors bayésiens."""
|
||||
n = update_bayesian_posteriors()
|
||||
return {"updated": n, "message": f"{n} pattern(s) mis à jour"}
|
||||
|
||||
|
||||
@router.get("/regime-clusters")
|
||||
def regime_cluster_history(days: int = Query(default=90, ge=7, le=365)):
|
||||
"""Historique des assignations de clusters de régimes macro."""
|
||||
return {"clusters": get_regime_cluster_history(days=days)}
|
||||
|
||||
|
||||
@router.get("/regime-transitions")
|
||||
def regime_transition_matrix(days: int = Query(default=180, ge=30, le=730)):
|
||||
"""Matrice de transition entre clusters de régimes."""
|
||||
return get_regime_transition_matrix(days=days)
|
||||
|
||||
|
||||
@router.post("/regime-detect")
|
||||
def trigger_regime_detect(n_clusters: int = Query(default=4, ge=2, le=6)):
|
||||
"""Force la détection et sauvegarde du cluster de régime courant."""
|
||||
result = detect_and_save_regime_clusters(n_clusters=n_clusters)
|
||||
return result
|
||||
|
||||
|
||||
@router.get("/embeddings")
|
||||
def pattern_embeddings_summary():
|
||||
"""Liste des patterns avec embedding vectoriel disponible."""
|
||||
return {"embeddings": get_all_pattern_embeddings_summary()}
|
||||
|
||||
@@ -49,6 +49,44 @@ def _max_similarity_vs_existing(candidate_kws: List[str], existing: List[Dict])
|
||||
return max((_jaccard(candidate_kws, p.get("keywords") or []) for p in existing), default=0.0)
|
||||
|
||||
|
||||
_EMBED_SIM_THRESHOLD = 0.75 # seuil cosinus pour considérer deux patterns comme doublons
|
||||
|
||||
|
||||
def _is_duplicate_pattern(
|
||||
candidate: Dict,
|
||||
existing: List[Dict],
|
||||
api_key: str,
|
||||
jaccard_threshold: float = 0.30,
|
||||
) -> bool:
|
||||
"""
|
||||
Retourne True si le candidat est trop similaire à un pattern existant.
|
||||
Essaie les embeddings cosinus (Sprint 4.3) avec fallback Jaccard.
|
||||
"""
|
||||
if not existing:
|
||||
return False
|
||||
|
||||
# Tentative embedding cosinus
|
||||
if api_key:
|
||||
text = ((candidate.get("name") or "") + " " + (candidate.get("description") or "")).strip()
|
||||
if text:
|
||||
try:
|
||||
from services.database import max_cosine_similarity_vs_existing
|
||||
sim = max_cosine_similarity_vs_existing(
|
||||
candidate_text=text,
|
||||
existing_patterns=existing,
|
||||
api_key=api_key,
|
||||
candidate_id=candidate.get("id"),
|
||||
)
|
||||
if sim > 0: # embedding a fonctionné
|
||||
return sim >= _EMBED_SIM_THRESHOLD
|
||||
except Exception as _emb_err:
|
||||
logger.debug(f"[Cycle] Embedding failed, fallback Jaccard: {_emb_err}")
|
||||
|
||||
# Fallback Jaccard
|
||||
sim = _max_similarity_vs_existing(candidate.get("keywords") or [], existing)
|
||||
return sim >= jaccard_threshold
|
||||
|
||||
|
||||
# ── Core cycle logic ──────────────────────────────────────────────────────────
|
||||
|
||||
def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
|
||||
@@ -230,20 +268,19 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
|
||||
summary["patterns_suggested"] = len(suggestions)
|
||||
logger.info(f"[Cycle {run_id[:16]}] Suggested {len(suggestions)} patterns from AI")
|
||||
|
||||
# ── Step 3: Filter by similarity ──────────────────────────────────────
|
||||
# ── Step 3: Filter by similarity (Embedding cosinus ou Jaccard fallback) ─
|
||||
existing = get_custom_patterns()
|
||||
logger.info(f"[Cycle {run_id[:16]}] Step 3: {len(existing)} existing patterns, threshold={sim_threshold}")
|
||||
added_count = 0
|
||||
for s in suggestions:
|
||||
kws = s.get("keywords") or []
|
||||
sim = _max_similarity_vs_existing(kws, existing)
|
||||
if sim < sim_threshold:
|
||||
if not _is_duplicate_pattern(s, existing, ai_key, jaccard_threshold=sim_threshold):
|
||||
|
||||
# Capture returned ID so the pattern has a valid id for scoring
|
||||
assigned_id = save_custom_pattern(s)
|
||||
s["id"] = assigned_id
|
||||
existing.append(s)
|
||||
added_count += 1
|
||||
logger.info(f"[Cycle] Added pattern '{s.get('name')}' id={assigned_id} (sim={sim:.2f})")
|
||||
logger.info(f"[Cycle] Added pattern '{s.get('name')}' id={assigned_id}")
|
||||
# ── Save suggestion reasoning trace ───────────────────────────
|
||||
_top_news_ctx = [
|
||||
{"title": n.get("title", "")[:120], "impact": round(float(n.get("impact_score") or 0), 2), "source": n.get("source", "")}
|
||||
@@ -276,7 +313,7 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
|
||||
macro_dominant=dominant,
|
||||
)
|
||||
else:
|
||||
logger.debug(f"[Cycle] Filtered '{s.get('name')}' — sim={sim:.2f} >= {sim_threshold}")
|
||||
logger.debug(f"[Cycle] Filtered '{s.get('name')}' — doublon détecté")
|
||||
|
||||
summary["patterns_added"] = added_count
|
||||
|
||||
@@ -434,6 +471,28 @@ def run_cycle_once(trigger: str = "auto") -> Dict[str, Any]:
|
||||
"fetched_at": datetime.utcnow().isoformat(), "cached": False}
|
||||
_macro_cache["ts"] = _dt.datetime.utcnow()
|
||||
|
||||
# ── Step 5.5: Bayesian posterior update (Sprint 4.1) ─────────────────
|
||||
try:
|
||||
from services.database import update_bayesian_posteriors
|
||||
_n_updated = update_bayesian_posteriors()
|
||||
if _n_updated:
|
||||
logger.info(f"[Cycle {run_id[:16]}] Bayesian posteriors mis à jour : {_n_updated} patterns")
|
||||
except Exception as _be:
|
||||
logger.warning(f"[Cycle] Bayesian update failed (non-blocking): {_be}")
|
||||
|
||||
# ── Step 5.6: Régime clustering (Sprint 4.2) ──────────────────────────
|
||||
try:
|
||||
from services.database import detect_and_save_regime_clusters
|
||||
_cluster_result = detect_and_save_regime_clusters(n_clusters=4, days=180)
|
||||
if "current_cluster" in _cluster_result:
|
||||
logger.info(
|
||||
f"[Cycle {run_id[:16]}] Régime cluster : {_cluster_result.get('current_label')} "
|
||||
f"(cluster {_cluster_result.get('current_cluster')}, "
|
||||
f"anomalie={_cluster_result.get('current_anomaly')})"
|
||||
)
|
||||
except Exception as _ce:
|
||||
logger.warning(f"[Cycle] Régime clustering failed (non-blocking): {_ce}")
|
||||
|
||||
# ── Step 6: GPT-4o cycle commentary ──────────────────────────────────
|
||||
logger.info(f"[Cycle {run_id[:16]}] Step 6: generating commentary")
|
||||
commentary = _generate_cycle_commentary(
|
||||
|
||||
@@ -72,12 +72,48 @@ def init_db():
|
||||
"ALTER TABLE pattern_score_history ADD COLUMN predicted_probability REAL",
|
||||
"ALTER TABLE knowledge_base ADD COLUMN expires_at TEXT",
|
||||
"ALTER TABLE knowledge_base ADD COLUMN confidence_decay_days INTEGER DEFAULT 90",
|
||||
# Phase 4.1 — Bayesian posteriors
|
||||
"ALTER TABLE custom_patterns ADD COLUMN bayesian_alpha REAL DEFAULT 1.0",
|
||||
"ALTER TABLE custom_patterns ADD COLUMN bayesian_beta REAL DEFAULT 1.0",
|
||||
"ALTER TABLE custom_patterns ADD COLUMN bayesian_win_rate REAL",
|
||||
"ALTER TABLE custom_patterns ADD COLUMN bayesian_updated_at TEXT",
|
||||
"ALTER TABLE custom_patterns ADD COLUMN bayesian_sample_size INTEGER DEFAULT 0",
|
||||
]:
|
||||
try:
|
||||
c.execute(_sql)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Phase 4.2 — Régime clusters (K-Means sur gauges macro)
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS regime_clusters (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
timestamp TEXT NOT NULL,
|
||||
cluster_id INTEGER NOT NULL,
|
||||
cluster_label TEXT,
|
||||
dominant_regime TEXT,
|
||||
gauges_json TEXT DEFAULT '{}',
|
||||
anomaly_flag INTEGER DEFAULT 0,
|
||||
created_at TEXT DEFAULT (datetime('now'))
|
||||
)""")
|
||||
try:
|
||||
c.execute("CREATE INDEX IF NOT EXISTS idx_rc_ts ON regime_clusters(timestamp DESC)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Phase 4.3 — Embeddings sémantiques des patterns
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS pattern_embeddings (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
pattern_id TEXT NOT NULL UNIQUE,
|
||||
embedding_json TEXT NOT NULL,
|
||||
model_version TEXT DEFAULT 'text-embedding-3-small',
|
||||
created_at TEXT DEFAULT (datetime('now')),
|
||||
updated_at TEXT DEFAULT (datetime('now'))
|
||||
)""")
|
||||
try:
|
||||
c.execute("CREATE INDEX IF NOT EXISTS idx_pe_pattern ON pattern_embeddings(pattern_id)")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
c.execute("""CREATE TABLE IF NOT EXISTS config (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL,
|
||||
@@ -2249,6 +2285,439 @@ def get_risk_dashboard() -> Dict:
|
||||
}
|
||||
|
||||
|
||||
# ╔══════════════════════════════════════════════════════════════════════════════╗
|
||||
# ║ PHASE 4 — Moteur Probabiliste & Apprentissage Automatique ║
|
||||
# ╚══════════════════════════════════════════════════════════════════════════════╝
|
||||
|
||||
# ── Sprint 4.1 — Bayesian Updating ───────────────────────────────────────────
|
||||
|
||||
def update_bayesian_posteriors() -> int:
|
||||
"""
|
||||
Met à jour les posteriors Beta(α,β) de chaque pattern selon ses trades matures.
|
||||
Prior faible : α₀=1, β₀=1 (Laplace smoothing).
|
||||
Posterior : α = 1 + wins, β = 1 + losses → win_rate bayésien = α/(α+β)
|
||||
Retourne le nombre de patterns mis à jour.
|
||||
"""
|
||||
import math as _math
|
||||
from datetime import date as _date
|
||||
|
||||
conn = get_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT pattern_id, pnl_pct, entry_date, horizon_days FROM trade_entry_prices WHERE pnl_pct IS NOT NULL"
|
||||
).fetchall()
|
||||
conn.close()
|
||||
|
||||
today = _date.today()
|
||||
by_pattern: Dict[str, list] = {}
|
||||
for row in rows:
|
||||
r = dict(row)
|
||||
try:
|
||||
entry = _date.fromisoformat(r["entry_date"])
|
||||
days_held = (today - entry).days
|
||||
except Exception:
|
||||
continue
|
||||
horizon = r.get("horizon_days") or 30
|
||||
if days_held / max(horizon, 1) < 0.35:
|
||||
continue # trades immatures exclus
|
||||
by_pattern.setdefault(r["pattern_id"], []).append(float(r["pnl_pct"] or 0))
|
||||
|
||||
if not by_pattern:
|
||||
return 0
|
||||
|
||||
conn = get_conn()
|
||||
c = conn.cursor()
|
||||
updated = 0
|
||||
now_iso = datetime.utcnow().isoformat()
|
||||
for pid, pnls in by_pattern.items():
|
||||
n = len(pnls)
|
||||
wins = sum(1 for p in pnls if p > 0)
|
||||
losses = n - wins
|
||||
alpha = 1.0 + wins # posterior alpha
|
||||
beta = 1.0 + losses # posterior beta
|
||||
bayes_wr = alpha / (alpha + beta)
|
||||
c.execute("""
|
||||
UPDATE custom_patterns
|
||||
SET bayesian_alpha=?, bayesian_beta=?, bayesian_win_rate=?,
|
||||
bayesian_updated_at=?, bayesian_sample_size=?
|
||||
WHERE id=?
|
||||
""", (round(alpha, 1), round(beta, 1), round(bayes_wr, 4), now_iso, n, pid))
|
||||
if c.rowcount:
|
||||
updated += 1
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return updated
|
||||
|
||||
|
||||
def get_bayesian_posteriors() -> List[Dict]:
|
||||
"""
|
||||
Retourne tous les patterns avec leurs posteriors bayésiens + intervalle de crédibilité 95%.
|
||||
CI 95% via approximation normale : ±1.96 × σ = ±1.96 × sqrt(αβ / (α+β)²(α+β+1))
|
||||
"""
|
||||
import math as _math
|
||||
conn = get_conn()
|
||||
rows = conn.execute("""
|
||||
SELECT id, name, probability, bayesian_alpha, bayesian_beta,
|
||||
bayesian_win_rate, bayesian_sample_size, bayesian_updated_at,
|
||||
asset_class, is_active
|
||||
FROM custom_patterns
|
||||
WHERE is_active=1
|
||||
ORDER BY bayesian_sample_size DESC, name
|
||||
""").fetchall()
|
||||
conn.close()
|
||||
|
||||
result = []
|
||||
for row in rows:
|
||||
r = dict(row)
|
||||
alpha = r.get("bayesian_alpha") or 1.0
|
||||
beta = r.get("bayesian_beta") or 1.0
|
||||
n = alpha + beta
|
||||
# 95% credible interval (Beta distribution approximation)
|
||||
variance = (alpha * beta) / (n * n * (n + 1))
|
||||
std = _math.sqrt(max(variance, 0))
|
||||
bayes_wr = r.get("bayesian_win_rate") or alpha / n
|
||||
lower = max(0.0, bayes_wr - 1.96 * std)
|
||||
upper = min(1.0, bayes_wr + 1.96 * std)
|
||||
sample_size = r.get("bayesian_sample_size") or 0
|
||||
|
||||
# Écart entre prior GPT et posterior bayésien
|
||||
prior = r.get("probability") or 0.5
|
||||
drift = round(bayes_wr - prior, 3)
|
||||
|
||||
result.append({
|
||||
"pattern_id": r["id"],
|
||||
"pattern_name": r["name"],
|
||||
"asset_class": r.get("asset_class"),
|
||||
"prior_probability": round(prior, 3),
|
||||
"bayesian_win_rate": round(bayes_wr, 3),
|
||||
"bayesian_win_rate_pct": round(bayes_wr * 100, 1),
|
||||
"lower_ci_95": round(lower, 3),
|
||||
"upper_ci_95": round(upper, 3),
|
||||
"ci_width": round(upper - lower, 3),
|
||||
"sample_size": sample_size,
|
||||
"alpha": alpha,
|
||||
"beta": beta,
|
||||
"prior_vs_posterior_drift": drift,
|
||||
"updated_at": r.get("bayesian_updated_at"),
|
||||
"confidence_level": (
|
||||
"haute" if sample_size >= 10
|
||||
else "moyenne" if sample_size >= 5
|
||||
else "faible"
|
||||
),
|
||||
})
|
||||
return result
|
||||
|
||||
|
||||
# ── Sprint 4.2 — Détection automatique de régimes ────────────────────────────
|
||||
|
||||
_GAUGE_FEATURES = [
|
||||
"vix", "slope_10y3m", "dxy", "brent", "gold", "copper", "spx_vs_200d"
|
||||
]
|
||||
|
||||
_CLUSTER_LABELS = {
|
||||
0: "Stress Géopolitique",
|
||||
1: "Expansion Tranquille",
|
||||
2: "Récession / Risk-Off",
|
||||
3: "Stagflation / Inflation",
|
||||
4: "Transition / Incertain",
|
||||
}
|
||||
|
||||
|
||||
def _kmeans_numpy(X, n_clusters: int = 4, max_iter: int = 100, seed: int = 42):
|
||||
"""K-Means minimal en numpy pur (pas de sklearn nécessaire)."""
|
||||
import numpy as np
|
||||
rng = np.random.default_rng(seed)
|
||||
idx = rng.choice(len(X), size=n_clusters, replace=False)
|
||||
centroids = X[idx].copy()
|
||||
|
||||
for _ in range(max_iter):
|
||||
dists = np.linalg.norm(X[:, None, :] - centroids[None, :, :], axis=2)
|
||||
labels = np.argmin(dists, axis=1)
|
||||
new_centroids = np.array([
|
||||
X[labels == k].mean(axis=0) if (labels == k).any() else centroids[k]
|
||||
for k in range(n_clusters)
|
||||
])
|
||||
if np.allclose(centroids, new_centroids, atol=1e-6):
|
||||
break
|
||||
centroids = new_centroids
|
||||
return labels, centroids
|
||||
|
||||
|
||||
def detect_and_save_regime_clusters(n_clusters: int = 4, days: int = 180) -> Dict:
|
||||
"""
|
||||
Applique K-Means sur l'historique des gauges macro (VIX, pente, DXY…).
|
||||
Sauvegarde l'assignation courante dans regime_clusters.
|
||||
Retourne le cluster actuel + les centroïdes labellisés.
|
||||
"""
|
||||
import numpy as np
|
||||
import json as _json
|
||||
|
||||
conn = get_conn()
|
||||
rows = conn.execute("""
|
||||
SELECT timestamp, dominant, gauges_summary_json
|
||||
FROM macro_regime_history
|
||||
WHERE timestamp >= datetime('now', ?)
|
||||
ORDER BY timestamp ASC
|
||||
""", (f"-{days} days",)).fetchall()
|
||||
conn.close()
|
||||
|
||||
if len(rows) < max(n_clusters * 2, 8):
|
||||
return {"error": "Données insuffisantes", "min_required": max(n_clusters * 2, 8)}
|
||||
|
||||
snapshots = []
|
||||
timestamps = []
|
||||
dominants = []
|
||||
for row in rows:
|
||||
r = dict(row)
|
||||
try:
|
||||
gauges = _json.loads(r.get("gauges_summary_json") or "{}")
|
||||
except Exception:
|
||||
continue
|
||||
vec = []
|
||||
for feat in _GAUGE_FEATURES:
|
||||
g = gauges.get(feat, {})
|
||||
val = g.get("value") if isinstance(g, dict) else g
|
||||
vec.append(float(val) if val is not None else 0.0)
|
||||
snapshots.append(vec)
|
||||
timestamps.append(r["timestamp"])
|
||||
dominants.append(r.get("dominant", "incertain"))
|
||||
|
||||
if not snapshots:
|
||||
return {"error": "Aucune donnée de gauges exploitable"}
|
||||
|
||||
X = np.array(snapshots, dtype=float)
|
||||
# Normalisation z-score par feature
|
||||
mean = X.mean(axis=0)
|
||||
std = X.std(axis=0)
|
||||
std[std == 0] = 1.0
|
||||
X_norm = (X - mean) / std
|
||||
|
||||
# Anomaly detection : points > 3σ du vecteur global
|
||||
global_dist = np.linalg.norm(X_norm, axis=1)
|
||||
anomaly_threshold = global_dist.mean() + 3 * global_dist.std()
|
||||
|
||||
labels, centroids = _kmeans_numpy(X_norm, n_clusters=n_clusters)
|
||||
|
||||
# Labelliser les clusters selon le dominant le plus fréquent
|
||||
cluster_regimes: Dict[int, str] = {}
|
||||
for k in range(n_clusters):
|
||||
idxs = [i for i, l in enumerate(labels) if l == k]
|
||||
if idxs:
|
||||
dom_counts: Dict[str, int] = {}
|
||||
for i in idxs:
|
||||
d = dominants[i]
|
||||
dom_counts[d] = dom_counts.get(d, 0) + 1
|
||||
cluster_regimes[k] = max(dom_counts, key=dom_counts.get)
|
||||
|
||||
# Sauvegarder le snapshot le plus récent
|
||||
last_idx = len(labels) - 1
|
||||
current_cluster = int(labels[last_idx])
|
||||
current_anomaly = bool(global_dist[last_idx] > anomaly_threshold)
|
||||
|
||||
conn = get_conn()
|
||||
conn.execute("""
|
||||
INSERT INTO regime_clusters
|
||||
(timestamp, cluster_id, cluster_label, dominant_regime, gauges_json, anomaly_flag)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
""", (
|
||||
timestamps[last_idx],
|
||||
current_cluster,
|
||||
_CLUSTER_LABELS.get(current_cluster, f"Cluster {current_cluster}"),
|
||||
cluster_regimes.get(current_cluster, dominants[last_idx]),
|
||||
_json.dumps({f: round(float(snapshots[last_idx][i]), 3) for i, f in enumerate(_GAUGE_FEATURES)}),
|
||||
int(current_anomaly),
|
||||
))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
# Statistiques par cluster
|
||||
cluster_stats = {}
|
||||
for k in range(n_clusters):
|
||||
k_idxs = [i for i, l in enumerate(labels) if l == k]
|
||||
cluster_stats[k] = {
|
||||
"cluster_id": k,
|
||||
"label": _CLUSTER_LABELS.get(k, f"Cluster {k}"),
|
||||
"dominant_regime": cluster_regimes.get(k, "incertain"),
|
||||
"count": len(k_idxs),
|
||||
"pct": round(len(k_idxs) / len(labels) * 100, 1),
|
||||
"centroid": {f: round(float(centroids[k][i]), 3) for i, f in enumerate(_GAUGE_FEATURES)},
|
||||
}
|
||||
|
||||
return {
|
||||
"current_cluster": current_cluster,
|
||||
"current_label": _CLUSTER_LABELS.get(current_cluster, f"Cluster {current_cluster}"),
|
||||
"current_anomaly": current_anomaly,
|
||||
"cluster_stats": list(cluster_stats.values()),
|
||||
"n_snapshots": len(labels),
|
||||
"features": _GAUGE_FEATURES,
|
||||
}
|
||||
|
||||
|
||||
def get_regime_cluster_history(days: int = 90) -> List[Dict]:
|
||||
"""Retourne l'historique des assignations de clusters."""
|
||||
conn = get_conn()
|
||||
rows = conn.execute("""
|
||||
SELECT timestamp, cluster_id, cluster_label, dominant_regime, anomaly_flag, created_at
|
||||
FROM regime_clusters
|
||||
WHERE timestamp >= datetime('now', ?)
|
||||
ORDER BY timestamp DESC
|
||||
LIMIT 200
|
||||
""", (f"-{days} days",)).fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def get_regime_transition_matrix(days: int = 180) -> Dict:
|
||||
"""
|
||||
Calcule la matrice de transition entre clusters :
|
||||
P(cluster_j | cluster_i) = nb transitions i→j / nb fois où on était en i.
|
||||
"""
|
||||
rows = get_regime_cluster_history(days=days)
|
||||
if len(rows) < 4:
|
||||
return {"matrix": {}, "labels": {}, "n_transitions": 0}
|
||||
|
||||
# rows trié DESC → inverser pour avoir l'ordre chronologique
|
||||
seq = list(reversed(rows))
|
||||
cluster_ids = sorted({r["cluster_id"] for r in seq})
|
||||
|
||||
counts: Dict[int, Dict[int, int]] = {c: {d: 0 for d in cluster_ids} for c in cluster_ids}
|
||||
totals: Dict[int, int] = {c: 0 for c in cluster_ids}
|
||||
|
||||
for i in range(len(seq) - 1):
|
||||
src = seq[i]["cluster_id"]
|
||||
dst = seq[i + 1]["cluster_id"]
|
||||
counts[src][dst] = counts[src].get(dst, 0) + 1
|
||||
totals[src] = totals.get(src, 0) + 1
|
||||
|
||||
matrix = {}
|
||||
for src in cluster_ids:
|
||||
matrix[src] = {}
|
||||
total = totals.get(src, 0)
|
||||
for dst in cluster_ids:
|
||||
matrix[src][dst] = round(counts[src].get(dst, 0) / total, 3) if total else 0.0
|
||||
|
||||
labels = {k: _CLUSTER_LABELS.get(k, f"Cluster {k}") for k in cluster_ids}
|
||||
n_trans = sum(totals.values())
|
||||
|
||||
return {"matrix": matrix, "labels": labels, "cluster_ids": cluster_ids, "n_transitions": n_trans}
|
||||
|
||||
|
||||
# ── Sprint 4.3 — Embeddings sémantiques ──────────────────────────────────────
|
||||
|
||||
def _cosine_similarity(a: List[float], b: List[float]) -> float:
|
||||
"""Similarité cosinus entre deux vecteurs."""
|
||||
import math as _math
|
||||
dot = sum(x * y for x, y in zip(a, b))
|
||||
na = _math.sqrt(sum(x * x for x in a))
|
||||
nb = _math.sqrt(sum(y * y for y in b))
|
||||
if na == 0 or nb == 0:
|
||||
return 0.0
|
||||
return dot / (na * nb)
|
||||
|
||||
|
||||
def get_or_create_pattern_embedding(pattern_id: str, text: str, api_key: str) -> Optional[List[float]]:
|
||||
"""
|
||||
Retourne l'embedding stocké pour ce pattern, ou le crée via OpenAI text-embedding-3-small.
|
||||
Le vecteur est stocké en JSON dans pattern_embeddings.
|
||||
"""
|
||||
import json as _json
|
||||
conn = get_conn()
|
||||
row = conn.execute(
|
||||
"SELECT embedding_json FROM pattern_embeddings WHERE pattern_id=?", (pattern_id,)
|
||||
).fetchone()
|
||||
conn.close()
|
||||
|
||||
if row:
|
||||
try:
|
||||
return _json.loads(row["embedding_json"])
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Créer via OpenAI
|
||||
try:
|
||||
import urllib.request as _req
|
||||
import urllib.error as _uerr
|
||||
payload = _json.dumps({
|
||||
"input": text[:8000],
|
||||
"model": "text-embedding-3-small",
|
||||
}).encode("utf-8")
|
||||
request = _req.Request(
|
||||
"https://api.openai.com/v1/embeddings",
|
||||
data=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"},
|
||||
method="POST",
|
||||
)
|
||||
with _req.urlopen(request, timeout=15) as resp:
|
||||
data = _json.loads(resp.read())
|
||||
vec = data["data"][0]["embedding"]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# Persister
|
||||
vec_json = _json.dumps(vec)
|
||||
conn = get_conn()
|
||||
conn.execute("""
|
||||
INSERT OR REPLACE INTO pattern_embeddings (pattern_id, embedding_json, model_version, updated_at)
|
||||
VALUES (?, ?, 'text-embedding-3-small', ?)
|
||||
""", (pattern_id, vec_json, datetime.utcnow().isoformat()))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return vec
|
||||
|
||||
|
||||
def max_cosine_similarity_vs_existing(
|
||||
candidate_text: str,
|
||||
existing_patterns: List[Dict],
|
||||
api_key: str,
|
||||
candidate_id: Optional[str] = None,
|
||||
) -> float:
|
||||
"""
|
||||
Calcule la similarité cosinus maximale entre le candidat et les patterns existants.
|
||||
Retourne 0.0 si les embeddings ne sont pas disponibles (fallback Jaccard dans l'appelant).
|
||||
"""
|
||||
import json as _json
|
||||
if not api_key or not existing_patterns:
|
||||
return 0.0
|
||||
|
||||
# Obtenir l'embedding du candidat
|
||||
_tmp_id = candidate_id or f"_tmp_{hash(candidate_text) & 0xFFFFFF}"
|
||||
cand_vec = get_or_create_pattern_embedding(_tmp_id, candidate_text, api_key)
|
||||
if not cand_vec:
|
||||
return 0.0
|
||||
|
||||
max_sim = 0.0
|
||||
conn = get_conn()
|
||||
for p in existing_patterns:
|
||||
pid = p.get("id")
|
||||
if not pid:
|
||||
continue
|
||||
row = conn.execute(
|
||||
"SELECT embedding_json FROM pattern_embeddings WHERE pattern_id=?", (pid,)
|
||||
).fetchone()
|
||||
if row:
|
||||
try:
|
||||
vec = _json.loads(row["embedding_json"])
|
||||
sim = _cosine_similarity(cand_vec, vec)
|
||||
if sim > max_sim:
|
||||
max_sim = sim
|
||||
except Exception:
|
||||
pass
|
||||
conn.close()
|
||||
return max_sim
|
||||
|
||||
|
||||
def get_all_pattern_embeddings_summary() -> List[Dict]:
|
||||
"""Liste des patterns avec embedding disponible (pour l'Analytics Dashboard)."""
|
||||
conn = get_conn()
|
||||
rows = conn.execute("""
|
||||
SELECT pe.pattern_id, pe.model_version, pe.updated_at, cp.name, cp.asset_class
|
||||
FROM pattern_embeddings pe
|
||||
LEFT JOIN custom_patterns cp ON cp.id = pe.pattern_id
|
||||
ORDER BY pe.updated_at DESC
|
||||
""").fetchall()
|
||||
conn.close()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
def _build_risk_recommendation(
|
||||
saturated: List[str],
|
||||
div_score: float,
|
||||
|
||||
Reference in New Issue
Block a user