feat: 4 remaining institutional reports — Earnings, VX curve, Central Bank RSS, Sentiment
New fetchers (no API keys required): - earnings_fetcher.py: yfinance EPS calendar + surprise tracking for 23 geo-relevant tickers - vx_fetcher.py: VIX term structure (^VIX/^VXV/^VXMT) + CBOE delayed futures, regime detection - central_bank_fetcher.py: Fed + ECB RSS feeds, keyword-based hawkish/dovish classification - sentiment_fetcher.py: CNN Fear & Greed (primary) + NAAIM + AAII (optional fallbacks) Wiring: - institutional_scheduler.py: all 4 now scheduled daily (≥08:00 UTC), deduplicated per day - institutional.py /refresh: all 6 types handled with _run() helper - ai_analyzer.py build_institutional_block(): limit 6→12, generic header text - InstitutionalReports.tsx: 6-type color map, individual refresh buttons, expanded filters Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
315
backend/services/sentiment_fetcher.py
Normal file
315
backend/services/sentiment_fetcher.py
Normal file
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
Market sentiment tracker — CNN Fear & Greed + NAAIM + AAII.
|
||||
All free/public sources. No API key required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CNN_FG_URL = "https://production.dataviz.cnn.io/index/fearandgreed/graphdata/"
|
||||
|
||||
NAAIM_URLS = [
|
||||
"https://www.naaim.org/wp-content/uploads/NAAIM-Exposure-Index-Weekly-Data.csv",
|
||||
"https://www.naaim.org/programs/naaim-exposure-index/",
|
||||
]
|
||||
|
||||
AAII_URL = "https://www.aaii.com/sentimentsurvey/sent_results.js"
|
||||
|
||||
_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; GeoOptions/1.0)"}
|
||||
_TIMEOUT = 15
|
||||
|
||||
|
||||
def _fetch_cnn_fear_greed() -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
resp = requests.get(CNN_FG_URL, timeout=_TIMEOUT, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
logger.warning(f"[SENTIMENT] CNN F&G fetch failed: {e}")
|
||||
return None
|
||||
|
||||
fg = data.get("fear_and_greed", {})
|
||||
try:
|
||||
score = float(fg.get("score", 0))
|
||||
rating = str(fg.get("rating", "unknown")).lower()
|
||||
prev_close = float(fg.get("previous_close", score))
|
||||
prev_1_week = float(fg.get("previous_1_week", score))
|
||||
prev_1_month = float(fg.get("previous_1_month", score))
|
||||
prev_1_year = float(fg.get("previous_1_year", score))
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.warning(f"[SENTIMENT] CNN F&G parse error: {e}")
|
||||
return None
|
||||
|
||||
logger.info(f"[SENTIMENT] CNN F&G: {score:.1f} / 100 ({rating})")
|
||||
return {
|
||||
"score": score,
|
||||
"rating": rating,
|
||||
"previous_close": prev_close,
|
||||
"previous_1_week": prev_1_week,
|
||||
"previous_1_month": prev_1_month,
|
||||
"previous_1_year": prev_1_year,
|
||||
}
|
||||
|
||||
|
||||
_NAAIM_NUMBER_RE = re.compile(r"(\d+\.?\d*)\s*(?:Average|Mean|Exposure)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _parse_naaim_csv(text: str) -> Optional[float]:
|
||||
lines = [l.strip() for l in text.splitlines() if l.strip()]
|
||||
if len(lines) < 2:
|
||||
return None
|
||||
data_line = lines[1]
|
||||
parts = re.split(r"[,\t]", data_line)
|
||||
for part in reversed(parts):
|
||||
try:
|
||||
val = float(part.strip())
|
||||
if 0.0 <= val <= 200.0:
|
||||
return val
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_naaim() -> Optional[float]:
|
||||
try:
|
||||
resp = requests.get(NAAIM_URLS[0], timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code == 200 and resp.text.strip():
|
||||
val = _parse_naaim_csv(resp.text)
|
||||
if val is not None:
|
||||
logger.info(f"[SENTIMENT] NAAIM exposure (CSV): {val:.1f}")
|
||||
return val
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] NAAIM CSV failed: {e}")
|
||||
|
||||
try:
|
||||
resp = requests.get(NAAIM_URLS[1], timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code == 200:
|
||||
m = _NAAIM_NUMBER_RE.search(resp.text)
|
||||
if m:
|
||||
val = float(m.group(1))
|
||||
logger.info(f"[SENTIMENT] NAAIM exposure (HTML): {val:.1f}")
|
||||
return val
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] NAAIM HTML failed: {e}")
|
||||
|
||||
logger.info("[SENTIMENT] NAAIM not available — skipping")
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_aaii() -> Optional[Tuple[float, float, float]]:
|
||||
try:
|
||||
resp = requests.get(AAII_URL, timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
text = resp.text
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] AAII fetch failed: {e}")
|
||||
return None
|
||||
|
||||
try:
|
||||
bull_m = re.search(r'"bullish"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
bear_m = re.search(r'"bearish"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
neut_m = re.search(r'"neutral"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
|
||||
if bull_m and bear_m:
|
||||
bull = float(bull_m.group(1))
|
||||
bear = float(bear_m.group(1))
|
||||
neut = float(neut_m.group(1)) if neut_m else max(0.0, 100.0 - bull - bear)
|
||||
|
||||
if 0 <= bull <= 100 and 0 <= bear <= 100:
|
||||
logger.info(f"[SENTIMENT] AAII Bulls={bull:.1f}% Bears={bear:.1f}%")
|
||||
return bull, bear, neut
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.debug(f"[SENTIMENT] AAII parse error: {e}")
|
||||
|
||||
logger.info("[SENTIMENT] AAII not available — skipping")
|
||||
return None
|
||||
|
||||
|
||||
def _signal_indices_from_fg(score: float) -> str:
|
||||
if score < 25:
|
||||
return "bullish"
|
||||
if score > 75:
|
||||
return "bearish"
|
||||
if score < 40:
|
||||
return "bullish"
|
||||
if score > 60:
|
||||
return "bearish"
|
||||
return "neutral"
|
||||
|
||||
|
||||
def _build_key_points(
|
||||
fg: Dict[str, Any],
|
||||
naaim: Optional[float],
|
||||
aaii: Optional[Tuple[float, float, float]],
|
||||
) -> List[str]:
|
||||
points: List[str] = []
|
||||
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
prev_week = fg["previous_1_week"]
|
||||
wow_delta = score - prev_week
|
||||
|
||||
points.append(
|
||||
f"CNN Fear & Greed: {score:.0f}/100 ({rating.upper()}) — WoW {wow_delta:+.0f}pt"
|
||||
)
|
||||
|
||||
prev_close = fg["previous_close"]
|
||||
dod_delta = score - prev_close
|
||||
points.append(f"CNN F&G DoD change: {dod_delta:+.0f}pt (prev close {prev_close:.0f})")
|
||||
|
||||
if naaim is not None:
|
||||
naaim_level = "HIGH" if naaim > 80 else ("LOW" if naaim < 30 else "NORMAL")
|
||||
points.append(f"NAAIM Exposure Index: {naaim:.0f}/200 ({naaim_level})")
|
||||
if naaim > 80:
|
||||
points.append("NAAIM: Active managers at high equity exposure — limited incremental buying power")
|
||||
elif naaim < 30:
|
||||
points.append("NAAIM: Active managers heavily derisked — potential fuel for snapback rally")
|
||||
|
||||
if aaii is not None:
|
||||
bull, bear, neut = aaii
|
||||
points.append(f"AAII Bulls: {bull:.0f}% | Bears: {bear:.0f}% | Neutral: {neut:.0f}%")
|
||||
if bull > 40:
|
||||
points.append(f"AAII: Bullish sentiment elevated ({bull:.0f}%) — mild contrarian bearish signal")
|
||||
elif bull < 20:
|
||||
points.append(f"AAII: Extreme pessimism ({bull:.0f}% bulls) — contrarian bullish signal")
|
||||
if bear > 40:
|
||||
points.append(f"AAII: High bear reading ({bear:.0f}%) — contrarian bullish signal")
|
||||
|
||||
if score < 20:
|
||||
points.append("EXTREME FEAR: historically a strong mean-reversion buy signal (>80% 1M forward returns positive)")
|
||||
elif score > 80:
|
||||
points.append("EXTREME GREED: market complacency elevated — tail-risk hedging warranted")
|
||||
|
||||
return points
|
||||
|
||||
|
||||
def _build_implications(
|
||||
fg: Dict[str, Any],
|
||||
naaim: Optional[float],
|
||||
aaii: Optional[Tuple[float, float, float]],
|
||||
) -> List[str]:
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
implications: List[str] = []
|
||||
|
||||
if rating == "extreme_fear" or score < 20:
|
||||
implications.append(
|
||||
"Extreme Fear reading — historically strong contrarian buy signal for long delta"
|
||||
)
|
||||
elif rating == "extreme_greed" or score > 80:
|
||||
implications.append(
|
||||
"Extreme Greed — market may be overbought, consider protective puts or vol selling"
|
||||
)
|
||||
elif score < 40:
|
||||
implications.append(
|
||||
"Fear sentiment — elevated risk premium may support long vol or mean-reversion longs"
|
||||
)
|
||||
elif score > 60:
|
||||
implications.append(
|
||||
"Greed sentiment — consider trimming high-beta risk, watch for vol compression unwind"
|
||||
)
|
||||
else:
|
||||
implications.append(
|
||||
"Neutral sentiment — no contrarian edge; follow underlying trend and macro catalysts"
|
||||
)
|
||||
|
||||
if naaim is not None:
|
||||
if naaim > 80:
|
||||
implications.append(
|
||||
"Active managers fully exposed — limited buying power remaining, correction risk elevated"
|
||||
)
|
||||
elif naaim < 30:
|
||||
implications.append(
|
||||
"Active managers heavily derisked — potential fuel for snapback rally on positive catalyst"
|
||||
)
|
||||
|
||||
if aaii is not None:
|
||||
bull, bear, _ = aaii
|
||||
if bull > 40 and score > 60:
|
||||
implications.append(
|
||||
"AAII + CNN both elevated — dual sentiment warning, consider vol hedges"
|
||||
)
|
||||
elif bear > 40 and score < 40:
|
||||
implications.append(
|
||||
"AAII bears elevated + CNN fear — strong contrarian setup, watch for reversal catalyst"
|
||||
)
|
||||
|
||||
return implications
|
||||
|
||||
|
||||
def fetch_sentiment_report() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch CNN Fear & Greed (primary) + NAAIM + AAII (optional).
|
||||
Returns a single structured report dict compatible with institutional_reports table.
|
||||
Returns None only if CNN F&G fetch fails entirely.
|
||||
"""
|
||||
fg = _fetch_cnn_fear_greed()
|
||||
if fg is None:
|
||||
logger.error("[SENTIMENT] CNN Fear & Greed unavailable — aborting sentiment report")
|
||||
return None
|
||||
|
||||
naaim = _fetch_naaim()
|
||||
aaii = _fetch_aaii()
|
||||
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
|
||||
key_points = _build_key_points(fg, naaim, aaii)
|
||||
implications = _build_implications(fg, naaim, aaii)
|
||||
signal_indices = _signal_indices_from_fg(score)
|
||||
|
||||
importance = 3 if (score < 20 or score > 80) else 2
|
||||
|
||||
report_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
|
||||
sources = ["CNN Fear & Greed"]
|
||||
if naaim is not None:
|
||||
sources.append("NAAIM")
|
||||
if aaii is not None:
|
||||
sources.append("AAII")
|
||||
source_label = " + ".join(sources)
|
||||
|
||||
raw: Dict[str, Any] = {"cnn_fear_greed": fg}
|
||||
if naaim is not None:
|
||||
raw["naaim"] = {
|
||||
"exposure_index": naaim,
|
||||
"level": "HIGH" if naaim > 80 else ("LOW" if naaim < 30 else "NORMAL"),
|
||||
}
|
||||
if aaii is not None:
|
||||
bull, bear, neut = aaii
|
||||
raw["aaii"] = {"bullish_pct": bull, "bearish_pct": bear, "neutral_pct": neut}
|
||||
|
||||
ai_summary = (
|
||||
f"Market Sentiment ({report_date}). "
|
||||
f"CNN F&G: {score:.0f}/100 ({rating.upper()}), "
|
||||
f"WoW {score - fg['previous_1_week']:+.0f}pt. "
|
||||
+ (f"NAAIM: {naaim:.0f}/200. " if naaim is not None else "")
|
||||
+ (f"AAII Bulls: {aaii[0]:.0f}%. " if aaii is not None else "")
|
||||
+ f"Signal: {signal_indices.upper()}. "
|
||||
+ implications[0]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_type": "sentiment",
|
||||
"report_date": report_date,
|
||||
"title": f"Market Sentiment — CNN F&G {score:.0f}/100 ({rating.upper()}) — {report_date}",
|
||||
"source": source_label,
|
||||
"importance": importance,
|
||||
"category": "sentiment",
|
||||
"raw_data": raw,
|
||||
"key_points": key_points,
|
||||
"trading_implications": " | ".join(implications),
|
||||
"signal_energy": "neutral",
|
||||
"signal_metals": "neutral",
|
||||
"signal_indices": signal_indices,
|
||||
"signal_forex": "neutral",
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
Reference in New Issue
Block a user