feat: 4 remaining institutional reports — Earnings, VX curve, Central Bank RSS, Sentiment
New fetchers (no API keys required): - earnings_fetcher.py: yfinance EPS calendar + surprise tracking for 23 geo-relevant tickers - vx_fetcher.py: VIX term structure (^VIX/^VXV/^VXMT) + CBOE delayed futures, regime detection - central_bank_fetcher.py: Fed + ECB RSS feeds, keyword-based hawkish/dovish classification - sentiment_fetcher.py: CNN Fear & Greed (primary) + NAAIM + AAII (optional fallbacks) Wiring: - institutional_scheduler.py: all 4 now scheduled daily (≥08:00 UTC), deduplicated per day - institutional.py /refresh: all 6 types handled with _run() helper - ai_analyzer.py build_institutional_block(): limit 6→12, generic header text - InstitutionalReports.tsx: 6-type color map, individual refresh buttons, expanded filters Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -151,40 +151,49 @@ def get_report(report_id: int):
|
||||
|
||||
@router.post("/refresh")
|
||||
def refresh_reports(report_type: Optional[str] = Query(None)):
|
||||
"""Trigger a live fetch of COT and/or EIA reports."""
|
||||
"""Trigger a live fetch of any/all institutional report types."""
|
||||
results: Dict[str, str] = {}
|
||||
|
||||
if not report_type or report_type == "cot":
|
||||
def _run(label: str, fetch_fn, *args):
|
||||
try:
|
||||
from services.cot_fetcher import fetch_cot_report
|
||||
report = fetch_cot_report()
|
||||
report = fetch_fn(*args)
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
results["cot"] = "ok"
|
||||
logger.info(f"[Institutional] COT report saved: {report['report_date']}")
|
||||
results[label] = "ok"
|
||||
logger.info(f"[Institutional] {label} saved: {report['report_date']}")
|
||||
else:
|
||||
results["cot"] = "no_data"
|
||||
results[label] = "no_data"
|
||||
except Exception as e:
|
||||
logger.warning(f"[Institutional] COT refresh failed: {e}")
|
||||
results["cot"] = f"error: {str(e)[:100]}"
|
||||
logger.warning(f"[Institutional] {label} refresh failed: {e}")
|
||||
results[label] = f"error: {str(e)[:100]}"
|
||||
|
||||
if not report_type or report_type == "cot":
|
||||
from services.cot_fetcher import fetch_cot_report
|
||||
_run("cot", fetch_cot_report)
|
||||
|
||||
if not report_type or report_type == "eia":
|
||||
try:
|
||||
eia_key = get_config("eia_api_key") or ""
|
||||
if eia_key:
|
||||
from services.eia_fetcher import fetch_eia_report
|
||||
report = fetch_eia_report(eia_key)
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
results["eia"] = "ok"
|
||||
logger.info(f"[Institutional] EIA report saved: {report['report_date']}")
|
||||
else:
|
||||
results["eia"] = "no_data"
|
||||
else:
|
||||
results["eia"] = "no_api_key"
|
||||
except Exception as e:
|
||||
logger.warning(f"[Institutional] EIA refresh failed: {e}")
|
||||
results["eia"] = f"error: {str(e)[:100]}"
|
||||
eia_key = get_config("eia_api_key") or ""
|
||||
if eia_key:
|
||||
from services.eia_fetcher import fetch_eia_report
|
||||
_run("eia", fetch_eia_report, eia_key)
|
||||
else:
|
||||
results["eia"] = "no_api_key"
|
||||
|
||||
if not report_type or report_type == "earnings":
|
||||
from services.earnings_fetcher import fetch_earnings_report
|
||||
_run("earnings", fetch_earnings_report)
|
||||
|
||||
if not report_type or report_type == "vx_curve":
|
||||
from services.vx_fetcher import fetch_vx_report
|
||||
_run("vx_curve", fetch_vx_report)
|
||||
|
||||
if not report_type or report_type == "central_bank":
|
||||
from services.central_bank_fetcher import fetch_central_bank_reports
|
||||
_run("central_bank", fetch_central_bank_reports)
|
||||
|
||||
if not report_type or report_type == "sentiment":
|
||||
from services.sentiment_fetcher import fetch_sentiment_report
|
||||
_run("sentiment", fetch_sentiment_report)
|
||||
|
||||
return {"status": "done", "results": results}
|
||||
|
||||
|
||||
@@ -1208,7 +1208,7 @@ def build_institutional_block(days: int = 7) -> str:
|
||||
rows = conn.execute(
|
||||
"SELECT report_type, report_date, key_points_json, trading_implications, "
|
||||
"signal_energy, signal_metals, signal_indices, signal_forex, importance "
|
||||
"FROM institutional_reports WHERE report_date >= ? ORDER BY report_date DESC LIMIT 6",
|
||||
"FROM institutional_reports WHERE report_date >= ? ORDER BY importance DESC, report_date DESC LIMIT 12",
|
||||
(cutoff,),
|
||||
).fetchall()
|
||||
finally:
|
||||
@@ -1218,7 +1218,7 @@ def build_institutional_block(days: int = 7) -> str:
|
||||
return ""
|
||||
|
||||
import json as _json
|
||||
lines = ["## INSTITUTIONAL REPORTS (CFTC COT + EIA — last 7 days)"]
|
||||
lines = [f"## INSTITUTIONAL REPORTS (COT · EIA · Earnings · VX · Central Banks · Sentiment — last {days}d)"]
|
||||
for r in rows:
|
||||
rtype = r["report_type"].upper()
|
||||
rdate = r["report_date"]
|
||||
|
||||
300
backend/services/central_bank_fetcher.py
Normal file
300
backend/services/central_bank_fetcher.py
Normal file
@@ -0,0 +1,300 @@
|
||||
"""
|
||||
Central bank publication tracker — Fed + ECB RSS feeds.
|
||||
No API key required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FED_FEEDS: List[Tuple[str, str]] = [
|
||||
("https://www.federalreserve.gov/feeds/press_monetary.xml", "FOMC/Monetary Policy"),
|
||||
("https://www.federalreserve.gov/feeds/speeches.xml", "Fed Speeches"),
|
||||
]
|
||||
|
||||
ECB_FEEDS: List[Tuple[str, str]] = [
|
||||
("https://www.ecb.europa.eu/rss/press.html", "ECB Press"),
|
||||
("https://www.ecb.europa.eu/press/govcounc/monpol/html/index.en.html", "ECB Monetary Policy"),
|
||||
]
|
||||
|
||||
MONETARY_KEYWORDS = [
|
||||
"rate", "policy", "inflation", "fomc", "minutes", "statement",
|
||||
"monetary", "decision", "outlook", "hike", "cut", "pause", "hold",
|
||||
"employment", "balance sheet", "quantitative",
|
||||
]
|
||||
|
||||
HAWKISH_WORDS = ["hike", "hawkish", "raised", "tightening", "tighter"]
|
||||
DOVISH_WORDS = ["cut", "dovish", "easing", "lowered", "accommodative"]
|
||||
|
||||
_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; GeoOptions/1.0)"}
|
||||
_LOOKBACK_DAYS = 14
|
||||
_MAX_KEY_POINTS = 8
|
||||
|
||||
_ATOM_NS = "http://www.w3.org/2005/Atom"
|
||||
|
||||
|
||||
def _parse_date(raw: Optional[str]) -> Optional[datetime]:
|
||||
if not raw:
|
||||
return None
|
||||
raw = raw.strip()
|
||||
try:
|
||||
dt = parsedate_to_datetime(raw)
|
||||
return dt.astimezone(timezone.utc)
|
||||
except Exception:
|
||||
pass
|
||||
for fmt in ("%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S%z", "%Y-%m-%d"):
|
||||
try:
|
||||
dt = datetime.strptime(raw[: len(fmt)], fmt)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
logger.debug(f"[CB] Could not parse date: {raw!r}")
|
||||
return None
|
||||
|
||||
|
||||
def _text(element: Optional[ET.Element]) -> str:
|
||||
if element is None:
|
||||
return ""
|
||||
return (element.text or "").strip()
|
||||
|
||||
|
||||
def _fetch_feed(url: str, label: str) -> List[Dict[str, str]]:
|
||||
try:
|
||||
resp = requests.get(url, timeout=15, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
logger.warning(f"[CB] Failed to fetch feed {label} ({url}): {e}")
|
||||
return []
|
||||
|
||||
try:
|
||||
root = ET.fromstring(resp.text)
|
||||
except ET.ParseError as e:
|
||||
logger.warning(f"[CB] XML parse error for {label}: {e}")
|
||||
return []
|
||||
|
||||
items: List[Dict[str, str]] = []
|
||||
tag = root.tag
|
||||
|
||||
if "Atom" in tag or tag.endswith("}feed"):
|
||||
for entry in root.findall(f"{{{_ATOM_NS}}}entry"):
|
||||
title_el = entry.find(f"{{{_ATOM_NS}}}title")
|
||||
link_el = entry.find(f"{{{_ATOM_NS}}}link")
|
||||
updated_el = entry.find(f"{{{_ATOM_NS}}}updated")
|
||||
summary_el = entry.find(f"{{{_ATOM_NS}}}summary")
|
||||
content_el = entry.find(f"{{{_ATOM_NS}}}content")
|
||||
|
||||
link_href = ""
|
||||
if link_el is not None:
|
||||
link_href = link_el.get("href", "") or _text(link_el)
|
||||
|
||||
description = _text(summary_el) or _text(content_el)
|
||||
|
||||
items.append({
|
||||
"title": _text(title_el),
|
||||
"link": link_href,
|
||||
"pub_date_raw": _text(updated_el),
|
||||
"description": description[:500],
|
||||
"label": label,
|
||||
})
|
||||
else:
|
||||
channel = root.find("channel") or root
|
||||
for item in channel.findall("item"):
|
||||
items.append({
|
||||
"title": _text(item.find("title")),
|
||||
"link": _text(item.find("link")),
|
||||
"pub_date_raw": _text(item.find("pubDate")),
|
||||
"description": _text(item.find("description"))[:500],
|
||||
"label": label,
|
||||
})
|
||||
|
||||
logger.info(f"[CB] Fetched {len(items)} items from {label}")
|
||||
return items
|
||||
|
||||
|
||||
def _is_relevant(title: str, description: str) -> bool:
|
||||
text = (title + " " + description).lower()
|
||||
return any(kw in text for kw in MONETARY_KEYWORDS)
|
||||
|
||||
|
||||
def _within_lookback(pub_date_raw: str, days: int = _LOOKBACK_DAYS) -> bool:
|
||||
dt = _parse_date(pub_date_raw)
|
||||
if dt is None:
|
||||
return True
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(days=days)
|
||||
return dt >= cutoff
|
||||
|
||||
|
||||
_RATE_RE = re.compile(r"(\d+\.?\d*)\s*%")
|
||||
_ACTION_RE = re.compile(r"\b(raised|lowered|held\s+steady|unanimous|dissent)\b", re.IGNORECASE)
|
||||
_TARGET_RE = re.compile(r"\b(2%\s+inflation\s+target|maximum\s+employment)\b", re.IGNORECASE)
|
||||
_GUIDANCE_RE = re.compile(r"\b(gradual|data[-\s]dependent|patient|vigilant)\b", re.IGNORECASE)
|
||||
|
||||
|
||||
def _extract_key_points(items: List[Dict[str, str]]) -> List[str]:
|
||||
points: List[str] = []
|
||||
for item in items:
|
||||
label_lower = item["label"].lower()
|
||||
prefix = "[FED]" if ("fed" in label_lower or "fomc" in label_lower) else "[ECB]"
|
||||
title = item["title"]
|
||||
combined = f"{title} {item['description']}"
|
||||
|
||||
headline = f"{prefix} {title[:100]}"
|
||||
|
||||
details: List[str] = []
|
||||
rate_matches = _RATE_RE.findall(combined)
|
||||
if rate_matches:
|
||||
details.append(f"Rate ref: {', '.join(set(rate_matches[:3]))}%")
|
||||
action = _ACTION_RE.search(combined)
|
||||
if action:
|
||||
details.append(action.group(0).lower())
|
||||
target = _TARGET_RE.search(combined)
|
||||
if target:
|
||||
details.append(target.group(0))
|
||||
guidance = _GUIDANCE_RE.search(combined)
|
||||
if guidance:
|
||||
details.append(guidance.group(0).lower())
|
||||
|
||||
if details:
|
||||
headline += f" [{'; '.join(details)}]"
|
||||
|
||||
points.append(headline)
|
||||
if len(points) >= _MAX_KEY_POINTS:
|
||||
break
|
||||
|
||||
return points
|
||||
|
||||
|
||||
def _derive_signals(items: List[Dict[str, str]]) -> Tuple[str, str]:
|
||||
combined = " ".join(i["title"] for i in items).lower()
|
||||
|
||||
hawkish_hit = any(w in combined for w in HAWKISH_WORDS)
|
||||
dovish_hit = any(w in combined for w in DOVISH_WORDS)
|
||||
|
||||
if hawkish_hit and not dovish_hit:
|
||||
return "bullish", "bearish"
|
||||
if dovish_hit and not hawkish_hit:
|
||||
return "bearish", "bullish"
|
||||
return "neutral", "neutral"
|
||||
|
||||
|
||||
def fetch_central_bank_reports() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch Fed + ECB RSS feeds and return a single structured report dict
|
||||
compatible with the institutional_reports table.
|
||||
Returns None if no relevant items are found within the lookback window.
|
||||
"""
|
||||
all_items: List[Dict[str, str]] = []
|
||||
|
||||
for url, label in FED_FEEDS + ECB_FEEDS:
|
||||
raw_items = _fetch_feed(url, label)
|
||||
for item in raw_items:
|
||||
if _within_lookback(item["pub_date_raw"]) and _is_relevant(
|
||||
item["title"], item["description"]
|
||||
):
|
||||
all_items.append(item)
|
||||
|
||||
if not all_items:
|
||||
logger.warning("[CB] No relevant central bank items found in last 14 days")
|
||||
return None
|
||||
|
||||
seen: set = set()
|
||||
unique_items: List[Dict[str, str]] = []
|
||||
for item in all_items:
|
||||
key = item["title"].strip().lower()[:80]
|
||||
if key not in seen:
|
||||
seen.add(key)
|
||||
unique_items.append(item)
|
||||
|
||||
report_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
key_points = _extract_key_points(unique_items)
|
||||
signal_forex, signal_indices = _derive_signals(unique_items)
|
||||
|
||||
implications: List[str] = []
|
||||
combined_titles = " ".join(i["title"] for i in unique_items).lower()
|
||||
|
||||
if signal_forex == "bullish":
|
||||
implications.append(
|
||||
"Hawkish Fed/ECB rhetoric — supportive of USD strength, headwind for risk assets"
|
||||
)
|
||||
elif signal_forex == "bearish":
|
||||
implications.append(
|
||||
"Dovish pivot signals — USD pressure, potential tailwind for equities and EM"
|
||||
)
|
||||
|
||||
if "balance sheet" in combined_titles or "quantitative" in combined_titles:
|
||||
implications.append(
|
||||
"Balance sheet / QT language present — watch long-end rates and credit spreads"
|
||||
)
|
||||
if "inflation" in combined_titles:
|
||||
implications.append(
|
||||
"Inflation remains a focal point — monitor breakevens and real yields"
|
||||
)
|
||||
if "minutes" in combined_titles:
|
||||
implications.append(
|
||||
"FOMC/ECB minutes release — expect intra-day vol spike on language parsing"
|
||||
)
|
||||
|
||||
if not implications:
|
||||
implications = ["Central bank communication in focus — monitor for forward guidance shifts"]
|
||||
|
||||
fed_count = sum(
|
||||
1 for i in unique_items
|
||||
if "fed" in i["label"].lower() or "fomc" in i["label"].lower()
|
||||
)
|
||||
ecb_count = len(unique_items) - fed_count
|
||||
source_parts: List[str] = []
|
||||
if fed_count:
|
||||
source_parts.append(f"Fed ({fed_count})")
|
||||
if ecb_count:
|
||||
source_parts.append(f"ECB ({ecb_count})")
|
||||
source_label = " + ".join(source_parts) + " via RSS" if source_parts else "Fed/ECB RSS"
|
||||
|
||||
importance = 3 if any(
|
||||
w in combined_titles for w in ["hike", "cut", "minutes", "decision", "statement"]
|
||||
) else 2
|
||||
|
||||
ai_summary = (
|
||||
f"Central Bank Watch ({report_date}). "
|
||||
f"{len(unique_items)} relevant items: {source_label}. "
|
||||
f"USD signal: {signal_forex.upper()}, Indices: {signal_indices.upper()}. "
|
||||
+ implications[0]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_type": "central_bank",
|
||||
"report_date": report_date,
|
||||
"title": f"Central Bank Monitor — Fed + ECB — {report_date}",
|
||||
"source": source_label,
|
||||
"importance": importance,
|
||||
"category": "macro",
|
||||
"raw_data": {
|
||||
"items": [
|
||||
{
|
||||
"title": i["title"],
|
||||
"link": i["link"],
|
||||
"pub_date": i["pub_date_raw"],
|
||||
"label": i["label"],
|
||||
}
|
||||
for i in unique_items
|
||||
],
|
||||
"fed_count": fed_count,
|
||||
"ecb_count": ecb_count,
|
||||
},
|
||||
"key_points": key_points,
|
||||
"trading_implications": " | ".join(implications),
|
||||
"signal_energy": "neutral",
|
||||
"signal_metals": "neutral",
|
||||
"signal_indices": signal_indices,
|
||||
"signal_forex": signal_forex,
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
228
backend/services/earnings_fetcher.py
Normal file
228
backend/services/earnings_fetcher.py
Normal file
@@ -0,0 +1,228 @@
|
||||
"""
|
||||
Earnings calendar + EPS surprise tracker.
|
||||
Uses yfinance — no API key required.
|
||||
"""
|
||||
import logging
|
||||
import math
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pandas as pd
|
||||
import yfinance as yf
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
EARNINGS_TICKERS: Dict[str, List[str]] = {
|
||||
"energy": ["XOM", "CVX", "COP", "SLB", "MPC"],
|
||||
"metals": ["NEM", "FCX", "AA", "GOLD", "X"],
|
||||
"defense": ["RTX", "LMT", "GD", "NOC", "BA"],
|
||||
"banks": ["JPM", "GS", "BAC", "C"],
|
||||
"indices": ["AAPL", "MSFT", "NVDA", "AMZN", "META"],
|
||||
}
|
||||
|
||||
BEAT_THRESHOLD = 5.0
|
||||
MISS_THRESHOLD = -5.0
|
||||
UPCOMING_DAYS = 30
|
||||
HISTORY_DAYS = 90
|
||||
|
||||
|
||||
def _safe_float(val: Any) -> Optional[float]:
|
||||
try:
|
||||
f = float(val)
|
||||
if math.isnan(f):
|
||||
return None
|
||||
return f
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_ticker_earnings(ticker: str) -> Optional[Dict]:
|
||||
try:
|
||||
t = yf.Ticker(ticker)
|
||||
df = t.earnings_dates
|
||||
if df is None or df.empty:
|
||||
return None
|
||||
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
cutoff_future = now + timedelta(days=UPCOMING_DAYS)
|
||||
cutoff_past = now - timedelta(days=HISTORY_DAYS)
|
||||
|
||||
upcoming = []
|
||||
surprises = []
|
||||
|
||||
for idx, row in df.iterrows():
|
||||
try:
|
||||
date_ts = pd.Timestamp(idx)
|
||||
if date_ts.tzinfo is None:
|
||||
date_ts = date_ts.tz_localize("UTC")
|
||||
else:
|
||||
date_ts = date_ts.tz_convert("UTC")
|
||||
|
||||
date_dt = date_ts.to_pydatetime()
|
||||
date_str = date_ts.strftime("%Y-%m-%d")
|
||||
|
||||
if date_dt > now and date_dt <= cutoff_future:
|
||||
eps_est = _safe_float(row.get("EPS Estimate"))
|
||||
upcoming.append({
|
||||
"ticker": ticker,
|
||||
"date": date_str,
|
||||
"eps_estimate": eps_est,
|
||||
})
|
||||
|
||||
elif date_dt <= now and date_dt >= cutoff_past:
|
||||
surprise_pct = _safe_float(row.get("Surprise(%)"))
|
||||
reported_eps = _safe_float(row.get("Reported EPS"))
|
||||
eps_est = _safe_float(row.get("EPS Estimate"))
|
||||
if surprise_pct is None:
|
||||
continue
|
||||
surprises.append({
|
||||
"ticker": ticker,
|
||||
"date": date_str,
|
||||
"surprise_pct": surprise_pct,
|
||||
"reported_eps": reported_eps,
|
||||
"eps_estimate": eps_est,
|
||||
})
|
||||
except Exception as e:
|
||||
logger.debug(f"[EARNINGS] Row error {ticker}: {e}")
|
||||
continue
|
||||
|
||||
return {"upcoming": upcoming, "surprises": surprises}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[EARNINGS] Failed to fetch {ticker}: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_earnings_report() -> Optional[Dict]:
|
||||
"""Fetch earnings calendar and EPS surprises for geo-relevant tickers. Returns None if no data."""
|
||||
now = datetime.now(tz=timezone.utc)
|
||||
report_date = now.strftime("%Y-%m-%d")
|
||||
|
||||
all_upcoming: List[Dict] = []
|
||||
all_surprises: List[Dict] = []
|
||||
raw_data: Dict[str, Any] = {}
|
||||
|
||||
for sector, tickers in EARNINGS_TICKERS.items():
|
||||
for ticker in tickers:
|
||||
result = _fetch_ticker_earnings(ticker)
|
||||
if result is None:
|
||||
continue
|
||||
raw_data[ticker] = {**result, "sector": sector}
|
||||
all_upcoming.extend(result["upcoming"])
|
||||
all_surprises.extend(result["surprises"])
|
||||
|
||||
if not all_upcoming and not all_surprises:
|
||||
logger.warning("[EARNINGS] No earnings data returned")
|
||||
return None
|
||||
|
||||
beats = [s for s in all_surprises if s["surprise_pct"] > BEAT_THRESHOLD]
|
||||
misses = [s for s in all_surprises if s["surprise_pct"] < MISS_THRESHOLD]
|
||||
n_beats = len(beats)
|
||||
n_misses = len(misses)
|
||||
|
||||
all_upcoming_sorted = sorted(all_upcoming, key=lambda x: x["date"])
|
||||
beats_sorted = sorted(beats, key=lambda x: abs(x["surprise_pct"]), reverse=True)
|
||||
misses_sorted = sorted(misses, key=lambda x: abs(x["surprise_pct"]), reverse=True)
|
||||
|
||||
key_points: List[str] = []
|
||||
|
||||
if all_upcoming_sorted:
|
||||
upcoming_strs = [
|
||||
f"{u['ticker']} ({u['date']})"
|
||||
+ (f" EPS est. {u['eps_estimate']:.2f}" if u["eps_estimate"] is not None else "")
|
||||
for u in all_upcoming_sorted[:5]
|
||||
]
|
||||
key_points.append(f"Upcoming earnings (next 30d): {', '.join(upcoming_strs)}")
|
||||
|
||||
if beats_sorted:
|
||||
beat_strs = [
|
||||
f"{b['ticker']} +{b['surprise_pct']:.1f}% ({b['date']})"
|
||||
for b in beats_sorted[:3]
|
||||
]
|
||||
key_points.append(f"Recent beats: {', '.join(beat_strs)}")
|
||||
|
||||
if misses_sorted:
|
||||
miss_strs = [
|
||||
f"{m['ticker']} {m['surprise_pct']:.1f}% ({m['date']})"
|
||||
for m in misses_sorted[:3]
|
||||
]
|
||||
key_points.append(f"Recent misses: {', '.join(miss_strs)}")
|
||||
|
||||
if not beats and not misses:
|
||||
key_points.append("No significant EPS surprises (>5%) in the past 90 days")
|
||||
|
||||
sector_beats: Dict[str, int] = {}
|
||||
sector_misses: Dict[str, int] = {}
|
||||
for b in beats:
|
||||
ticker = b["ticker"]
|
||||
for sector, tickers in EARNINGS_TICKERS.items():
|
||||
if ticker in tickers:
|
||||
sector_beats[sector] = sector_beats.get(sector, 0) + 1
|
||||
for m in misses:
|
||||
ticker = m["ticker"]
|
||||
for sector, tickers in EARNINGS_TICKERS.items():
|
||||
if ticker in tickers:
|
||||
sector_misses[sector] = sector_misses.get(sector, 0) + 1
|
||||
|
||||
if n_beats > n_misses * 1.5 and n_beats > 0:
|
||||
signal_indices = "bullish"
|
||||
elif n_misses > n_beats * 1.5 and n_misses > 0:
|
||||
signal_indices = "bearish"
|
||||
else:
|
||||
signal_indices = "neutral"
|
||||
|
||||
implications: List[str] = []
|
||||
|
||||
if all_upcoming_sorted:
|
||||
sectors_upcoming = set()
|
||||
for u in all_upcoming_sorted:
|
||||
for sector, tickers in EARNINGS_TICKERS.items():
|
||||
if u["ticker"] in tickers:
|
||||
sectors_upcoming.add(sector)
|
||||
implications.append(
|
||||
f"Earnings season active — {len(all_upcoming_sorted)} reports due "
|
||||
f"in next 30d across {', '.join(sorted(sectors_upcoming))}"
|
||||
)
|
||||
|
||||
dominant_beat_sector = max(sector_beats, key=sector_beats.get) if sector_beats else None
|
||||
dominant_miss_sector = max(sector_misses, key=sector_misses.get) if sector_misses else None
|
||||
|
||||
if dominant_beat_sector:
|
||||
implications.append(
|
||||
f"{dominant_beat_sector.capitalize()} sector leading on beats "
|
||||
f"({sector_beats[dominant_beat_sector]} beat(s)) — positive momentum"
|
||||
)
|
||||
if dominant_miss_sector and dominant_miss_sector != dominant_beat_sector:
|
||||
implications.append(
|
||||
f"{dominant_miss_sector.capitalize()} sector showing misses "
|
||||
f"({sector_misses[dominant_miss_sector]} miss(es)) — watch for guidance cuts"
|
||||
)
|
||||
if not implications:
|
||||
implications = ["Earnings data broadly in line — no sector concentration signal"]
|
||||
|
||||
importance = 3 if (n_beats > 3 or n_misses > 3) else 2
|
||||
|
||||
ai_summary = (
|
||||
f"Earnings tracker ({report_date}). "
|
||||
f"Upcoming (30d): {len(all_upcoming_sorted)} events. "
|
||||
f"Beats: {n_beats}, Misses: {n_misses}. "
|
||||
f"Signal: {signal_indices.upper()}. "
|
||||
+ implications[0]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_type": "earnings",
|
||||
"report_date": report_date,
|
||||
"title": f"Earnings Calendar & EPS Surprises — {report_date}",
|
||||
"source": "yfinance",
|
||||
"importance": importance,
|
||||
"category": "equities",
|
||||
"raw_data": raw_data,
|
||||
"key_points": key_points,
|
||||
"trading_implications": " | ".join(implications),
|
||||
"signal_energy": "neutral",
|
||||
"signal_metals": "neutral",
|
||||
"signal_indices": signal_indices,
|
||||
"signal_forex": "neutral",
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
@@ -1,8 +1,9 @@
|
||||
"""
|
||||
Weekly scheduler for institutional reports:
|
||||
Scheduler for institutional reports:
|
||||
- CFTC COT: released every Friday ~15:30 ET → fetch Saturday UTC
|
||||
- EIA Petroleum Weekly: released every Wednesday ~10:30 ET → fetch Wednesday afternoon UTC
|
||||
Checks once per hour; only fetches when the day matches and report not yet fetched this week.
|
||||
- Earnings / VX curve / Central bank RSS / Sentiment: fetch daily (after 8am UTC)
|
||||
Checks once per hour; only fetches when the day matches and report not yet fetched today.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
@@ -17,17 +18,20 @@ _CHECK_INTERVAL_S = 3600 # check every hour
|
||||
|
||||
|
||||
def _should_fetch_cot() -> bool:
|
||||
"""Saturday UTC = day after COT release."""
|
||||
now = datetime.utcnow()
|
||||
return now.weekday() == 5 # Saturday
|
||||
|
||||
|
||||
def _should_fetch_eia() -> bool:
|
||||
"""Wednesday afternoon UTC."""
|
||||
now = datetime.utcnow()
|
||||
return now.weekday() == 2 and now.hour >= 16 # Wednesday ≥16:00 UTC
|
||||
|
||||
|
||||
def _should_fetch_daily() -> bool:
|
||||
"""Earnings, VX, central bank, sentiment — fetch once per day after 8am UTC."""
|
||||
return datetime.utcnow().hour >= 8
|
||||
|
||||
|
||||
def _last_fetch_date(report_type: str) -> Optional[str]:
|
||||
try:
|
||||
from services.database import get_conn
|
||||
@@ -44,44 +48,67 @@ def _last_fetch_date(report_type: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_and_save(label: str, fetch_fn, *args):
|
||||
"""Generic helper: call fetch_fn(*args), save result if not None."""
|
||||
try:
|
||||
from routers.institutional import save_institutional_report
|
||||
report = fetch_fn(*args)
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
logger.info(f"[InstitutionalScheduler] {label} saved: {report['report_date']}")
|
||||
else:
|
||||
logger.info(f"[InstitutionalScheduler] {label} returned no data")
|
||||
except Exception as e:
|
||||
logger.warning(f"[InstitutionalScheduler] {label} fetch failed: {e}")
|
||||
|
||||
|
||||
def _run_loop():
|
||||
logger.info("[InstitutionalScheduler] Started")
|
||||
while not _stop_event.is_set():
|
||||
try:
|
||||
today = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
|
||||
if _should_fetch_cot():
|
||||
last = _last_fetch_date("cot")
|
||||
if last != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching COT...")
|
||||
try:
|
||||
from services.cot_fetcher import fetch_cot_report
|
||||
from routers.institutional import save_institutional_report
|
||||
report = fetch_cot_report()
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
logger.info(f"[InstitutionalScheduler] COT saved: {report['report_date']}")
|
||||
except Exception as e:
|
||||
logger.warning(f"[InstitutionalScheduler] COT fetch failed: {e}")
|
||||
# ── Weekly: COT (Saturday) ──────────────────────────────────────
|
||||
if _should_fetch_cot() and _last_fetch_date("cot") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching COT...")
|
||||
from services.cot_fetcher import fetch_cot_report
|
||||
_fetch_and_save("COT", fetch_cot_report)
|
||||
|
||||
if _should_fetch_eia():
|
||||
last = _last_fetch_date("eia")
|
||||
if last != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching EIA...")
|
||||
try:
|
||||
from services.database import get_config
|
||||
from services.eia_fetcher import fetch_eia_report
|
||||
from routers.institutional import save_institutional_report
|
||||
key = get_config("eia_api_key") or ""
|
||||
if key:
|
||||
report = fetch_eia_report(key)
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
logger.info(f"[InstitutionalScheduler] EIA saved: {report['report_date']}")
|
||||
else:
|
||||
logger.info("[InstitutionalScheduler] EIA skipped — no API key configured")
|
||||
except Exception as e:
|
||||
logger.warning(f"[InstitutionalScheduler] EIA fetch failed: {e}")
|
||||
# ── Weekly: EIA (Wednesday ≥16:00 UTC) ─────────────────────────
|
||||
if _should_fetch_eia() and _last_fetch_date("eia") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching EIA...")
|
||||
try:
|
||||
from services.database import get_config
|
||||
from services.eia_fetcher import fetch_eia_report
|
||||
key = get_config("eia_api_key") or ""
|
||||
if key:
|
||||
_fetch_and_save("EIA", fetch_eia_report, key)
|
||||
else:
|
||||
logger.info("[InstitutionalScheduler] EIA skipped — no API key configured")
|
||||
except Exception as e:
|
||||
logger.warning(f"[InstitutionalScheduler] EIA fetch failed: {e}")
|
||||
|
||||
# ── Daily: Earnings, VX curve, Central banks, Sentiment ─────────
|
||||
if _should_fetch_daily():
|
||||
if _last_fetch_date("earnings") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching Earnings...")
|
||||
from services.earnings_fetcher import fetch_earnings_report
|
||||
_fetch_and_save("Earnings", fetch_earnings_report)
|
||||
|
||||
if _last_fetch_date("vx_curve") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching VX term structure...")
|
||||
from services.vx_fetcher import fetch_vx_report
|
||||
_fetch_and_save("VX", fetch_vx_report)
|
||||
|
||||
if _last_fetch_date("central_bank") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching Central Bank RSS...")
|
||||
from services.central_bank_fetcher import fetch_central_bank_reports
|
||||
_fetch_and_save("CentralBank", fetch_central_bank_reports)
|
||||
|
||||
if _last_fetch_date("sentiment") != today:
|
||||
logger.info("[InstitutionalScheduler] Fetching Sentiment...")
|
||||
from services.sentiment_fetcher import fetch_sentiment_report
|
||||
_fetch_and_save("Sentiment", fetch_sentiment_report)
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[InstitutionalScheduler] Loop error: {e}")
|
||||
|
||||
315
backend/services/sentiment_fetcher.py
Normal file
315
backend/services/sentiment_fetcher.py
Normal file
@@ -0,0 +1,315 @@
|
||||
"""
|
||||
Market sentiment tracker — CNN Fear & Greed + NAAIM + AAII.
|
||||
All free/public sources. No API key required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CNN_FG_URL = "https://production.dataviz.cnn.io/index/fearandgreed/graphdata/"
|
||||
|
||||
NAAIM_URLS = [
|
||||
"https://www.naaim.org/wp-content/uploads/NAAIM-Exposure-Index-Weekly-Data.csv",
|
||||
"https://www.naaim.org/programs/naaim-exposure-index/",
|
||||
]
|
||||
|
||||
AAII_URL = "https://www.aaii.com/sentimentsurvey/sent_results.js"
|
||||
|
||||
_HEADERS = {"User-Agent": "Mozilla/5.0 (compatible; GeoOptions/1.0)"}
|
||||
_TIMEOUT = 15
|
||||
|
||||
|
||||
def _fetch_cnn_fear_greed() -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
resp = requests.get(CNN_FG_URL, timeout=_TIMEOUT, headers=_HEADERS)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
except Exception as e:
|
||||
logger.warning(f"[SENTIMENT] CNN F&G fetch failed: {e}")
|
||||
return None
|
||||
|
||||
fg = data.get("fear_and_greed", {})
|
||||
try:
|
||||
score = float(fg.get("score", 0))
|
||||
rating = str(fg.get("rating", "unknown")).lower()
|
||||
prev_close = float(fg.get("previous_close", score))
|
||||
prev_1_week = float(fg.get("previous_1_week", score))
|
||||
prev_1_month = float(fg.get("previous_1_month", score))
|
||||
prev_1_year = float(fg.get("previous_1_year", score))
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.warning(f"[SENTIMENT] CNN F&G parse error: {e}")
|
||||
return None
|
||||
|
||||
logger.info(f"[SENTIMENT] CNN F&G: {score:.1f} / 100 ({rating})")
|
||||
return {
|
||||
"score": score,
|
||||
"rating": rating,
|
||||
"previous_close": prev_close,
|
||||
"previous_1_week": prev_1_week,
|
||||
"previous_1_month": prev_1_month,
|
||||
"previous_1_year": prev_1_year,
|
||||
}
|
||||
|
||||
|
||||
_NAAIM_NUMBER_RE = re.compile(r"(\d+\.?\d*)\s*(?:Average|Mean|Exposure)", re.IGNORECASE)
|
||||
|
||||
|
||||
def _parse_naaim_csv(text: str) -> Optional[float]:
|
||||
lines = [l.strip() for l in text.splitlines() if l.strip()]
|
||||
if len(lines) < 2:
|
||||
return None
|
||||
data_line = lines[1]
|
||||
parts = re.split(r"[,\t]", data_line)
|
||||
for part in reversed(parts):
|
||||
try:
|
||||
val = float(part.strip())
|
||||
if 0.0 <= val <= 200.0:
|
||||
return val
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_naaim() -> Optional[float]:
|
||||
try:
|
||||
resp = requests.get(NAAIM_URLS[0], timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code == 200 and resp.text.strip():
|
||||
val = _parse_naaim_csv(resp.text)
|
||||
if val is not None:
|
||||
logger.info(f"[SENTIMENT] NAAIM exposure (CSV): {val:.1f}")
|
||||
return val
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] NAAIM CSV failed: {e}")
|
||||
|
||||
try:
|
||||
resp = requests.get(NAAIM_URLS[1], timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code == 200:
|
||||
m = _NAAIM_NUMBER_RE.search(resp.text)
|
||||
if m:
|
||||
val = float(m.group(1))
|
||||
logger.info(f"[SENTIMENT] NAAIM exposure (HTML): {val:.1f}")
|
||||
return val
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] NAAIM HTML failed: {e}")
|
||||
|
||||
logger.info("[SENTIMENT] NAAIM not available — skipping")
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_aaii() -> Optional[Tuple[float, float, float]]:
|
||||
try:
|
||||
resp = requests.get(AAII_URL, timeout=_TIMEOUT, headers=_HEADERS)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
text = resp.text
|
||||
except Exception as e:
|
||||
logger.debug(f"[SENTIMENT] AAII fetch failed: {e}")
|
||||
return None
|
||||
|
||||
try:
|
||||
bull_m = re.search(r'"bullish"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
bear_m = re.search(r'"bearish"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
neut_m = re.search(r'"neutral"\s*:\s*([\d.]+)', text, re.IGNORECASE)
|
||||
|
||||
if bull_m and bear_m:
|
||||
bull = float(bull_m.group(1))
|
||||
bear = float(bear_m.group(1))
|
||||
neut = float(neut_m.group(1)) if neut_m else max(0.0, 100.0 - bull - bear)
|
||||
|
||||
if 0 <= bull <= 100 and 0 <= bear <= 100:
|
||||
logger.info(f"[SENTIMENT] AAII Bulls={bull:.1f}% Bears={bear:.1f}%")
|
||||
return bull, bear, neut
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.debug(f"[SENTIMENT] AAII parse error: {e}")
|
||||
|
||||
logger.info("[SENTIMENT] AAII not available — skipping")
|
||||
return None
|
||||
|
||||
|
||||
def _signal_indices_from_fg(score: float) -> str:
|
||||
if score < 25:
|
||||
return "bullish"
|
||||
if score > 75:
|
||||
return "bearish"
|
||||
if score < 40:
|
||||
return "bullish"
|
||||
if score > 60:
|
||||
return "bearish"
|
||||
return "neutral"
|
||||
|
||||
|
||||
def _build_key_points(
|
||||
fg: Dict[str, Any],
|
||||
naaim: Optional[float],
|
||||
aaii: Optional[Tuple[float, float, float]],
|
||||
) -> List[str]:
|
||||
points: List[str] = []
|
||||
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
prev_week = fg["previous_1_week"]
|
||||
wow_delta = score - prev_week
|
||||
|
||||
points.append(
|
||||
f"CNN Fear & Greed: {score:.0f}/100 ({rating.upper()}) — WoW {wow_delta:+.0f}pt"
|
||||
)
|
||||
|
||||
prev_close = fg["previous_close"]
|
||||
dod_delta = score - prev_close
|
||||
points.append(f"CNN F&G DoD change: {dod_delta:+.0f}pt (prev close {prev_close:.0f})")
|
||||
|
||||
if naaim is not None:
|
||||
naaim_level = "HIGH" if naaim > 80 else ("LOW" if naaim < 30 else "NORMAL")
|
||||
points.append(f"NAAIM Exposure Index: {naaim:.0f}/200 ({naaim_level})")
|
||||
if naaim > 80:
|
||||
points.append("NAAIM: Active managers at high equity exposure — limited incremental buying power")
|
||||
elif naaim < 30:
|
||||
points.append("NAAIM: Active managers heavily derisked — potential fuel for snapback rally")
|
||||
|
||||
if aaii is not None:
|
||||
bull, bear, neut = aaii
|
||||
points.append(f"AAII Bulls: {bull:.0f}% | Bears: {bear:.0f}% | Neutral: {neut:.0f}%")
|
||||
if bull > 40:
|
||||
points.append(f"AAII: Bullish sentiment elevated ({bull:.0f}%) — mild contrarian bearish signal")
|
||||
elif bull < 20:
|
||||
points.append(f"AAII: Extreme pessimism ({bull:.0f}% bulls) — contrarian bullish signal")
|
||||
if bear > 40:
|
||||
points.append(f"AAII: High bear reading ({bear:.0f}%) — contrarian bullish signal")
|
||||
|
||||
if score < 20:
|
||||
points.append("EXTREME FEAR: historically a strong mean-reversion buy signal (>80% 1M forward returns positive)")
|
||||
elif score > 80:
|
||||
points.append("EXTREME GREED: market complacency elevated — tail-risk hedging warranted")
|
||||
|
||||
return points
|
||||
|
||||
|
||||
def _build_implications(
|
||||
fg: Dict[str, Any],
|
||||
naaim: Optional[float],
|
||||
aaii: Optional[Tuple[float, float, float]],
|
||||
) -> List[str]:
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
implications: List[str] = []
|
||||
|
||||
if rating == "extreme_fear" or score < 20:
|
||||
implications.append(
|
||||
"Extreme Fear reading — historically strong contrarian buy signal for long delta"
|
||||
)
|
||||
elif rating == "extreme_greed" or score > 80:
|
||||
implications.append(
|
||||
"Extreme Greed — market may be overbought, consider protective puts or vol selling"
|
||||
)
|
||||
elif score < 40:
|
||||
implications.append(
|
||||
"Fear sentiment — elevated risk premium may support long vol or mean-reversion longs"
|
||||
)
|
||||
elif score > 60:
|
||||
implications.append(
|
||||
"Greed sentiment — consider trimming high-beta risk, watch for vol compression unwind"
|
||||
)
|
||||
else:
|
||||
implications.append(
|
||||
"Neutral sentiment — no contrarian edge; follow underlying trend and macro catalysts"
|
||||
)
|
||||
|
||||
if naaim is not None:
|
||||
if naaim > 80:
|
||||
implications.append(
|
||||
"Active managers fully exposed — limited buying power remaining, correction risk elevated"
|
||||
)
|
||||
elif naaim < 30:
|
||||
implications.append(
|
||||
"Active managers heavily derisked — potential fuel for snapback rally on positive catalyst"
|
||||
)
|
||||
|
||||
if aaii is not None:
|
||||
bull, bear, _ = aaii
|
||||
if bull > 40 and score > 60:
|
||||
implications.append(
|
||||
"AAII + CNN both elevated — dual sentiment warning, consider vol hedges"
|
||||
)
|
||||
elif bear > 40 and score < 40:
|
||||
implications.append(
|
||||
"AAII bears elevated + CNN fear — strong contrarian setup, watch for reversal catalyst"
|
||||
)
|
||||
|
||||
return implications
|
||||
|
||||
|
||||
def fetch_sentiment_report() -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
Fetch CNN Fear & Greed (primary) + NAAIM + AAII (optional).
|
||||
Returns a single structured report dict compatible with institutional_reports table.
|
||||
Returns None only if CNN F&G fetch fails entirely.
|
||||
"""
|
||||
fg = _fetch_cnn_fear_greed()
|
||||
if fg is None:
|
||||
logger.error("[SENTIMENT] CNN Fear & Greed unavailable — aborting sentiment report")
|
||||
return None
|
||||
|
||||
naaim = _fetch_naaim()
|
||||
aaii = _fetch_aaii()
|
||||
|
||||
score = fg["score"]
|
||||
rating = fg["rating"]
|
||||
|
||||
key_points = _build_key_points(fg, naaim, aaii)
|
||||
implications = _build_implications(fg, naaim, aaii)
|
||||
signal_indices = _signal_indices_from_fg(score)
|
||||
|
||||
importance = 3 if (score < 20 or score > 80) else 2
|
||||
|
||||
report_date = datetime.utcnow().strftime("%Y-%m-%d")
|
||||
|
||||
sources = ["CNN Fear & Greed"]
|
||||
if naaim is not None:
|
||||
sources.append("NAAIM")
|
||||
if aaii is not None:
|
||||
sources.append("AAII")
|
||||
source_label = " + ".join(sources)
|
||||
|
||||
raw: Dict[str, Any] = {"cnn_fear_greed": fg}
|
||||
if naaim is not None:
|
||||
raw["naaim"] = {
|
||||
"exposure_index": naaim,
|
||||
"level": "HIGH" if naaim > 80 else ("LOW" if naaim < 30 else "NORMAL"),
|
||||
}
|
||||
if aaii is not None:
|
||||
bull, bear, neut = aaii
|
||||
raw["aaii"] = {"bullish_pct": bull, "bearish_pct": bear, "neutral_pct": neut}
|
||||
|
||||
ai_summary = (
|
||||
f"Market Sentiment ({report_date}). "
|
||||
f"CNN F&G: {score:.0f}/100 ({rating.upper()}), "
|
||||
f"WoW {score - fg['previous_1_week']:+.0f}pt. "
|
||||
+ (f"NAAIM: {naaim:.0f}/200. " if naaim is not None else "")
|
||||
+ (f"AAII Bulls: {aaii[0]:.0f}%. " if aaii is not None else "")
|
||||
+ f"Signal: {signal_indices.upper()}. "
|
||||
+ implications[0]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_type": "sentiment",
|
||||
"report_date": report_date,
|
||||
"title": f"Market Sentiment — CNN F&G {score:.0f}/100 ({rating.upper()}) — {report_date}",
|
||||
"source": source_label,
|
||||
"importance": importance,
|
||||
"category": "sentiment",
|
||||
"raw_data": raw,
|
||||
"key_points": key_points,
|
||||
"trading_implications": " | ".join(implications),
|
||||
"signal_energy": "neutral",
|
||||
"signal_metals": "neutral",
|
||||
"signal_indices": signal_indices,
|
||||
"signal_forex": "neutral",
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
188
backend/services/vx_fetcher.py
Normal file
188
backend/services/vx_fetcher.py
Normal file
@@ -0,0 +1,188 @@
|
||||
"""
|
||||
VIX term structure fetcher — VX futures curve, contango/backwardation regime.
|
||||
Uses yfinance (^VIX, ^VXMT, ^VXV) + CBOE delayed quotes API as secondary.
|
||||
No API key required.
|
||||
"""
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
import yfinance as yf
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CBOE_VX_URL = "https://cdn.cboe.com/api/global/delayed_quotes/futures/VX.json"
|
||||
|
||||
VIX_TICKERS = ["^VIX", "^VXV", "^VXMT"]
|
||||
|
||||
|
||||
def _detect_regime(vix_spot: float, slope_1m_3m: float) -> str:
|
||||
if vix_spot > 30:
|
||||
return "crisis"
|
||||
elif slope_1m_3m < -2:
|
||||
return "backwardation_strong"
|
||||
elif slope_1m_3m < 0:
|
||||
return "backwardation_mild"
|
||||
elif slope_1m_3m > 5:
|
||||
return "contango_steep"
|
||||
return "contango_mild"
|
||||
|
||||
|
||||
def _fetch_cboe_vx_futures() -> Optional[List[Dict]]:
|
||||
try:
|
||||
resp = requests.get(CBOE_VX_URL, timeout=15)
|
||||
resp.raise_for_status()
|
||||
payload = resp.json()
|
||||
data = payload.get("data", [])
|
||||
contracts = []
|
||||
for item in data[:5]:
|
||||
contracts.append({
|
||||
"symbol": item.get("symbol"),
|
||||
"expiration": item.get("expiration"),
|
||||
"last": item.get("last"),
|
||||
"bid": item.get("bid"),
|
||||
"ask": item.get("ask"),
|
||||
})
|
||||
return contracts if contracts else None
|
||||
except Exception as e:
|
||||
logger.warning(f"[VX] CBOE API unavailable: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def fetch_vx_report() -> Optional[Dict]:
|
||||
"""Fetch VIX term structure and detect contango/backwardation regime. Returns None if yfinance fails."""
|
||||
report_date = datetime.now(tz=timezone.utc).strftime("%Y-%m-%d")
|
||||
|
||||
try:
|
||||
data = yf.download(VIX_TICKERS, period="5d", interval="1d", progress=False, auto_adjust=True)
|
||||
if data is None or data.empty:
|
||||
logger.warning("[VX] yfinance returned empty data")
|
||||
return None
|
||||
|
||||
close = data["Close"] if "Close" in data.columns else data
|
||||
|
||||
def latest_close(ticker: str) -> Optional[float]:
|
||||
try:
|
||||
col = close[ticker].dropna()
|
||||
if col.empty:
|
||||
return None
|
||||
return float(col.iloc[-1])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
vix_spot = latest_close("^VIX")
|
||||
vix_3m = latest_close("^VXV")
|
||||
vix_6m = latest_close("^VXMT")
|
||||
|
||||
if vix_spot is None:
|
||||
logger.warning("[VX] ^VIX data unavailable")
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"[VX] yfinance download failed: {e}")
|
||||
return None
|
||||
|
||||
slope_1m_3m = (vix_3m - vix_spot) if vix_3m is not None else None
|
||||
slope_3m_6m = (vix_6m - vix_3m) if (vix_6m is not None and vix_3m is not None) else None
|
||||
contango_pct = ((vix_3m - vix_spot) / vix_spot * 100) if vix_3m is not None else None
|
||||
|
||||
regime = _detect_regime(vix_spot, slope_1m_3m if slope_1m_3m is not None else 0.0)
|
||||
|
||||
cboe_contracts = _fetch_cboe_vx_futures()
|
||||
|
||||
raw_data: Dict[str, Any] = {
|
||||
"vix_spot": round(vix_spot, 2),
|
||||
"vix_3m": round(vix_3m, 2) if vix_3m is not None else None,
|
||||
"vix_6m": round(vix_6m, 2) if vix_6m is not None else None,
|
||||
"slope_1m_3m": round(slope_1m_3m, 2) if slope_1m_3m is not None else None,
|
||||
"slope_3m_6m": round(slope_3m_6m, 2) if slope_3m_6m is not None else None,
|
||||
"contango_pct": round(contango_pct, 2) if contango_pct is not None else None,
|
||||
"regime": regime,
|
||||
"cboe_futures": cboe_contracts,
|
||||
}
|
||||
|
||||
key_points: List[str] = []
|
||||
|
||||
vix_3m_str = f"{vix_3m:.1f}" if vix_3m is not None else "N/A"
|
||||
vix_6m_str = f"{vix_6m:.1f}" if vix_6m is not None else "N/A"
|
||||
key_points.append(f"VIX spot: {vix_spot:.1f} | 3M: {vix_3m_str} | 6M: {vix_6m_str}")
|
||||
|
||||
if slope_1m_3m is not None:
|
||||
key_points.append(
|
||||
f"Term structure slope (1M→3M): {slope_1m_3m:+.1f}pt → {regime}"
|
||||
)
|
||||
|
||||
if regime in ("backwardation_strong", "backwardation_mild"):
|
||||
key_points.append("BACKWARDATION: front > back → elevated fear, options expensive")
|
||||
|
||||
if contango_pct is not None and contango_pct > 10:
|
||||
monthly_decay = round(contango_pct / 12, 1)
|
||||
key_points.append(
|
||||
f"Steep contango: VXX roll decay ~{monthly_decay}%/month → premium selling environment"
|
||||
)
|
||||
|
||||
if contango_pct is not None:
|
||||
key_points.append(f"Contango: {contango_pct:+.1f}%")
|
||||
|
||||
if regime in ("crisis", "backwardation_strong"):
|
||||
signal_indices = "bearish"
|
||||
elif regime == "contango_steep":
|
||||
signal_indices = "bullish"
|
||||
else:
|
||||
signal_indices = "neutral"
|
||||
|
||||
implications: List[str] = []
|
||||
|
||||
if regime in ("backwardation_strong", "backwardation_mild"):
|
||||
implications.append("VXX/UVXY positive carry — consider long vol as hedge")
|
||||
elif regime == "contango_steep":
|
||||
implications.append(
|
||||
"High roll decay in VXX — premium selling favored, spreads over naked buys"
|
||||
)
|
||||
elif regime == "crisis":
|
||||
implications.append(
|
||||
"Extreme fear — tail protection is expensive, wait for calmer entry"
|
||||
)
|
||||
|
||||
if vix_spot > 20:
|
||||
implications.append(
|
||||
f"Elevated VIX ({vix_spot:.1f}) — implied vol rich, consider selling premium with defined risk"
|
||||
)
|
||||
elif vix_spot < 14:
|
||||
implications.append(
|
||||
f"Low VIX ({vix_spot:.1f}) — vol cheap, consider buying tail protection"
|
||||
)
|
||||
|
||||
if not implications:
|
||||
implications = ["VIX term structure neutral — no extreme regime detected"]
|
||||
|
||||
abs_slope = abs(slope_1m_3m) if slope_1m_3m is not None else 0.0
|
||||
importance = 3 if (regime == "crisis" or abs_slope > 5) else 2
|
||||
|
||||
slope_str = f"{slope_1m_3m:+.1f}pt" if slope_1m_3m is not None else "N/A"
|
||||
ai_summary = (
|
||||
f"VIX term structure ({report_date}). "
|
||||
f"Spot {vix_spot:.1f} | 3M {vix_3m_str} | 6M {vix_6m_str}. "
|
||||
f"Slope 1M→3M: {slope_str}. "
|
||||
f"Regime: {regime.upper()}. "
|
||||
f"Signal: {signal_indices.upper()}. "
|
||||
+ implications[0]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_type": "vx_curve",
|
||||
"report_date": report_date,
|
||||
"title": f"VIX Term Structure — {report_date}",
|
||||
"source": "yfinance + CBOE delayed API",
|
||||
"importance": importance,
|
||||
"category": "volatility",
|
||||
"raw_data": raw_data,
|
||||
"key_points": key_points,
|
||||
"trading_implications": " | ".join(implications),
|
||||
"signal_energy": "neutral",
|
||||
"signal_metals": "neutral",
|
||||
"signal_indices": signal_indices,
|
||||
"signal_forex": "neutral",
|
||||
"ai_summary": ai_summary,
|
||||
}
|
||||
@@ -81,10 +81,27 @@ function AbsorptionBadge({ score }: { score: number | null }) {
|
||||
)
|
||||
}
|
||||
|
||||
const TYPE_COLORS: Record<string, string> = {
|
||||
cot: 'bg-purple-900/30 text-purple-300 border-purple-700/40',
|
||||
eia: 'bg-orange-900/30 text-orange-300 border-orange-700/40',
|
||||
earnings: 'bg-blue-900/30 text-blue-300 border-blue-700/40',
|
||||
vx_curve: 'bg-rose-900/30 text-rose-300 border-rose-700/40',
|
||||
central_bank: 'bg-cyan-900/30 text-cyan-300 border-cyan-700/40',
|
||||
sentiment: 'bg-yellow-900/30 text-yellow-300 border-yellow-700/40',
|
||||
}
|
||||
|
||||
const TYPE_LABELS: Record<string, string> = {
|
||||
cot: 'COT',
|
||||
eia: 'EIA',
|
||||
earnings: 'Earnings',
|
||||
vx_curve: 'VX Curve',
|
||||
central_bank: 'Central Banks',
|
||||
sentiment: 'Sentiment',
|
||||
}
|
||||
|
||||
function ReportCard({ report }: { report: InstitutionalReport }) {
|
||||
const [expanded, setExpanded] = useState(false)
|
||||
const isCot = report.report_type === 'cot'
|
||||
const typeColor = isCot ? 'bg-purple-900/30 text-purple-300 border-purple-700/40' : 'bg-orange-900/30 text-orange-300 border-orange-700/40'
|
||||
const typeColor = TYPE_COLORS[report.report_type] ?? 'bg-slate-800 text-slate-400 border-slate-700/40'
|
||||
|
||||
return (
|
||||
<div className="bg-dark-800 border border-slate-700/40 rounded-lg overflow-hidden hover:border-slate-600/60 transition-colors">
|
||||
@@ -93,7 +110,7 @@ function ReportCard({ report }: { report: InstitutionalReport }) {
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="flex items-center gap-2 flex-wrap mb-1.5">
|
||||
<span className={clsx('px-2 py-0.5 rounded border text-xs font-bold uppercase tracking-wider', typeColor)}>
|
||||
{report.report_type}
|
||||
{TYPE_LABELS[report.report_type] ?? report.report_type}
|
||||
</span>
|
||||
<ImportanceStars n={report.importance} />
|
||||
<span className="text-slate-500 text-xs">
|
||||
@@ -214,40 +231,44 @@ export default function InstitutionalReports() {
|
||||
<div>
|
||||
<h1 className="text-xl font-bold text-white">Institutional Reports</h1>
|
||||
<p className="text-slate-400 text-sm mt-0.5">
|
||||
CFTC COT (weekly) + EIA Petroleum (weekly) — positioning & supply signals
|
||||
COT · EIA · Earnings · VX Curve · Central Banks · Sentiment — scheduled daily/weekly
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button
|
||||
onClick={() => handleRefresh('cot')}
|
||||
disabled={refresh.isPending}
|
||||
className="flex items-center gap-1.5 px-3 py-1.5 text-xs bg-purple-900/40 text-purple-300 border border-purple-700/40 rounded hover:bg-purple-800/50 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw className={clsx('w-3.5 h-3.5', refresh.isPending && 'animate-spin')} />
|
||||
Refresh COT
|
||||
</button>
|
||||
<button
|
||||
onClick={() => handleRefresh('eia')}
|
||||
disabled={refresh.isPending}
|
||||
className="flex items-center gap-1.5 px-3 py-1.5 text-xs bg-orange-900/40 text-orange-300 border border-orange-700/40 rounded hover:bg-orange-800/50 disabled:opacity-50"
|
||||
>
|
||||
<RefreshCw className={clsx('w-3.5 h-3.5', refresh.isPending && 'animate-spin')} />
|
||||
Refresh EIA
|
||||
</button>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{([
|
||||
{ type: 'cot', label: 'COT', cls: 'bg-purple-900/40 text-purple-300 border-purple-700/40 hover:bg-purple-800/50' },
|
||||
{ type: 'eia', label: 'EIA', cls: 'bg-orange-900/40 text-orange-300 border-orange-700/40 hover:bg-orange-800/50' },
|
||||
{ type: 'earnings', label: 'Earn.', cls: 'bg-blue-900/40 text-blue-300 border-blue-700/40 hover:bg-blue-800/50' },
|
||||
{ type: 'vx_curve', label: 'VX', cls: 'bg-rose-900/40 text-rose-300 border-rose-700/40 hover:bg-rose-800/50' },
|
||||
{ type: 'central_bank', label: 'CB RSS', cls: 'bg-cyan-900/40 text-cyan-300 border-cyan-700/40 hover:bg-cyan-800/50' },
|
||||
{ type: 'sentiment', label: 'Senti.', cls: 'bg-yellow-900/40 text-yellow-300 border-yellow-700/40 hover:bg-yellow-800/50' },
|
||||
] as const).map(({ type, label, cls }) => (
|
||||
<button
|
||||
key={type}
|
||||
onClick={() => handleRefresh(type)}
|
||||
disabled={refresh.isPending}
|
||||
className={clsx('flex items-center gap-1 px-2.5 py-1.5 text-xs border rounded disabled:opacity-50', cls)}
|
||||
>
|
||||
<RefreshCw className={clsx('w-3 h-3', refresh.isPending && 'animate-spin')} />
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Stats bar */}
|
||||
{stats && (
|
||||
<div className="grid grid-cols-2 sm:grid-cols-4 gap-3">
|
||||
<div className="grid grid-cols-3 sm:grid-cols-6 gap-2">
|
||||
{[
|
||||
{ label: 'Total reports', value: stats.total },
|
||||
{ label: 'COT reports', value: stats.by_type?.cot ?? 0 },
|
||||
{ label: 'EIA reports', value: stats.by_type?.eia ?? 0 },
|
||||
{ label: 'Absorbed (>30%)', value: stats.absorbed_count },
|
||||
].map(({ label, value }) => (
|
||||
{ label: 'COT', value: stats.by_type?.cot ?? 0, cls: 'text-purple-400' },
|
||||
{ label: 'EIA', value: stats.by_type?.eia ?? 0, cls: 'text-orange-400' },
|
||||
{ label: 'Earnings', value: stats.by_type?.earnings ?? 0, cls: 'text-blue-400' },
|
||||
{ label: 'VX Curve', value: stats.by_type?.vx_curve ?? 0, cls: 'text-rose-400' },
|
||||
{ label: 'Central Bank', value: stats.by_type?.central_bank ?? 0, cls: 'text-cyan-400' },
|
||||
{ label: 'Sentiment', value: stats.by_type?.sentiment ?? 0, cls: 'text-yellow-400' },
|
||||
].map(({ label, value, cls }) => (
|
||||
<div key={label} className="bg-dark-800 border border-slate-700/40 rounded p-3 text-center">
|
||||
<div className="text-2xl font-bold text-white">{value}</div>
|
||||
<div className={clsx('text-xl font-bold', cls)}>{value}</div>
|
||||
<div className="text-xs text-slate-500 mt-0.5">{label}</div>
|
||||
</div>
|
||||
))}
|
||||
@@ -255,19 +276,21 @@ export default function InstitutionalReports() {
|
||||
)}
|
||||
|
||||
{/* Latest dates */}
|
||||
{stats && (stats.latest_cot || stats.latest_eia) && (
|
||||
<div className="flex gap-4 text-xs text-slate-500">
|
||||
{stats.latest_cot && <span>Latest COT: <span className="text-slate-300">{stats.latest_cot}</span></span>}
|
||||
{stats.latest_eia && <span>Latest EIA: <span className="text-slate-300">{stats.latest_eia}</span></span>}
|
||||
{stats && (
|
||||
<div className="flex flex-wrap gap-4 text-xs text-slate-500">
|
||||
{stats.latest_cot && <span>COT: <span className="text-purple-400">{stats.latest_cot}</span></span>}
|
||||
{stats.latest_eia && <span>EIA: <span className="text-orange-400">{stats.latest_eia}</span></span>}
|
||||
{stats.by_type?.earnings ? <span>Earnings: <span className="text-blue-400">{Object.keys(stats.by_type).includes('earnings') ? 'tracked' : ''}</span></span> : null}
|
||||
<span className="text-slate-600">Total: {stats.total} · Absorbed: {stats.absorbed_count}</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Filters */}
|
||||
<div className="flex flex-wrap gap-3 bg-dark-800 border border-slate-700/40 rounded-lg p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-xs text-slate-500">Type:</span>
|
||||
<div className="flex gap-1">
|
||||
{['', 'cot', 'eia'].map(t => (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{(['', 'cot', 'eia', 'earnings', 'vx_curve', 'central_bank', 'sentiment'] as const).map(t => (
|
||||
<button
|
||||
key={t}
|
||||
onClick={() => setFilterType(t)}
|
||||
@@ -278,16 +301,16 @@ export default function InstitutionalReports() {
|
||||
: 'bg-dark-700 text-slate-400 border-slate-700/40 hover:border-slate-600'
|
||||
)}
|
||||
>
|
||||
{t === '' ? 'All' : t.toUpperCase()}
|
||||
{t === '' ? 'All' : (TYPE_LABELS[t] ?? t)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-2">
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<span className="text-xs text-slate-500">Category:</span>
|
||||
<div className="flex gap-1">
|
||||
{['', 'energy', 'metals', 'equities', 'forex', 'multi'].map(c => (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{['', 'energy', 'metals', 'equities', 'forex', 'volatility', 'macro', 'sentiment', 'multi'].map(c => (
|
||||
<button
|
||||
key={c}
|
||||
onClick={() => setFilterCategory(c)}
|
||||
@@ -349,8 +372,8 @@ export default function InstitutionalReports() {
|
||||
<div className="text-slate-600 text-4xl">📋</div>
|
||||
<div className="text-slate-400 font-medium">No institutional reports yet</div>
|
||||
<p className="text-slate-600 text-sm max-w-sm mx-auto">
|
||||
Click <strong className="text-slate-400">Refresh COT</strong> to fetch the latest CFTC positioning data,
|
||||
or configure your EIA API key in <strong className="text-slate-400">Configuration</strong> to enable petroleum reports.
|
||||
Use the refresh buttons above to fetch reports. COT and EIA require API keys.
|
||||
Earnings, VX, Central Bank RSS, and Sentiment fetch automatically.
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
|
||||
Reference in New Issue
Block a user