feat: institutional reports — CFTC COT + EIA petroleum weekly
- New institutional_reports table (DB) with importance, signals per asset class, key points, absorption tracking
- cot_fetcher.py: CFTC Socrata API (6dca-aqww), 7 instruments (Gold/Silver/Copper/WTI/NatGas/SP500/EURUSD), net positioning + 52-week z-score
- eia_fetcher.py: EIA API v2, 4 series (crude/Cushing/gasoline/distillates), WoW surprise detection
- institutional.py router: GET /reports, GET /reports/{id}, POST /refresh, GET /stats
- institutional_scheduler.py: weekly auto-fetch (COT Saturdays, EIA Wednesday afternoons)
- ai_analyzer.py: build_institutional_block() + institutional_block param injected into AI scoring prompt
- auto_cycle.py: inject institutional block into suggestion + scoring, absorption tracking via keyword overlap after each cycle commentary
- InstitutionalReports.tsx: full page with filter bar (type/category/importance/period), cards with key point bullets, EXTREME alerts highlighted, signal badges, absorption badge, trading implications, expandable detail
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
222
backend/routers/institutional.py
Normal file
222
backend/routers/institutional.py
Normal file
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
Institutional reports router — CFTC COT + EIA petroleum weekly.
|
||||
GET /api/institutional/reports — list with filters
|
||||
GET /api/institutional/reports/{id} — single report detail
|
||||
POST /api/institutional/refresh — trigger fetch
|
||||
GET /api/institutional/stats — counts + latest dates
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Query
|
||||
|
||||
from services.database import get_conn, get_config
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/api/institutional", tags=["institutional"])
|
||||
|
||||
|
||||
# ── DB helpers ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def _row_to_dict(row) -> Dict:
|
||||
d = dict(row)
|
||||
for field in ("raw_data_json", "key_points_json"):
|
||||
raw = d.pop(field, None) or ("[]" if "key_points" in field else "{}")
|
||||
try:
|
||||
d[field.replace("_json", "")] = json.loads(raw)
|
||||
except Exception:
|
||||
d[field.replace("_json", "")] = [] if "key_points" in field else {}
|
||||
return d
|
||||
|
||||
|
||||
def save_institutional_report(report: Dict) -> Optional[int]:
|
||||
"""Insert or update an institutional report. Returns row id."""
|
||||
conn = get_conn()
|
||||
try:
|
||||
existing = conn.execute(
|
||||
"SELECT id FROM institutional_reports WHERE report_type=? AND report_date=?",
|
||||
(report["report_type"], report["report_date"]),
|
||||
).fetchone()
|
||||
|
||||
raw_json = json.dumps(report.get("raw_data", {}), ensure_ascii=False)
|
||||
kp_json = json.dumps(report.get("key_points", []), ensure_ascii=False)
|
||||
|
||||
if existing:
|
||||
conn.execute(
|
||||
"""UPDATE institutional_reports SET
|
||||
fetch_date=datetime('now'), raw_data_json=?, key_points_json=?,
|
||||
trading_implications=?, signal_energy=?, signal_metals=?,
|
||||
signal_indices=?, signal_forex=?, ai_summary=?, importance=?,
|
||||
title=?
|
||||
WHERE id=?""",
|
||||
(
|
||||
raw_json, kp_json,
|
||||
report.get("trading_implications", ""),
|
||||
report.get("signal_energy", "neutral"),
|
||||
report.get("signal_metals", "neutral"),
|
||||
report.get("signal_indices", "neutral"),
|
||||
report.get("signal_forex", "neutral"),
|
||||
report.get("ai_summary", ""),
|
||||
report.get("importance", 2),
|
||||
report.get("title", ""),
|
||||
existing["id"],
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return existing["id"]
|
||||
|
||||
cursor = conn.execute(
|
||||
"""INSERT INTO institutional_reports
|
||||
(report_type, report_date, title, source, importance, category,
|
||||
raw_data_json, key_points_json, trading_implications,
|
||||
signal_energy, signal_metals, signal_indices, signal_forex, ai_summary)
|
||||
VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)""",
|
||||
(
|
||||
report["report_type"],
|
||||
report["report_date"],
|
||||
report.get("title", ""),
|
||||
report.get("source", ""),
|
||||
report.get("importance", 2),
|
||||
report.get("category", "multi"),
|
||||
raw_json, kp_json,
|
||||
report.get("trading_implications", ""),
|
||||
report.get("signal_energy", "neutral"),
|
||||
report.get("signal_metals", "neutral"),
|
||||
report.get("signal_indices", "neutral"),
|
||||
report.get("signal_forex", "neutral"),
|
||||
report.get("ai_summary", ""),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
return cursor.lastrowid
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ── Routes ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@router.get("/reports")
|
||||
def list_reports(
|
||||
report_type: Optional[str] = Query(None),
|
||||
category: Optional[str] = Query(None),
|
||||
importance: Optional[int] = Query(None),
|
||||
days: int = Query(90),
|
||||
limit: int = Query(50),
|
||||
):
|
||||
conn = get_conn()
|
||||
try:
|
||||
conditions = ["1=1"]
|
||||
params: List[Any] = []
|
||||
|
||||
if report_type:
|
||||
conditions.append("report_type = ?")
|
||||
params.append(report_type)
|
||||
if category:
|
||||
conditions.append("category = ?")
|
||||
params.append(category)
|
||||
if importance:
|
||||
conditions.append("importance >= ?")
|
||||
params.append(importance)
|
||||
if days:
|
||||
cutoff = (datetime.utcnow() - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
conditions.append("report_date >= ?")
|
||||
params.append(cutoff)
|
||||
|
||||
where = " AND ".join(conditions)
|
||||
rows = conn.execute(
|
||||
f"SELECT * FROM institutional_reports WHERE {where} "
|
||||
f"ORDER BY report_date DESC, importance DESC LIMIT ?",
|
||||
params + [limit],
|
||||
).fetchall()
|
||||
return [_row_to_dict(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@router.get("/reports/{report_id}")
|
||||
def get_report(report_id: int):
|
||||
conn = get_conn()
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM institutional_reports WHERE id=?", (report_id,)
|
||||
).fetchone()
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Report not found")
|
||||
return _row_to_dict(row)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
@router.post("/refresh")
|
||||
def refresh_reports(report_type: Optional[str] = Query(None)):
|
||||
"""Trigger a live fetch of COT and/or EIA reports."""
|
||||
results: Dict[str, str] = {}
|
||||
|
||||
if not report_type or report_type == "cot":
|
||||
try:
|
||||
from services.cot_fetcher import fetch_cot_report
|
||||
report = fetch_cot_report()
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
results["cot"] = "ok"
|
||||
logger.info(f"[Institutional] COT report saved: {report['report_date']}")
|
||||
else:
|
||||
results["cot"] = "no_data"
|
||||
except Exception as e:
|
||||
logger.warning(f"[Institutional] COT refresh failed: {e}")
|
||||
results["cot"] = f"error: {str(e)[:100]}"
|
||||
|
||||
if not report_type or report_type == "eia":
|
||||
try:
|
||||
eia_key = get_config("eia_api_key") or ""
|
||||
if eia_key:
|
||||
from services.eia_fetcher import fetch_eia_report
|
||||
report = fetch_eia_report(eia_key)
|
||||
if report:
|
||||
save_institutional_report(report)
|
||||
results["eia"] = "ok"
|
||||
logger.info(f"[Institutional] EIA report saved: {report['report_date']}")
|
||||
else:
|
||||
results["eia"] = "no_data"
|
||||
else:
|
||||
results["eia"] = "no_api_key"
|
||||
except Exception as e:
|
||||
logger.warning(f"[Institutional] EIA refresh failed: {e}")
|
||||
results["eia"] = f"error: {str(e)[:100]}"
|
||||
|
||||
return {"status": "done", "results": results}
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_stats():
|
||||
conn = get_conn()
|
||||
try:
|
||||
total = conn.execute("SELECT COUNT(*) FROM institutional_reports").fetchone()[0]
|
||||
by_type = dict(conn.execute(
|
||||
"SELECT report_type, COUNT(*) FROM institutional_reports GROUP BY report_type"
|
||||
).fetchall())
|
||||
latest_cot = conn.execute(
|
||||
"SELECT MAX(report_date) FROM institutional_reports WHERE report_type='cot'"
|
||||
).fetchone()[0]
|
||||
latest_eia = conn.execute(
|
||||
"SELECT MAX(report_date) FROM institutional_reports WHERE report_type='eia'"
|
||||
).fetchone()[0]
|
||||
absorbed = conn.execute(
|
||||
"SELECT COUNT(*) FROM institutional_reports "
|
||||
"WHERE absorbed_score IS NOT NULL AND absorbed_score > 0.3"
|
||||
).fetchone()[0]
|
||||
high_importance = conn.execute(
|
||||
"SELECT COUNT(*) FROM institutional_reports WHERE importance=3"
|
||||
).fetchone()[0]
|
||||
return {
|
||||
"total": total,
|
||||
"by_type": by_type,
|
||||
"latest_cot": latest_cot,
|
||||
"latest_eia": latest_eia,
|
||||
"absorbed_count": absorbed,
|
||||
"high_importance_count": high_importance,
|
||||
}
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user