feat: macro series

This commit is contained in:
OpenSquared
2026-06-30 19:08:33 +02:00
parent b8a1d2ad07
commit bb614936c3
5 changed files with 417 additions and 91 deletions

View File

@@ -740,3 +740,63 @@ def eco_status() -> Dict[str, Any]:
}
finally:
conn.close()
# ── Macro Series Log ──────────────────────────────────────────────────────────
@router.get("/series-log")
def get_series_log(
series_id: str = Query(...),
from_date: str = Query("2020-01-01"),
limit: int = Query(500, le=2000),
) -> List[Dict[str, Any]]:
"""Return logged actual/forecast snapshots for a given series (change events only)."""
from services.database import get_conn
conn = get_conn()
try:
rows = conn.execute(
"""SELECT series_id, event_name, event_date, actual_value, forecast_value,
previous_value, currency, impact, logged_at, source, changed
FROM macro_series_log
WHERE series_id = ? AND event_date >= ?
ORDER BY event_date ASC, logged_at ASC
LIMIT ?""",
(series_id, from_date, limit)
).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
@router.get("/series-log/distinct-series")
def list_logged_series() -> List[Dict[str, Any]]:
"""Return all series_id that have entries in macro_series_log."""
from services.database import get_conn
conn = get_conn()
try:
rows = conn.execute(
"""SELECT series_id, event_name,
COUNT(*) as log_count,
MIN(event_date) as first_date,
MAX(event_date) as last_date,
MAX(logged_at) as last_logged
FROM macro_series_log
GROUP BY series_id
ORDER BY last_date DESC"""
).fetchall()
return [dict(r) for r in rows]
finally:
conn.close()
@router.post("/series-log/backfill")
def backfill_series_log() -> Dict[str, Any]:
"""Seed macro_series_log from existing ff_calendar rows (one-time historical import)."""
from services.database import get_conn
from services.macro_series_log import backfill_from_ff_calendar
conn = get_conn()
try:
result = backfill_from_ff_calendar(conn)
return result
finally:
conn.close()

View File

@@ -47,6 +47,17 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
result = {"error": str(e)}
source = "error"
# After calendar upsert, log any forecast/actual changes to macro_series_log
try:
from services.database import get_conn
from services.macro_series_log import scan_and_log_all
_conn = get_conn()
log_result = scan_and_log_all(_conn, source=source)
_conn.close()
result["_log"] = log_result
except Exception as e:
logger.warning(f"[calendar_sync] macro_series_log scan failed: {e}")
_sync_status["running"] = False
_sync_status["last_result"] = result
_sync_status["source"] = source

View File

@@ -715,6 +715,29 @@ def init_db():
except Exception:
pass # Column already exists
# ── Macro Series Log ───────────────────────────────────────────────────────
# One row per change: actual or forecast changed vs previous snapshot.
# Allows tracking forecast revisions over time + actual vs consensus history.
c.execute("""CREATE TABLE IF NOT EXISTS macro_series_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
series_id TEXT NOT NULL,
event_name TEXT NOT NULL DEFAULT '',
event_date TEXT NOT NULL,
actual_value REAL,
forecast_value REAL,
previous_value REAL,
currency TEXT DEFAULT '',
impact TEXT DEFAULT '',
logged_at TEXT NOT NULL DEFAULT (datetime('now')),
source TEXT DEFAULT 'calendar_sync',
changed TEXT DEFAULT 'new'
)""")
try:
c.execute("CREATE INDEX IF NOT EXISTS idx_msl_series ON macro_series_log(series_id, event_date)")
c.execute("CREATE INDEX IF NOT EXISTS idx_msl_logged ON macro_series_log(logged_at DESC)")
except Exception:
pass
# ── Specialist Desks ───────────────────────────────────────────────────────
c.execute("""CREATE TABLE IF NOT EXISTS specialist_reports (
id TEXT PRIMARY KEY,

View File

@@ -0,0 +1,159 @@
"""
Macro series log — tracks changes to actual_value and forecast_value over time.
Called after each calendar sync to detect and persist revisions.
"""
import logging
from datetime import datetime, timezone
from typing import Optional
logger = logging.getLogger(__name__)
def _parse_num(v) -> Optional[float]:
"""Convert FF text value (e.g. '178K', '0.3%', '-92K') to float or None."""
if v is None:
return None
s = str(v).strip().replace('%', '').replace('K', 'e3').replace('M', 'e6').replace('B', 'e9')
try:
return float(s)
except Exception:
return None
def log_if_changed(conn, series_id: str, event_name: str, event_date: str,
actual_value, forecast_value, previous_value,
currency: str = '', impact: str = '',
source: str = 'calendar_sync') -> bool:
"""
Compare (actual, forecast) against the last log entry for this series+date.
Insert a new row only if something changed or it's the first time we see it.
Returns True if a row was inserted.
"""
actual_num = _parse_num(actual_value)
forecast_num = _parse_num(forecast_value)
previous_num = _parse_num(previous_value)
# Nothing to log if both actual and forecast are None
if actual_num is None and forecast_num is None:
return False
last = conn.execute(
"""SELECT actual_value, forecast_value FROM macro_series_log
WHERE series_id=? AND event_date=?
ORDER BY logged_at DESC LIMIT 1""",
(series_id, event_date)
).fetchone()
if last is None:
changed = 'new'
else:
prev_actual = last['actual_value']
prev_forecast = last['forecast_value']
actual_changed = actual_num != prev_actual and actual_num is not None
forecast_changed = forecast_num != prev_forecast and forecast_num is not None
if not actual_changed and not forecast_changed:
return False
if actual_changed and forecast_changed:
changed = 'both'
elif actual_changed:
changed = 'actual'
else:
changed = 'forecast'
conn.execute(
"""INSERT INTO macro_series_log
(series_id, event_name, event_date, actual_value, forecast_value,
previous_value, currency, impact, logged_at, source, changed)
VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
(series_id, event_name or '', event_date,
actual_num, forecast_num, previous_num,
currency or '', impact or '',
datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M:%S'),
source, changed)
)
conn.commit()
return True
def backfill_from_ff_calendar(conn) -> dict:
"""
Seed macro_series_log from existing ff_calendar rows (historical).
Only inserts rows that don't already exist (idempotent).
Skips events without series_id.
"""
rows = conn.execute(
"""SELECT series_id, event_name, event_date, actual_value, forecast_value,
previous_value, currency, impact, fetched_at
FROM ff_calendar
WHERE series_id IS NOT NULL AND series_id != ''
AND (actual_value IS NOT NULL OR forecast_value IS NOT NULL)
ORDER BY event_date ASC"""
).fetchall()
inserted = 0
for r in rows:
already = conn.execute(
"SELECT 1 FROM macro_series_log WHERE series_id=? AND event_date=? AND source='backfill' LIMIT 1",
(r['series_id'], r['event_date'])
).fetchone()
if already:
continue
actual_num = _parse_num(r['actual_value'])
forecast_num = _parse_num(r['forecast_value'])
previous_num = _parse_num(r['previous_value'])
if actual_num is None and forecast_num is None:
continue
logged_at = r['fetched_at'] or r['event_date']
changed = 'actual' if actual_num is not None else 'forecast'
conn.execute(
"""INSERT OR IGNORE INTO macro_series_log
(series_id, event_name, event_date, actual_value, forecast_value,
previous_value, currency, impact, logged_at, source, changed)
VALUES (?,?,?,?,?,?,?,?,?,?,?)""",
(r['series_id'], r['event_name'] or '', r['event_date'],
actual_num, forecast_num, previous_num,
r['currency'] or '', r['impact'] or '',
logged_at, 'backfill', changed)
)
inserted += 1
conn.commit()
logger.info(f"[macro_series_log] backfill: {inserted} rows inserted from ff_calendar")
return {"inserted": inserted, "scanned": len(rows)}
def scan_and_log_all(conn, source: str = 'calendar_sync') -> dict:
"""
After a calendar sync, scan all ff_calendar rows with series_id and log any changes.
Typically called at end of each 6h sync cycle.
"""
rows = conn.execute(
"""SELECT series_id, event_name, event_date, actual_value, forecast_value,
previous_value, currency, impact
FROM ff_calendar
WHERE series_id IS NOT NULL AND series_id != ''
AND (actual_value IS NOT NULL OR forecast_value IS NOT NULL)"""
).fetchall()
logged = 0
for r in rows:
did_log = log_if_changed(
conn,
series_id=r['series_id'],
event_name=r['event_name'] or '',
event_date=r['event_date'],
actual_value=r['actual_value'],
forecast_value=r['forecast_value'],
previous_value=r['previous_value'],
currency=r['currency'] or '',
impact=r['impact'] or '',
source=source,
)
if did_log:
logged += 1
logger.info(f"[macro_series_log] scan_and_log_all: {logged} changes logged out of {len(rows)} rows")
return {"logged": logged, "scanned": len(rows)}

View File

@@ -1,9 +1,9 @@
import { useState, useEffect, useCallback } from 'react'
import clsx from 'clsx'
import { RefreshCw, TrendingUp, TrendingDown, Minus } from 'lucide-react'
import { RefreshCw, TrendingUp, TrendingDown, Minus, Database } from 'lucide-react'
import {
ComposedChart, Line, XAxis, YAxis, CartesianGrid, Tooltip,
ResponsiveContainer, ReferenceLine, Area,
ComposedChart, Line, Bar, XAxis, YAxis, CartesianGrid, Tooltip,
ResponsiveContainer, ReferenceLine, Area, Legend,
} from 'recharts'
const API = ''
@@ -46,6 +46,17 @@ interface SeriesHistory {
events: FFEvent[]
}
interface LogEntry {
series_id: string
event_name: string
event_date: string
actual_value: number | null
forecast_value: number | null
previous_value: number | null
logged_at: string
changed: string // 'new' | 'actual' | 'forecast' | 'both' | 'backfill'
}
// ── Constants ─────────────────────────────────────────────────────────────────
const CATEGORY_COLORS: Record<string, string> = {
@@ -254,9 +265,11 @@ export default function MacroSeriesPage() {
const [selectedId, setSelectedId] = useState<string>('PAYEMS')
const [range, setRange] = useState('5y')
const [history, setHistory] = useState<SeriesHistory | null>(null)
const [logEntries, setLogEntries] = useState<LogEntry[]>([])
const [loading, setLoading] = useState(false)
const [backfilling, setBackfilling] = useState(false)
const [tab, setTab] = useState<'chart' | 'log'>('chart')
// Load series catalog
useEffect(() => {
fetch(`${API}/api/eco/series`).then(r => r.json()).then(setSeriesList).catch(() => {})
}, [])
@@ -267,35 +280,57 @@ export default function MacroSeriesPage() {
try {
const months = RANGE_OPTIONS.find(r => r.key === range)?.months ?? 60
const from = fromDateForRange(months)
const r: SeriesHistory = await fetch(
`${API}/api/eco/series/${selectedId}/history?from_date=${from}`
).then(x => x.json())
setHistory(r)
const [hist, log] = await Promise.all([
fetch(`${API}/api/eco/series/${selectedId}/history?from_date=${from}`).then(x => x.json()),
fetch(`${API}/api/eco/series-log?series_id=${selectedId}&from_date=${from}&limit=1000`).then(x => x.json()),
])
setHistory(hist)
setLogEntries(Array.isArray(log) ? log : [])
} catch {}
finally { setLoading(false) }
}, [selectedId, range])
useEffect(() => { loadHistory() }, [loadHistory])
// Build chart data — merge FRED time series + FF events by date
const handleBackfill = async () => {
setBackfilling(true)
try {
await fetch(`${API}/api/eco/series-log/backfill`, { method: 'POST' })
await loadHistory()
} catch {}
finally { setBackfilling(false) }
}
// Build chart data — merge FRED time series + log entries by event_date
// For each release date: actual from log (last entry with actual) + last forecast before actual
const chartData = (() => {
if (!history) return []
const ffByDate = new Map<string, FFEvent>()
for (const ev of history.events) ffByDate.set(ev.event_date, ev)
// Group log entries by event_date, pick last forecast before actual and first actual
const logByDate = new Map<string, { actual: number | null; lastForecast: number | null }>()
for (const e of logEntries) {
const existing = logByDate.get(e.event_date) ?? { actual: null, lastForecast: null }
if (e.actual_value != null) existing.actual = e.actual_value
if (e.forecast_value != null && e.actual_value == null) existing.lastForecast = e.forecast_value
logByDate.set(e.event_date, existing)
}
return history.timeseries.map(pt => {
const ff = ffByDate.get(pt.event_date)
const logged = logByDate.get(pt.event_date)
return {
event_date: pt.event_date,
label: fmtDate(pt.event_date),
value: pt.actual_value,
zscore: pt.surprise_zscore,
direction: pt.surprise_direction,
ff_event: ff?.event_name ?? null,
ff_actual: ff?.actual_value ?? null,
ff_forecast:ff?.forecast_value ?? null,
// For reference lines — mark surprise events
is_surprise: pt.surprise_zscore !== null && Math.abs(pt.surprise_zscore) >= 1.5,
event_date: pt.event_date,
label: fmtDate(pt.event_date),
value: pt.actual_value,
forecast: logged?.lastForecast ?? (ff?.forecast_value ? parseFloat(ff.forecast_value) || null : null),
zscore: pt.surprise_zscore,
direction: pt.surprise_direction,
ff_event: ff?.event_name ?? null,
ff_actual: ff?.actual_value ?? null,
ff_forecast: ff?.forecast_value ?? null,
is_surprise: pt.surprise_zscore !== null && Math.abs(pt.surprise_zscore) >= 1.5,
}
})
})()
@@ -346,26 +381,33 @@ export default function MacroSeriesPage() {
</div>
<div className="flex items-center gap-2">
{/* Tab selector */}
<div className="flex gap-1 bg-slate-800 rounded p-0.5">
{(['chart', 'log'] as const).map(t => (
<button key={t} onClick={() => setTab(t)}
className={clsx('px-2.5 py-1 rounded text-xs font-medium transition-colors',
tab === t ? 'bg-slate-600 text-white' : 'text-slate-400 hover:text-slate-200')}>
{t === 'chart' ? 'Chart' : `Log (${logEntries.length})`}
</button>
))}
</div>
{/* Range selector */}
<div className="flex gap-1 bg-slate-800 rounded p-0.5">
{RANGE_OPTIONS.map(r => (
<button
key={r.key}
onClick={() => setRange(r.key)}
className={clsx(
'px-2.5 py-1 rounded text-xs font-medium transition-colors',
range === r.key ? 'bg-slate-600 text-white' : 'text-slate-400 hover:text-slate-200'
)}
>
<button key={r.key} onClick={() => setRange(r.key)}
className={clsx('px-2.5 py-1 rounded text-xs font-medium transition-colors',
range === r.key ? 'bg-slate-600 text-white' : 'text-slate-400 hover:text-slate-200')}>
{r.label}
</button>
))}
</div>
<button
onClick={loadHistory}
disabled={loading}
className="p-1.5 rounded bg-slate-700 hover:bg-slate-600 disabled:opacity-50"
>
<button onClick={handleBackfill} disabled={backfilling}
title="Importer l'historique ff_calendar → macro_series_log"
className="p-1.5 rounded bg-slate-700 hover:bg-slate-600 disabled:opacity-50">
<Database size={13} className={clsx(backfilling && 'animate-pulse')} />
</button>
<button onClick={loadHistory} disabled={loading}
className="p-1.5 rounded bg-slate-700 hover:bg-slate-600 disabled:opacity-50">
<RefreshCw size={13} className={clsx(loading && 'animate-spin')} />
</button>
</div>
@@ -397,107 +439,138 @@ export default function MacroSeriesPage() {
</div>
)}
{/* Chart */}
{tab === 'chart' && (
<div className="bg-slate-800 rounded-lg border border-slate-700 p-4">
<div className="flex items-center justify-between mb-3">
<span className="text-sm font-medium text-slate-300">
Time Series
{history && <span className="text-slate-500 text-xs ml-2">({chartData.length} data points)</span>}
Actual vs Consensus
{history && <span className="text-slate-500 text-xs ml-2">({chartData.length} releases)</span>}
</span>
{bigSurprises.length > 0 && (
<span className="text-xs text-slate-500">
{bigSurprises.length} big surprises (|z| 1.5)
</span>
<span className="text-xs text-slate-500">{bigSurprises.length} surprises (|z|1.5)</span>
)}
</div>
{loading && (
<div className="h-64 flex items-center justify-center text-slate-600 text-sm">
Loading
</div>
)}
{loading && <div className="h-64 flex items-center justify-center text-slate-600 text-sm">Chargement</div>}
{!loading && chartData.length === 0 && (
<div className="h-64 flex items-center justify-center text-slate-600 text-sm">
No data run FRED bootstrap first in the Calendar page.
Pas de données lancez le bootstrap FRED depuis Calendar.
</div>
)}
{!loading && chartData.length > 0 && (
<ResponsiveContainer width="100%" height={280}>
<ResponsiveContainer width="100%" height={300}>
<ComposedChart data={chartData} margin={{ top: 4, right: 16, bottom: 4, left: 0 }}>
<defs>
<linearGradient id={`grad-${selectedId}`} x1="0" y1="0" x2="0" y2="1">
<stop offset="5%" stopColor={color} stopOpacity={0.25} />
<stop offset="5%" stopColor={color} stopOpacity={0.3} />
<stop offset="95%" stopColor={color} stopOpacity={0} />
</linearGradient>
</defs>
<CartesianGrid strokeDasharray="3 3" stroke="#1e293b" />
<XAxis
dataKey="label"
tick={{ fontSize: 10, fill: '#64748b' }}
interval="preserveStartEnd"
tickLine={false}
/>
<YAxis
tick={{ fontSize: 10, fill: '#64748b' }}
tickLine={false}
axisLine={false}
width={50}
tickFormatter={v => `${v}${history?.unit === '%' || history?.unit === 'pp' ? '' : ''}`}
/>
<XAxis dataKey="label" tick={{ fontSize: 10, fill: '#64748b' }} interval="preserveStartEnd" tickLine={false} />
<YAxis tick={{ fontSize: 10, fill: '#64748b' }} tickLine={false} axisLine={false} width={50} />
<Tooltip content={<ChartTooltip unit={history?.unit ?? ''} />} />
<Legend wrapperStyle={{ fontSize: 11, color: '#94a3b8' }} />
{/* Zero line if series crosses zero */}
{minVal !== null && maxVal !== null && minVal < 0 && maxVal > 0 && (
<ReferenceLine y={0} stroke="#475569" strokeDasharray="4 2" />
)}
{/* Big surprise markers */}
{bigSurprises.map(s => (
<ReferenceLine
key={s.event_date}
x={fmtDate(s.event_date)}
<ReferenceLine key={s.event_date} x={fmtDate(s.event_date)}
stroke={s.direction === 'bullish' ? '#10b981' : s.direction === 'bearish' ? '#ef4444' : '#64748b'}
strokeWidth={1}
strokeDasharray="3 3"
opacity={0.6}
/>
strokeWidth={1} strokeDasharray="3 3" opacity={0.5} />
))}
<Area
type="monotone"
dataKey="value"
stroke={color}
strokeWidth={2}
fill={`url(#grad-${selectedId})`}
dot={false}
activeDot={{ r: 4, fill: color }}
/>
{/* Actual — area + line */}
<Area type="monotone" dataKey="value" name="Actual"
stroke={color} strokeWidth={2} fill={`url(#grad-${selectedId})`}
dot={false} activeDot={{ r: 4, fill: color }} />
{/* Forecast — dashed line */}
<Line type="monotone" dataKey="forecast" name="Forecast (consensus)"
stroke="#f59e0b" strokeWidth={1.5} strokeDasharray="4 3"
dot={false} activeDot={{ r: 3 }} connectNulls />
</ComposedChart>
</ResponsiveContainer>
)}
</div>
)}
{/* Legend for reference lines */}
{bigSurprises.length > 0 && (
<div className="flex items-center gap-4 mt-2 text-xs text-slate-500">
<span className="flex items-center gap-1">
<span className="w-4 border-t border-dashed border-emerald-500 inline-block" />
Bullish surprise (|z|1.5)
</span>
<span className="flex items-center gap-1">
<span className="w-4 border-t border-dashed border-red-500 inline-block" />
Bearish surprise
</span>
{tab === 'log' && (
<div className="bg-slate-800 rounded-lg border border-slate-700 p-4">
<div className="flex items-center justify-between mb-3">
<span className="text-sm font-medium text-slate-300">
Journal des changements {selectedId}
<span className="text-slate-500 text-xs ml-2">({logEntries.length} entrées)</span>
</span>
{logEntries.length === 0 && (
<button onClick={handleBackfill} disabled={backfilling}
className="text-xs text-violet-400 hover:text-violet-200 bg-violet-900/20 border border-violet-800/40 rounded px-2 py-1 disabled:opacity-50">
{backfilling ? 'Import…' : 'Importer historique ff_calendar'}
</button>
)}
</div>
{logEntries.length === 0 ? (
<div className="text-slate-600 text-sm text-center py-8">
Aucune entrée cliquez "Importer historique" ou attendez le prochain sync (6h).
</div>
) : (
<div className="overflow-x-auto">
<table className="w-full text-xs">
<thead>
<tr className="text-slate-500 border-b border-slate-800">
<th className="text-left py-1.5 pr-3">Release</th>
<th className="text-left py-1.5 pr-3">Capturé à</th>
<th className="text-right py-1.5 pr-3">Actual</th>
<th className="text-right py-1.5 pr-3">Forecast</th>
<th className="text-right py-1.5 pr-3">Previous</th>
<th className="text-left py-1.5">Changement</th>
</tr>
</thead>
<tbody>
{[...logEntries].reverse().map((e, i) => {
const isActual = e.actual_value != null
const isForecastChange = e.changed === 'forecast' || e.changed === 'both'
return (
<tr key={i} className={clsx('border-b border-slate-800/50 hover:bg-slate-800/30',
isActual ? 'bg-slate-800/20' : '')}>
<td className="py-1.5 pr-3 text-slate-400 tabular-nums">{e.event_date}</td>
<td className="py-1.5 pr-3 text-slate-600 tabular-nums">{e.logged_at.slice(0, 16)}</td>
<td className={clsx('py-1.5 pr-3 text-right tabular-nums font-medium',
isActual ? 'text-white' : 'text-slate-700')}>
{e.actual_value != null ? e.actual_value.toFixed(2) : '—'}
</td>
<td className={clsx('py-1.5 pr-3 text-right tabular-nums',
isForecastChange ? 'text-amber-400' : 'text-slate-400')}>
{e.forecast_value != null ? e.forecast_value.toFixed(2) : '—'}
</td>
<td className="py-1.5 pr-3 text-right tabular-nums text-slate-600">
{e.previous_value != null ? e.previous_value.toFixed(2) : '—'}
</td>
<td className="py-1.5">
<span className={clsx('px-1.5 py-0.5 rounded text-[10px]',
e.changed === 'actual' || e.changed === 'both' ? 'bg-emerald-900/40 text-emerald-400' :
e.changed === 'forecast' ? 'bg-amber-900/40 text-amber-400' :
'bg-slate-700/40 text-slate-500')}>
{e.changed}
</span>
</td>
</tr>
)
})}
</tbody>
</table>
</div>
)}
</div>
)}
{/* Events table */}
<div className="bg-slate-800 rounded-lg border border-slate-700 p-4">
<div className="text-sm font-medium text-slate-300 mb-3">
FF Calendar Events
{history && <span className="text-slate-500 text-xs ml-2">({history.events.length} linked)</span>}
{history && <span className="text-slate-500 text-xs ml-2">({history.events.length} liés)</span>}
</div>
{history && <EventsTable events={history.events} unit={history.unit} />}
</div>