From 87ded9434f8e88a77a402d18d5b1c25e24a617a7 Mon Sep 17 00:00:00 2001 From: OpenSquared Date: Fri, 26 Jun 2026 18:15:02 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20import=20Trading=20Economics=20HTML=20c?= =?UTF-8?q?alendar=20=E2=80=94=20993=20events=20with=20forecasts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a one-click upload+parse flow for the TE calendar page HTML. The TE page (tradingeconomics.com/calendar) contains ~1000 events with actuals, previous, and analyst consensus (forecast) values. - Add backend/services/te_html_parser.py: - parse_html(html): extracts events from rows - Maps 14 countries to major currencies (all Eurozone → EUR) - Impact inferred from data-category (high/medium/low) - Times converted from Europe/Zurich (CET/CEST) → UTC via zoneinfo - Actual=td[5], Forecast=td[7] (analyst consensus), Previous=td[6] - Add POST /api/eco/te-html-upload (saves file to /tmp) - Add POST /api/eco/te-html-import + GET /status (background parse) - Add "Import TE HTML" upload button in CalendarPage ImportPanel Tested locally: 993 events parsed, 854 with forecast, 861 with actual, date range 2025-03-31 → 2026-06-17, timezone conversion verified. Co-Authored-By: Claude Sonnet 4.6 --- backend/routers/eco.py | 77 ++++++++- backend/services/te_html_parser.py | 243 ++++++++++++++++++++++++++++ frontend/src/pages/CalendarPage.tsx | 64 +++++++- 3 files changed, 377 insertions(+), 7 deletions(-) create mode 100644 backend/services/te_html_parser.py diff --git a/backend/routers/eco.py b/backend/routers/eco.py index 0570c42..a1ce7ae 100644 --- a/backend/routers/eco.py +++ b/backend/routers/eco.py @@ -293,11 +293,27 @@ def upcoming_events() -> List[Dict[str, Any]]: # ── Forex Factory calendar ──────────────────────────────────────────────────── -_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None} -_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None} -_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None} -_te_sync_status: Dict[str, Any] = {"running": False, "last_result": None} -_fxs_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_import_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_ff_scrape_status: Dict[str, Any] = {"running": False, "last_result": None} +_te_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_fxs_sync_status: Dict[str, Any] = {"running": False, "last_result": None} +_te_html_status: Dict[str, Any] = {"running": False, "last_result": None} + +_TE_HTML_CANDIDATES = [ + "/app/calendar_data.txt", + "/tmp/calendar_data.txt", + os.path.join(os.path.dirname(__file__), "..", "..", "calendar_data.txt"), + os.path.join(os.path.dirname(__file__), "..", "calendar_data.txt"), +] + + +def _find_te_html() -> str | None: + for p in _TE_HTML_CANDIDATES: + full = os.path.abspath(p) + if os.path.exists(full): + return full + return None # CSV search order: Docker image (/app), uploaded to /tmp, local dev paths _FF_CSV_CANDIDATES = [ @@ -470,6 +486,57 @@ def fmp_sync_status_ep() -> Dict[str, Any]: return _te_sync_status +# ── Trading Economics HTML import ───────────────────────────────────────────── + +@router.post("/te-html-upload") +async def te_html_upload(file: UploadFile = File(...)) -> Dict[str, Any]: + """Upload calendar_data.txt (TE HTML page) from the browser.""" + dest = "/tmp/calendar_data.txt" + try: + content = await file.read() + with open(dest, "wb") as f: + f.write(content) + size_mb = round(len(content) / 1_048_576, 1) + print(f"[TE HTML upload] Saved {size_mb} MB to {dest}", flush=True) + return {"status": "uploaded", "path": dest, "size_mb": size_mb} + except Exception as e: + raise HTTPException(500, f"Upload failed: {e}") + + +def _run_te_html_import(): + global _te_html_status + _te_html_status["running"] = True + try: + from services.te_html_parser import import_html_file + path = _find_te_html() + if not path: + _te_html_status["last_result"] = {"error": "calendar_data.txt not found — upload first"} + return + result = import_html_file(path) + _te_html_status["last_result"] = result + except Exception as e: + logger.error(f"[eco/te-html-import] Failed: {e}") + _te_html_status["last_result"] = {"error": str(e)} + finally: + _te_html_status["running"] = False + + +@router.post("/te-html-import") +def te_html_import(background_tasks: BackgroundTasks) -> Dict[str, Any]: + """Parse calendar_data.txt (TE HTML) and upsert into ff_calendar.""" + if _te_html_status["running"]: + raise HTTPException(409, "TE HTML import already running") + if not _find_te_html(): + raise HTTPException(404, "calendar_data.txt not found — upload via POST /api/eco/te-html-upload") + background_tasks.add_task(_run_te_html_import) + return {"status": "started"} + + +@router.get("/te-html-import/status") +def te_html_import_status() -> Dict[str, Any]: + return _te_html_status + + # ── FXStreet calendar (no API key required) ─────────────────────────────────── def _run_fxs_sync(weeks_ahead: int): diff --git a/backend/services/te_html_parser.py b/backend/services/te_html_parser.py new file mode 100644 index 0000000..588f5f5 --- /dev/null +++ b/backend/services/te_html_parser.py @@ -0,0 +1,243 @@ +""" +Parser for Trading Economics economic calendar HTML page. +Converts scraped TE HTML → ff_calendar rows (no API key required). + +Column map in TE HTML table rows: + td[0] = time (AM/PM, Europe/Zurich timezone) + date as CSS class + td[3] = country code (JP, US, DE, EA, ...) + td[4] = event name + period + td[5] = actual value + td[6] = previous (+ revised if any) + td[7] = consensus (analyst forecast — this is what we store as forecast_value) + td[8] = TE model forecast (ignored) +""" +import logging +import re +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple + +logger = logging.getLogger(__name__) + +# Countries → major currency (Eurozone countries all → EUR) +_COUNTRY_TO_CCY: Dict[str, str] = { + "united states": "USD", + "euro area": "EUR", + "germany": "EUR", + "france": "EUR", + "italy": "EUR", + "spain": "EUR", + "netherlands": "EUR", + "austria": "EUR", + "portugal": "EUR", + "belgium": "EUR", + "greece": "EUR", + "finland": "EUR", + "ireland": "EUR", + "slovakia": "EUR", + "united kingdom": "GBP", + "japan": "JPY", + "china": "CNY", + "australia": "AUD", + "canada": "CAD", + "new zealand": "NZD", + "switzerland": "CHF", +} + +# High-impact categories +_HIGH_CATS = { + "interest rate", "non farm payrolls", "inflation rate", + "gdp growth rate", "gdp annual growth rate", "unemployment rate", +} +# Medium-impact categories +_MEDIUM_CATS = { + "manufacturing pmi", "non manufacturing pmi", "balance of trade", + "retail sales mom", "consumer confidence", "business confidence", + "core inflation rate", "core inflation rate mom", + "producer price inflation mom", "monthly gdp mom", + "job offers", "zew economic sentiment index", +} + +_TZ_ZURICH = None + +def _get_tz(): + global _TZ_ZURICH + if _TZ_ZURICH is None: + try: + from zoneinfo import ZoneInfo + _TZ_ZURICH = ZoneInfo("Europe/Zurich") + except Exception: + import pytz + _TZ_ZURICH = pytz.timezone("Europe/Zurich") + return _TZ_ZURICH + + +def _parse_time_utc(date_str: str, time_str: str) -> Tuple[str, str]: + """ + Convert 'YYYY-MM-DD' + '02:30 PM' (Europe/Zurich) → ('YYYY-MM-DD', 'HH:MM') UTC. + Falls back to storing the raw time if parsing fails. + """ + if not time_str or not date_str: + return date_str, "00:00" + try: + tz = _get_tz() + dt_local = datetime.strptime(f"{date_str} {time_str}", "%Y-%m-%d %I:%M %p") + dt_local = dt_local.replace(tzinfo=tz) + dt_utc = dt_local.astimezone(timezone.utc) + return dt_utc.strftime("%Y-%m-%d"), dt_utc.strftime("%H:%M") + except Exception: + return date_str, "00:00" + + +def _clean_value(v: Optional[str]) -> Optional[str]: + """Normalize numeric string: strip whitespace/units, return None if empty.""" + if not v: + return None + v = v.strip() + if not v or v in ("-", "—", "N/A", ""): + return None + return v + + +def _infer_impact(category: str) -> str: + cat = (category or "").lower().strip() + if cat in _HIGH_CATS: + return "high" + if cat in _MEDIUM_CATS: + return "medium" + return "low" + + +def parse_html(html: str) -> List[Dict[str, Any]]: + """ + Parse TE calendar HTML, return list of event dicts ready to upsert. + Each dict: event_date, event_time, currency, impact, event_name, + actual, forecast, previous + """ + try: + from bs4 import BeautifulSoup + except ImportError: + raise RuntimeError("beautifulsoup4 not installed — run: pip install beautifulsoup4") + + soup = BeautifulSoup(html, "html.parser") + rows = soup.find_all("tr", attrs={"data-event": True}) + logger.info(f"[TE HTML parser] {len(rows)} rows found") + + events = [] + skipped = 0 + + for row in rows: + country = (row.get("data-country") or "").lower().strip() + ccy = _COUNTRY_TO_CCY.get(country) + if not ccy: + skipped += 1 + continue + + category = (row.get("data-category") or "").lower().strip() + impact = _infer_impact(category) + + tds = row.find_all("td") + if len(tds) < 8: + continue + + # Date from td[0] CSS class (e.g. "2025-04-01") + td0_classes = tds[0].get("class") or [] + date_str = "" + for c in td0_classes: + if re.match(r"\d{4}-\d{2}-\d{2}", c): + date_str = c + break + if not date_str: + continue + + # Time from td[0] text + time_raw = tds[0].get_text(separator=" ").strip() + time_match = re.search(r"\d{1,2}:\d{2}\s*[AP]M", time_raw, re.I) + time_str = time_match.group(0).strip() if time_match else "" + + ev_date, ev_time = _parse_time_utc(date_str, time_str) + + # Event name from td[4] + ev_link = tds[4].find("a", class_="calendar-event") + if ev_link: + event_name = ev_link.get_text(strip=True) + else: + event_name = tds[4].get_text(strip=True) + # Strip appended period reference (MAR, Q1, APR, etc.) + event_name = re.sub(r"\s+(JAN|FEB|MAR|APR|MAY|JUN|JUL|AUG|SEP|OCT|NOV|DEC|Q[1-4])$", "", event_name, flags=re.I) + event_name = event_name.strip() + if not event_name: + continue + + # Values + actual = _clean_value(tds[5].get_text(strip=True)) + previous_span = tds[6].find("span", id="previous") + if previous_span: + previous = _clean_value(previous_span.get_text(strip=True)) + else: + previous = _clean_value(tds[6].get_text(strip=True)) + consensus = _clean_value(tds[7].get_text(strip=True)) + + events.append({ + "event_date": ev_date, + "event_time": ev_time, + "currency": ccy, + "impact": impact, + "event_name": event_name, + "actual": actual, + "forecast": consensus, # consensus = analyst forecast + "previous": previous, + }) + + logger.info(f"[TE HTML parser] {len(events)} valid events, {skipped} skipped") + return events + + +def import_html_file(file_path: str) -> Dict[str, Any]: + """ + Parse TE calendar HTML file and upsert all events into ff_calendar. + Returns summary dict. + """ + from services.database import get_conn + from services.ff_calendar import _upsert_batch, FF_TO_FRED + import os + + if not os.path.exists(file_path): + return {"error": f"File not found: {file_path}"} + + print(f"[TE HTML] Reading {file_path} …", flush=True) + with open(file_path, encoding="utf-8", errors="replace") as f: + html = f.read() + print(f"[TE HTML] File size: {len(html):,} chars", flush=True) + + events = parse_html(html) + if not events: + return {"error": "No events parsed — check file format"} + + batch = [] + for ev in events: + series_id = FF_TO_FRED.get(ev["event_name"]) + batch.append(( + ev["event_date"], ev["event_time"], ev["currency"], + ev["impact"], ev["event_name"], + ev["actual"], ev["forecast"], ev["previous"], + series_id, None, "te_html", + )) + + CHUNK = 500 + conn = get_conn() + for i in range(0, len(batch), CHUNK): + _upsert_batch(conn, batch[i:i + CHUNK]) + conn.commit() + conn.close() + + by_ccy: Dict[str, int] = {} + for ev in events: + by_ccy[ev["currency"]] = by_ccy.get(ev["currency"], 0) + 1 + + print(f"[TE HTML] {len(batch)} events upserted", flush=True) + return { + "total": len(batch), + "by_currency": by_ccy, + "date_from": min(ev["event_date"] for ev in events), + "date_to": max(ev["event_date"] for ev in events), + } diff --git a/frontend/src/pages/CalendarPage.tsx b/frontend/src/pages/CalendarPage.tsx index 1725099..a12c5e4 100644 --- a/frontend/src/pages/CalendarPage.tsx +++ b/frontend/src/pages/CalendarPage.tsx @@ -1,6 +1,6 @@ -import { useState, useEffect, useCallback, useRef, KeyboardEvent } from 'react' +import { useState, useEffect, useCallback, useRef, KeyboardEvent, useId } from 'react' import clsx from 'clsx' -import { RefreshCw, ChevronDown, AlertTriangle, Check, Key } from 'lucide-react' +import { RefreshCw, ChevronDown, AlertTriangle, Check, Key, Upload } from 'lucide-react' const API = '' @@ -190,6 +190,38 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => } catch { setFxsSyncing(false); setFxsMsg('FXStreet error') } } + // TE HTML upload + import + const [teHtmlMsg, setTeHtmlMsg] = useState(null) + const [teHtmlBusy, setTeHtmlBusy] = useState(false) + const teHtmlPollRef = useRef | null>(null) + const fileInputId = useId() + + const uploadAndImportTeHtml = async (file: File) => { + setTeHtmlBusy(true) + setTeHtmlMsg('uploading…') + try { + const form = new FormData() + form.append('file', file) + const up = await fetch(`${API}/api/eco/te-html-upload`, { method: 'POST', body: form }).then(x => x.json()) + if (up.error) { setTeHtmlMsg(`Upload error: ${up.error}`); setTeHtmlBusy(false); return } + setTeHtmlMsg(`uploaded (${up.size_mb} MB) — importing…`) + await fetch(`${API}/api/eco/te-html-import`, { method: 'POST' }) + teHtmlPollRef.current = setInterval(async () => { + const s = await fetch(`${API}/api/eco/te-html-import/status`).then(x => x.json()) + if (!s.running) { + clearInterval(teHtmlPollRef.current!) + setTeHtmlBusy(false) + const r = s.last_result || {} + if (r.error) { setTeHtmlMsg(`Error: ${r.error}`); return } + setTeHtmlMsg(`✓ TE HTML: ${r.total} events (${r.date_from} → ${r.date_to})`) + onImported() + } + }, 2000) + } catch { setTeHtmlBusy(false); setTeHtmlMsg('error') } + } + + useEffect(() => () => { if (teHtmlPollRef.current) clearInterval(teHtmlPollRef.current) }, []) + const hasData = stats.total > 0 const res = importStatus.last_result @@ -228,6 +260,29 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => Sync Upcoming (FXStreet) + {/* TE HTML import — paste calendar_data.txt from tradingeconomics.com */} + + { const f = e.target.files?.[0]; if (f) uploadAndImportTeHtml(f); e.target.value = '' }} + /> + {res && !importStatus.running && ( res.error ? Import error: {res.error} @@ -239,6 +294,11 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () => {fxsMsg} )} + {teHtmlMsg && ( + + {teHtmlMsg} + + )} ) }