feat: calendar eco
This commit is contained in:
@@ -1,4 +1,16 @@
|
|||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
|
# Windows consoles default to a legacy codepage (cp1252/cp850) that can't encode
|
||||||
|
# the arrows/checkmarks/emojis used throughout this codebase's print() calls —
|
||||||
|
# an uncaught UnicodeEncodeError on a background thread (e.g. the calendar sync
|
||||||
|
# loop) silently kills that task with no visible error. UTF-8 + replace makes
|
||||||
|
# every print() safe regardless of console codepage.
|
||||||
|
if sys.platform == "win32":
|
||||||
|
for _stream in (sys.stdout, sys.stderr):
|
||||||
|
try:
|
||||||
|
_stream.reconfigure(encoding="utf-8", errors="replace")
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from dotenv import load_dotenv
|
from dotenv import load_dotenv
|
||||||
# Explicit path: don't rely on the process's current working directory (a systemd
|
# Explicit path: don't rely on the process's current working directory (a systemd
|
||||||
|
|||||||
@@ -38,31 +38,42 @@ def calendar_sync(weeks_ahead: int = 8) -> Dict[str, Any]:
|
|||||||
from services.ff_calendar import sync_live, scrape_upcoming
|
from services.ff_calendar import sync_live, scrape_upcoming
|
||||||
|
|
||||||
# Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
|
# Step 1 — live JSON (thisweek + nextweek) — picks up actuals in real time
|
||||||
|
print("[calendar_sync] step 1/3 — FF live JSON (thisweek + nextweek)", flush=True)
|
||||||
r_live = sync_live()
|
r_live = sync_live()
|
||||||
result["live"] = r_live
|
result["live"] = r_live
|
||||||
|
print(f"[calendar_sync] live result: {r_live}", flush=True)
|
||||||
|
|
||||||
# Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
|
# Step 2 — HTML scraper for weeks 3..N ahead (forecasts only, no actuals yet)
|
||||||
scrape_weeks = max(0, weeks_ahead - 2)
|
scrape_weeks = max(0, weeks_ahead - 2)
|
||||||
if scrape_weeks > 0:
|
if scrape_weeks > 0:
|
||||||
|
print(f"[calendar_sync] step 2/3 — FF HTML scrape, {scrape_weeks} week(s) ahead", flush=True)
|
||||||
r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
|
r_scrape = scrape_upcoming(weeks_ahead=scrape_weeks)
|
||||||
result["scrape"] = r_scrape
|
result["scrape"] = r_scrape
|
||||||
|
print(f"[calendar_sync] scrape result: {r_scrape}", flush=True)
|
||||||
|
|
||||||
# FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can
|
# FF's scraper gets 403'd (Cloudflare) often enough that "weeks 3..N" can
|
||||||
# silently come back empty — fall back to FMP for that same range when it does.
|
# silently come back empty — fall back to FMP for that same range when it does.
|
||||||
if not r_scrape.get("total_upserted"):
|
if not r_scrape.get("total_upserted"):
|
||||||
|
print("[calendar_sync] step 3/3 — scrape got 0 events, checking FMP fallback", flush=True)
|
||||||
from services.fmp_calendar import check_fmp_key, fetch_upcoming
|
from services.fmp_calendar import check_fmp_key, fetch_upcoming
|
||||||
if check_fmp_key()["configured"]:
|
key_status = check_fmp_key()
|
||||||
|
print(f"[calendar_sync] FMP key configured: {key_status['configured']}", flush=True)
|
||||||
|
if key_status["configured"]:
|
||||||
r_fmp = fetch_upcoming(weeks_ahead=weeks_ahead)
|
r_fmp = fetch_upcoming(weeks_ahead=weeks_ahead)
|
||||||
result["fmp_fallback"] = r_fmp
|
result["fmp_fallback"] = r_fmp
|
||||||
|
print(f"[calendar_sync] FMP fallback result: {r_fmp}", flush=True)
|
||||||
if r_fmp.get("total_upserted"):
|
if r_fmp.get("total_upserted"):
|
||||||
source = "ff+fmp"
|
source = "ff+fmp"
|
||||||
else:
|
else:
|
||||||
result["fmp_fallback"] = {"skipped": "no_key"}
|
result["fmp_fallback"] = {"skipped": "no_key"}
|
||||||
|
else:
|
||||||
|
print("[calendar_sync] step 3/3 — scrape got events, FMP fallback not needed", flush=True)
|
||||||
else:
|
else:
|
||||||
result["scrape"] = {"skipped": True}
|
result["scrape"] = {"skipped": True}
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"[calendar_sync] FF sync failed: {e}")
|
logger.error(f"[calendar_sync] FF sync failed: {e}")
|
||||||
|
print(f"[calendar_sync] EXCEPTION: {e!r}", flush=True)
|
||||||
result = {"error": str(e)}
|
result = {"error": str(e)}
|
||||||
source = "error"
|
source = "error"
|
||||||
|
|
||||||
|
|||||||
@@ -1466,14 +1466,18 @@ def set_config(key: str, value: str):
|
|||||||
os.environ["OPENAI_API_KEY"] = value
|
os.environ["OPENAI_API_KEY"] = value
|
||||||
|
|
||||||
|
|
||||||
|
_MASKED_CONFIG_KEYS = ["openai_api_key", "newsapi_key", "eia_api_key", "fred_api_key", "fmp_api_key"]
|
||||||
|
|
||||||
|
|
||||||
def get_all_config() -> Dict[str, str]:
|
def get_all_config() -> Dict[str, str]:
|
||||||
conn = get_conn()
|
conn = get_conn()
|
||||||
rows = conn.execute("SELECT key, value FROM config").fetchall()
|
rows = conn.execute("SELECT key, value FROM config").fetchall()
|
||||||
conn.close()
|
conn.close()
|
||||||
result = {r["key"]: r["value"] for r in rows}
|
result = {r["key"]: r["value"] for r in rows}
|
||||||
if "openai_api_key" in result and result["openai_api_key"]:
|
for key in _MASKED_CONFIG_KEYS:
|
||||||
result["openai_api_key_set"] = True
|
if key in result and result[key]:
|
||||||
result["openai_api_key"] = "***"
|
result[f"{key}_set"] = True
|
||||||
|
result[key] = "***"
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -80,7 +80,7 @@ def fetch_upcoming(weeks_ahead: int = 6) -> Dict[str, Any]:
|
|||||||
"apikey": key,
|
"apikey": key,
|
||||||
}
|
}
|
||||||
|
|
||||||
print(f"[FMP calendar] fetching {today} → {date_to}", flush=True)
|
print(f"[FMP calendar] fetching {today} -> {date_to}", flush=True)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
resp = httpx.get(url, params=params, timeout=30, follow_redirects=True)
|
resp = httpx.get(url, params=params, timeout=30, follow_redirects=True)
|
||||||
|
|||||||
@@ -165,6 +165,7 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
|||||||
|
|
||||||
const [calStatus, setCalStatus] = useState<CalSyncStatus>({ running: false, last_run: null, last_result: null, source: null, next_run: null })
|
const [calStatus, setCalStatus] = useState<CalSyncStatus>({ running: false, last_run: null, last_result: null, source: null, next_run: null })
|
||||||
const [syncMsg, setSyncMsg] = useState<string | null>(null)
|
const [syncMsg, setSyncMsg] = useState<string | null>(null)
|
||||||
|
const [showSyncDebug, setShowSyncDebug] = useState(false)
|
||||||
const calPollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
const calPollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||||
|
|
||||||
const [ffPageMsg, setFfPageMsg] = useState<string | null>(null)
|
const [ffPageMsg, setFfPageMsg] = useState<string | null>(null)
|
||||||
@@ -211,10 +212,18 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
|||||||
if (r.error) {
|
if (r.error) {
|
||||||
setSyncMsg(`Erreur: ${r.error}`)
|
setSyncMsg(`Erreur: ${r.error}`)
|
||||||
} else {
|
} else {
|
||||||
const tw = r.live?.thisweek?.events ?? r.live?.events ?? '?'
|
const twStatus = r.live?.thisweek?.status
|
||||||
const nw = r.live?.nextweek?.events ?? '?'
|
const nwStatus = r.live?.nextweek?.status
|
||||||
|
const tw = twStatus === 'ok' ? r.live.thisweek.events : (r.live?.thisweek?.error ? `err` : (twStatus ?? '?'))
|
||||||
|
const nw = nwStatus === 'ok' ? r.live.nextweek.events : (r.live?.nextweek?.error ? `err` : (nwStatus ?? '?'))
|
||||||
const sc = r.scrape?.total_upserted ?? 0
|
const sc = r.scrape?.total_upserted ?? 0
|
||||||
setSyncMsg(`✓ FF: ${tw}+${nw} live, ${sc} upcoming`)
|
let msg = `✓ FF: ${tw}+${nw} live, ${sc} upcoming`
|
||||||
|
if (r.fmp_fallback) {
|
||||||
|
if (r.fmp_fallback.skipped === 'no_key') msg += ' · FMP fallback skipped (no key)'
|
||||||
|
else if (r.fmp_fallback.error) msg += ` · FMP error: ${r.fmp_fallback.error}`
|
||||||
|
else msg += ` · FMP +${r.fmp_fallback.total_upserted ?? 0}`
|
||||||
|
}
|
||||||
|
setSyncMsg(msg)
|
||||||
}
|
}
|
||||||
onImported()
|
onImported()
|
||||||
}
|
}
|
||||||
@@ -250,7 +259,7 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
|||||||
|
|
||||||
const hasData = stats.total > 0
|
const hasData = stats.total > 0
|
||||||
const res = importStatus.last_result
|
const res = importStatus.last_result
|
||||||
const srcLabel = calStatus.source === 'ff' ? 'ForexFactory' : calStatus.source ?? null
|
const srcLabel = calStatus.source === 'ff' ? 'ForexFactory' : calStatus.source === 'ff+fmp' ? 'ForexFactory+FMP' : calStatus.source ?? null
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="bg-slate-800 border border-slate-700 rounded-lg p-4 mb-4 flex flex-wrap items-center gap-4 text-sm">
|
<div className="bg-slate-800 border border-slate-700 rounded-lg p-4 mb-4 flex flex-wrap items-center gap-4 text-sm">
|
||||||
@@ -277,7 +286,7 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
|||||||
onClick={startCalendarSync}
|
onClick={startCalendarSync}
|
||||||
disabled={calStatus.running}
|
disabled={calStatus.running}
|
||||||
className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
|
className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
|
||||||
title="Sync economic calendar (FXStreet primary, FF fallback if blocked)"
|
title="Sync economic calendar (FF live JSON + HTML scraper; FMP fallback for weeks 3+ if FF scraper is blocked)"
|
||||||
>
|
>
|
||||||
<RefreshCw size={13} className={clsx(calStatus.running && 'animate-spin')} />
|
<RefreshCw size={13} className={clsx(calStatus.running && 'animate-spin')} />
|
||||||
{calStatus.running ? 'Syncing…' : 'Sync Now'}
|
{calStatus.running ? 'Syncing…' : 'Sync Now'}
|
||||||
@@ -316,6 +325,16 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
|||||||
{syncMsg}
|
{syncMsg}
|
||||||
</span>
|
</span>
|
||||||
)}
|
)}
|
||||||
|
{calStatus.last_result && (
|
||||||
|
<button onClick={() => setShowSyncDebug(v => !v)} className="text-[10px] text-slate-500 hover:text-slate-300 underline">
|
||||||
|
{showSyncDebug ? 'hide details' : 'details'}
|
||||||
|
</button>
|
||||||
|
)}
|
||||||
|
{showSyncDebug && calStatus.last_result && (
|
||||||
|
<pre className="basis-full text-[10px] text-slate-400 bg-dark-900 border border-slate-700/50 rounded p-2 overflow-x-auto max-h-64 overflow-y-auto">
|
||||||
|
{JSON.stringify(calStatus.last_result, null, 2)}
|
||||||
|
</pre>
|
||||||
|
)}
|
||||||
{ffPageMsg && (
|
{ffPageMsg && (
|
||||||
<span className={clsx('text-xs', ffPageMsg.startsWith('✓') ? 'text-emerald-400' : ffPageMsg.startsWith('Error') ? 'text-red-400' : 'text-yellow-400')}>
|
<span className={clsx('text-xs', ffPageMsg.startsWith('✓') ? 'text-emerald-400' : ffPageMsg.startsWith('Error') ? 'text-red-400' : 'text-yellow-400')}>
|
||||||
{ffPageMsg}
|
{ffPageMsg}
|
||||||
|
|||||||
@@ -1641,13 +1641,20 @@ export default function Config() {
|
|||||||
</div>
|
</div>
|
||||||
<div className="grid grid-cols-4 gap-4">
|
<div className="grid grid-cols-4 gap-4">
|
||||||
{[
|
{[
|
||||||
{ label: 'NewsAPI Key', value: newsapiKey, setter: setNewsapiKey, placeholder: 'Get at newsapi.org' },
|
{ label: 'NewsAPI Key', value: newsapiKey, setter: setNewsapiKey, placeholder: 'Get at newsapi.org', configKey: 'newsapi_key' },
|
||||||
{ label: 'EIA API Key', value: eiaKey, setter: setEiaKey, placeholder: 'Get at eia.gov' },
|
{ label: 'EIA API Key', value: eiaKey, setter: setEiaKey, placeholder: 'Get at eia.gov', configKey: 'eia_api_key' },
|
||||||
{ label: 'FRED API Key', value: fredKey, setter: setFredKey, placeholder: 'Get at fred.stlouisfed.org' },
|
{ label: 'FRED API Key', value: fredKey, setter: setFredKey, placeholder: 'Get at fred.stlouisfed.org', configKey: 'fred_api_key' },
|
||||||
{ label: 'FMP API Key', value: fmpKey, setter: setFmpKey, placeholder: 'Calendar fallback — financialmodelingprep.com' },
|
{ label: 'FMP API Key', value: fmpKey, setter: setFmpKey, placeholder: 'Calendar fallback — financialmodelingprep.com', configKey: 'fmp_api_key' },
|
||||||
].map(({ label, value, setter, placeholder }) => (
|
].map(({ label, value, setter, placeholder, configKey }) => (
|
||||||
<div key={label}>
|
<div key={label}>
|
||||||
<label className="text-xs text-slate-500 mb-1 block">{label}</label>
|
<label className="text-xs text-slate-500 mb-1 flex items-center justify-between">
|
||||||
|
<span>{label}</span>
|
||||||
|
{config?.[`${configKey}_set`] ? (
|
||||||
|
<span className="text-emerald-400 text-[10px] font-semibold">✓ active</span>
|
||||||
|
) : (
|
||||||
|
<span className="text-slate-600 text-[10px]">not set</span>
|
||||||
|
)}
|
||||||
|
</label>
|
||||||
<input
|
<input
|
||||||
type={showKeys ? 'text' : 'password'}
|
type={showKeys ? 'text' : 'password'}
|
||||||
value={value}
|
value={value}
|
||||||
|
|||||||
Reference in New Issue
Block a user