feat: FF HTML scraper for upcoming weeks with forecasts

- scrape_upcoming(weeks_ahead=5) in ff_calendar.py:
  fetches forexfactory.com/calendar?week=... HTML for N weeks ahead,
  parses calendar__table (date/time/currency/impact/event/forecast/previous),
  converts ET times to UTC, upserts into ff_calendar
- Daily scheduler in main.py: runs scrape_upcoming at startup (after 30s delay)
  then every 24h — no manual action needed
- New endpoints: POST /api/eco/ff-scrape?weeks=5, GET /api/eco/ff-scrape/status
- CalendarPage: "Scrape Upcoming (5w)" button (indigo) with polling + result

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
OpenSquared
2026-06-26 17:14:00 +02:00
parent a69ced8fff
commit c36b2b2198
4 changed files with 305 additions and 5 deletions

View File

@@ -159,11 +159,33 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
const s = await fetch(`${API}/api/eco/ff-sync/status`).then(x => x.json())
const res = s.last_result || {}
const tw = res.thisweek?.events ?? '?'
setSyncResult(`${tw} events synced`)
setSyncResult(` live: ${tw} events`)
onImported()
} catch { setSyncResult('sync error') }
}
const [scrapeMsg, setScrapeMsg] = useState<string | null>(null)
const scrapeRef = useRef<ReturnType<typeof setInterval> | null>(null)
const startScrape = async () => {
setScrapeMsg('scraping…')
try {
await fetch(`${API}/api/eco/ff-scrape`, { method: 'POST' })
scrapeRef.current = setInterval(async () => {
const s = await fetch(`${API}/api/eco/ff-scrape/status`).then(x => x.json())
if (!s.running) {
clearInterval(scrapeRef.current!)
const r = s.last_result || {}
if (r.error) { setScrapeMsg(`scrape error: ${r.error}`); return }
setScrapeMsg(`${r.total_upserted ?? '?'} events (${r.weeks_scraped} weeks)`)
onImported()
}
}, 3000)
} catch { setScrapeMsg('scrape error') }
}
useEffect(() => () => { if (scrapeRef.current) clearInterval(scrapeRef.current) }, [])
const hasData = stats.total > 0
const res = importStatus.last_result
@@ -176,17 +198,30 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
? <span className="text-yellow-400 animate-pulse">Importing CSV (auto-import on startup)</span>
: hasData
? <><span className="text-emerald-400">{stats.total.toLocaleString()} events</span> <span className="text-slate-500 text-xs">({stats.earliest?.slice(0,7)} {stats.latest?.slice(0,7)})</span></>
: <span className="text-slate-400 text-xs">Auto-import will run on next server start (CSV bundled in image)</span>
: <span className="text-slate-400 text-xs">Auto-import will run on next server start</span>
}
</div>
</div>
{/* Live sync — this week from faireconomy.media JSON */}
<button
onClick={startSync}
className="px-3 py-1.5 rounded bg-slate-600 hover:bg-slate-500 flex items-center gap-1.5 text-white"
title="Update this week actuals from faireconomy.media (Forex Factory JSON)"
>
<RefreshCw size={13} />
Sync Live (this week)
Sync Live
</button>
{/* HTML scrape — upcoming 5 weeks with forecasts */}
<button
onClick={startScrape}
disabled={scrapeMsg === 'scraping…'}
className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
title="Scrape forexfactory.com for upcoming 5 weeks with consensus forecasts"
>
<RefreshCw size={13} className={clsx(scrapeMsg === 'scraping…' && 'animate-spin')} />
Scrape Upcoming (5w)
</button>
{res && !importStatus.running && (
@@ -195,6 +230,11 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
: <span className="text-emerald-400 text-xs flex items-center gap-1"><Check size={12}/> {res.inserted?.toLocaleString()} inserted</span>
)}
{syncResult && <span className="text-sky-400 text-xs">{syncResult}</span>}
{scrapeMsg && (
<span className={clsx('text-xs', scrapeMsg.startsWith('✓') ? 'text-emerald-400' : scrapeMsg.includes('error') ? 'text-red-400' : 'text-yellow-400')}>
{scrapeMsg}
</span>
)}
</div>
)
}