feat: FF HTML scraper for upcoming weeks with forecasts
- scrape_upcoming(weeks_ahead=5) in ff_calendar.py: fetches forexfactory.com/calendar?week=... HTML for N weeks ahead, parses calendar__table (date/time/currency/impact/event/forecast/previous), converts ET times to UTC, upserts into ff_calendar - Daily scheduler in main.py: runs scrape_upcoming at startup (after 30s delay) then every 24h — no manual action needed - New endpoints: POST /api/eco/ff-scrape?weeks=5, GET /api/eco/ff-scrape/status - CalendarPage: "Scrape Upcoming (5w)" button (indigo) with polling + result Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -159,11 +159,33 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
||||
const s = await fetch(`${API}/api/eco/ff-sync/status`).then(x => x.json())
|
||||
const res = s.last_result || {}
|
||||
const tw = res.thisweek?.events ?? '?'
|
||||
setSyncResult(`✓ ${tw} events synced`)
|
||||
setSyncResult(`✓ live: ${tw} events`)
|
||||
onImported()
|
||||
} catch { setSyncResult('sync error') }
|
||||
}
|
||||
|
||||
const [scrapeMsg, setScrapeMsg] = useState<string | null>(null)
|
||||
const scrapeRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||
|
||||
const startScrape = async () => {
|
||||
setScrapeMsg('scraping…')
|
||||
try {
|
||||
await fetch(`${API}/api/eco/ff-scrape`, { method: 'POST' })
|
||||
scrapeRef.current = setInterval(async () => {
|
||||
const s = await fetch(`${API}/api/eco/ff-scrape/status`).then(x => x.json())
|
||||
if (!s.running) {
|
||||
clearInterval(scrapeRef.current!)
|
||||
const r = s.last_result || {}
|
||||
if (r.error) { setScrapeMsg(`scrape error: ${r.error}`); return }
|
||||
setScrapeMsg(`✓ ${r.total_upserted ?? '?'} events (${r.weeks_scraped} weeks)`)
|
||||
onImported()
|
||||
}
|
||||
}, 3000)
|
||||
} catch { setScrapeMsg('scrape error') }
|
||||
}
|
||||
|
||||
useEffect(() => () => { if (scrapeRef.current) clearInterval(scrapeRef.current) }, [])
|
||||
|
||||
const hasData = stats.total > 0
|
||||
const res = importStatus.last_result
|
||||
|
||||
@@ -176,17 +198,30 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
||||
? <span className="text-yellow-400 animate-pulse">Importing CSV… (auto-import on startup)</span>
|
||||
: hasData
|
||||
? <><span className="text-emerald-400">{stats.total.toLocaleString()} events</span> <span className="text-slate-500 text-xs">({stats.earliest?.slice(0,7)} → {stats.latest?.slice(0,7)})</span></>
|
||||
: <span className="text-slate-400 text-xs">Auto-import will run on next server start (CSV bundled in image)</span>
|
||||
: <span className="text-slate-400 text-xs">Auto-import will run on next server start</span>
|
||||
}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Live sync — this week from faireconomy.media JSON */}
|
||||
<button
|
||||
onClick={startSync}
|
||||
className="px-3 py-1.5 rounded bg-slate-600 hover:bg-slate-500 flex items-center gap-1.5 text-white"
|
||||
title="Update this week actuals from faireconomy.media (Forex Factory JSON)"
|
||||
>
|
||||
<RefreshCw size={13} />
|
||||
Sync Live (this week)
|
||||
Sync Live
|
||||
</button>
|
||||
|
||||
{/* HTML scrape — upcoming 5 weeks with forecasts */}
|
||||
<button
|
||||
onClick={startScrape}
|
||||
disabled={scrapeMsg === 'scraping…'}
|
||||
className="px-3 py-1.5 rounded bg-indigo-700 hover:bg-indigo-600 disabled:opacity-50 flex items-center gap-1.5 text-white"
|
||||
title="Scrape forexfactory.com for upcoming 5 weeks with consensus forecasts"
|
||||
>
|
||||
<RefreshCw size={13} className={clsx(scrapeMsg === 'scraping…' && 'animate-spin')} />
|
||||
Scrape Upcoming (5w)
|
||||
</button>
|
||||
|
||||
{res && !importStatus.running && (
|
||||
@@ -195,6 +230,11 @@ function ImportPanel({ stats, onImported }: { stats: FFStats; onImported: () =>
|
||||
: <span className="text-emerald-400 text-xs flex items-center gap-1"><Check size={12}/> {res.inserted?.toLocaleString()} inserted</span>
|
||||
)}
|
||||
{syncResult && <span className="text-sky-400 text-xs">{syncResult}</span>}
|
||||
{scrapeMsg && (
|
||||
<span className={clsx('text-xs', scrapeMsg.startsWith('✓') ? 'text-emerald-400' : scrapeMsg.includes('error') ? 'text-red-400' : 'text-yellow-400')}>
|
||||
{scrapeMsg}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user