feat: calendar eco
This commit is contained in:
@@ -435,18 +435,52 @@ def _parse_impact(impact_td) -> str:
|
||||
return "low"
|
||||
|
||||
|
||||
def _fetch_via_flaresolverr(url: str) -> Optional[str]:
|
||||
"""
|
||||
Fetch a page through FlareSolverr (a headless-browser proxy that solves Cloudflare's
|
||||
JS challenge) instead of a plain HTTP client. FF started 403'ing our direct requests
|
||||
— the response is Cloudflare's "Just a moment..." interstitial, which no amount of
|
||||
header/cookie spoofing can pass (confirmed: replaying a real logged-in session's
|
||||
cf_clearance cookie from a different IP still 403s — it's IP-bound). FlareSolverr runs
|
||||
as a sibling container (see deploy/docker-compose.yml) and is reached over the internal
|
||||
docker network; not configured (e.g. local dev without that container) → returns None
|
||||
so the caller falls back to the direct httpx path unchanged.
|
||||
"""
|
||||
import os
|
||||
base = os.environ.get("FLARESOLVERR_URL", "http://flaresolverr:8191/v1")
|
||||
try:
|
||||
resp = httpx.post(
|
||||
base,
|
||||
json={"cmd": "request.get", "url": url, "maxTimeout": 60000},
|
||||
timeout=70,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("status") != "ok":
|
||||
print(f"[FF scrape] FlareSolverr error for {url}: {data.get('message')}", flush=True)
|
||||
return None
|
||||
return data.get("solution", {}).get("response")
|
||||
except Exception as e:
|
||||
print(f"[FF scrape] FlareSolverr unavailable ({e}) — falling back to direct fetch", flush=True)
|
||||
return None
|
||||
|
||||
|
||||
def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]:
|
||||
"""Fetch and parse one week of FF calendar. Returns list of row tuples."""
|
||||
url = _week_url(monday)
|
||||
print(f"[FF scrape] GET {url}", flush=True)
|
||||
try:
|
||||
resp = client.get(url, timeout=20)
|
||||
resp.raise_for_status()
|
||||
except Exception as e:
|
||||
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(resp.text, "lxml")
|
||||
html = _fetch_via_flaresolverr(url)
|
||||
if html is None:
|
||||
try:
|
||||
resp = client.get(url, timeout=20)
|
||||
resp.raise_for_status()
|
||||
html = resp.text
|
||||
except Exception as e:
|
||||
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
table = soup.find("table", class_="calendar__table")
|
||||
if not table:
|
||||
# Try alternate selector (FF occasionally restructures)
|
||||
|
||||
Reference in New Issue
Block a user