feat: calendar eco

This commit is contained in:
OpenSquared
2026-07-21 14:49:59 +02:00
parent 59fbb9b023
commit 5f23141992
5 changed files with 67 additions and 13 deletions

View File

@@ -435,18 +435,52 @@ def _parse_impact(impact_td) -> str:
return "low"
def _fetch_via_flaresolverr(url: str) -> Optional[str]:
"""
Fetch a page through FlareSolverr (a headless-browser proxy that solves Cloudflare's
JS challenge) instead of a plain HTTP client. FF started 403'ing our direct requests
— the response is Cloudflare's "Just a moment..." interstitial, which no amount of
header/cookie spoofing can pass (confirmed: replaying a real logged-in session's
cf_clearance cookie from a different IP still 403s — it's IP-bound). FlareSolverr runs
as a sibling container (see deploy/docker-compose.yml) and is reached over the internal
docker network; not configured (e.g. local dev without that container) → returns None
so the caller falls back to the direct httpx path unchanged.
"""
import os
base = os.environ.get("FLARESOLVERR_URL", "http://flaresolverr:8191/v1")
try:
resp = httpx.post(
base,
json={"cmd": "request.get", "url": url, "maxTimeout": 60000},
timeout=70,
)
resp.raise_for_status()
data = resp.json()
if data.get("status") != "ok":
print(f"[FF scrape] FlareSolverr error for {url}: {data.get('message')}", flush=True)
return None
return data.get("solution", {}).get("response")
except Exception as e:
print(f"[FF scrape] FlareSolverr unavailable ({e}) — falling back to direct fetch", flush=True)
return None
def _scrape_week(client: httpx.Client, monday: date) -> list[tuple]:
"""Fetch and parse one week of FF calendar. Returns list of row tuples."""
url = _week_url(monday)
print(f"[FF scrape] GET {url}", flush=True)
try:
resp = client.get(url, timeout=20)
resp.raise_for_status()
except Exception as e:
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
return []
soup = BeautifulSoup(resp.text, "lxml")
html = _fetch_via_flaresolverr(url)
if html is None:
try:
resp = client.get(url, timeout=20)
resp.raise_for_status()
html = resp.text
except Exception as e:
print(f"[FF scrape] fetch error {url}: {e}", flush=True)
return []
soup = BeautifulSoup(html, "lxml")
table = soup.find("table", class_="calendar__table")
if not table:
# Try alternate selector (FF occasionally restructures)