From 474f18404f40a1a1b20b454795458555f7a68acd Mon Sep 17 00:00:00 2001 From: laurentbarontini Date: Wed, 6 May 2026 19:34:36 +0200 Subject: [PATCH] Bug crawl --- pipeline.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pipeline.py b/pipeline.py index 7326095..3daf6ba 100644 --- a/pipeline.py +++ b/pipeline.py @@ -95,9 +95,21 @@ def _log(session: SASession, run, msg: str) -> None: # ── Web fetchers ─────────────────────────────────────────────────────────────── +_HEADERS = { + 'User-Agent': ( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36' + ), + 'Accept-Language': 'en-US,en;q=0.9', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + + def _fetch_webpage(url: str) -> tuple: - headers = {'User-Agent': 'Mozilla/5.0 (compatible; OpenCare-Bot/1.0)'} - r = requests.get(url, headers=headers, timeout=20) + r = requests.get(url, headers=_HEADERS, timeout=20) + if r.status_code == 429: + raise requests.HTTPError(f'429 Too Many Requests – {url}', response=r) r.raise_for_status() soup = BeautifulSoup(r.content, 'html.parser') for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside', 'iframe', 'form']): @@ -136,8 +148,9 @@ def _fetch_index_page(url: str) -> list: 'login', 'search', 'contact', 'about', 'privacy', 'terms', 'twitter.com', 'facebook.com', 'instagram.com', 'youtube.com', ) - headers = {'User-Agent': 'Mozilla/5.0 (compatible; OpenCare-Bot/1.0)'} - r = requests.get(url, headers=headers, timeout=20) + r = requests.get(url, headers=_HEADERS, timeout=20) + if r.status_code == 429: + raise requests.HTTPError(f'429 Too Many Requests – {url}', response=r) r.raise_for_status() soup = BeautifulSoup(r.content, 'html.parser') base = urlparse(url)