admin/opencare
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bug crawl

Browse Source
This commit is contained in:
laurentbarontini
2026-05-06 19:34:36 +02:00
parent d8c24992ec
commit 474f18404f
1 changed files with 17 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
pipeline.py
View File

@@ -95,9 +95,21 @@ def _log(session: SASession, run, msg: str) -> None:
# ── Web fetchers ─────────────────────────────────────────────────────────────── # ── Web fetchers ───────────────────────────────────────────────────────────────
_HEADERS = {
'User-Agent': (
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
'AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/124.0.0.0 Safari/537.36'
),
'Accept-Language': 'en-US,en;q=0.9',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
}
def _fetch_webpage(url: str) -> tuple: def _fetch_webpage(url: str) -> tuple:
headers = {'User-Agent': 'Mozilla/5.0 (compatible; OpenCare-Bot/1.0)'} r = requests.get(url, headers=_HEADERS, timeout=20)
r = requests.get(url, headers=headers, timeout=20) if r.status_code == 429:
raise requests.HTTPError(f'429 Too Many Requests – {url}', response=r)
r.raise_for_status() r.raise_for_status()
soup = BeautifulSoup(r.content, 'html.parser') soup = BeautifulSoup(r.content, 'html.parser')
for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside', 'iframe', 'form']): for tag in soup(['script', 'style', 'nav', 'footer', 'header', 'aside', 'iframe', 'form']):
@@ -136,8 +148,9 @@ def _fetch_index_page(url: str) -> list:
'login', 'search', 'contact', 'about', 'privacy', 'terms', 'login', 'search', 'contact', 'about', 'privacy', 'terms',
'twitter.com', 'facebook.com', 'instagram.com', 'youtube.com', 'twitter.com', 'facebook.com', 'instagram.com', 'youtube.com',
) )
headers = {'User-Agent': 'Mozilla/5.0 (compatible; OpenCare-Bot/1.0)'} r = requests.get(url, headers=_HEADERS, timeout=20)
r = requests.get(url, headers=headers, timeout=20) if r.status_code == 429:
raise requests.HTTPError(f'429 Too Many Requests – {url}', response=r)
r.raise_for_status() r.raise_for_status()
soup = BeautifulSoup(r.content, 'html.parser') soup = BeautifulSoup(r.content, 'html.parser')
base = urlparse(url) base = urlparse(url)