Hazel/python-requests
1
0
Fork 0
generated from Hazel/python-project
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

implemented max retries

Browse Source
This commit is contained in:
Hazel Noack 2025-06-11 13:50:45 +02:00
parent 6743b939ad
commit 593c975418
1 changed files with 8 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
python_requests/connections.py
View File

@ -20,6 +20,7 @@ class Connection:
additional_delay_per_try: float = 1, additional_delay_per_try: float = 1,
error_status_codes: Optional[Set[int]] = None, error_status_codes: Optional[Set[int]] = None,
rate_limit_status_codes: Optional[Set[int]] = None, rate_limit_status_codes: Optional[Set[int]] = None,
max_retries: Optional[int] = 5,
) -> None: ) -> None:
self.session = session if session is not None else requests.Session() self.session = session if session is not None else requests.Session()
@ -56,6 +57,8 @@ class Connection:
509, # Bandwidth Limit Exceeded 509, # Bandwidth Limit Exceeded
} }
self.max_retries = max_retries
def generate_headers(self, referer: Optional[str] = None): def generate_headers(self, referer: Optional[str] = None):
headers = { headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0", "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
@ -119,8 +122,12 @@ class Connection:
self.last_request = time.time() self.last_request = time.time()
if not self.validate_response(response): if not self.validate_response(response):
if self.max_retries is not None and self.max_retries <= attempt:
raise requests.HTTPError(
f"Max retries exceeded, server is rate limiting {response.status_code}: {response.reason}",
response=response
)
return self.send_request(request, attempt=attempt+1) return self.send_request(request, attempt=attempt+1)
if self.cache_enable: if self.cache_enable: