added the config option how long to wait after yt returns 404

This commit is contained in:
Hellow
2023-06-22 14:30:26 +02:00
parent d08abe42cb
commit 400b7d3d6e
4 changed files with 22 additions and 11 deletions

View File

@@ -22,7 +22,8 @@ class Connection:
logger: logging.Logger = logging.getLogger("connection"),
header_values: Dict[str, str] = None,
accepted_response_codes: Set[int] = None,
semantic_not_found: bool = True
semantic_not_found: bool = True,
sleep_after_404: float = 0.0
):
if proxies is None:
proxies = PROXIES_LIST
@@ -39,6 +40,7 @@ class Connection:
self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200}
self.SEMANTIC_NOT_FOUND = semantic_not_found
self.sleep_after_404 = sleep_after_404
self.session = requests.Session()
self.session.headers = self.get_header(**self.HEADER_VALUES)
@@ -86,9 +88,11 @@ class Connection:
headers: dict,
refer_from_origin: bool = True,
raw_url: bool = False,
wait_on_403: bool = True,
sleep_after_404: float = None,
**kwargs
) -> Optional[requests.Response]:
if sleep_after_404 is None:
sleep_after_404 = self.sleep_after_404
if try_count >= self.TRIES:
return
@@ -127,9 +131,9 @@ class Connection:
self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} "
f"at {url}. ({try_count}-{self.TRIES})")
self.LOGGER.debug(r.content)
if wait_on_403:
self.LOGGER.warning(f"Waiting for 5 seconds.")
time.sleep(5)
if sleep_after_404 != 0:
self.LOGGER.warning(f"Waiting for {sleep_after_404} seconds.")
time.sleep(sleep_after_404)
self.rotate()
@@ -140,6 +144,7 @@ class Connection:
url=url,
timeout=timeout,
headers=headers,
sleep_after_404=sleep_after_404,
**kwargs
)