theoretically implemented the request partially

This commit is contained in:
Hellow 2023-04-20 19:37:41 +02:00
parent d6d20aaf1b
commit e3b1a866a1
2 changed files with 12 additions and 4 deletions

View File

@ -0,0 +1 @@
from .connection import Connection

View File

@ -7,8 +7,6 @@ import requests
from .rotating import RotatingProxy from .rotating import RotatingProxy
from ..utils.shared import PROXIES_LIST from ..utils.shared import PROXIES_LIST
LOGGER = logging.getLogger("connection")
class Connection: class Connection:
def __init__( def __init__(
@ -17,6 +15,7 @@ class Connection:
proxies: List[dict] = None, proxies: List[dict] = None,
tries: int = (len(PROXIES_LIST) + 1) * 2, tries: int = (len(PROXIES_LIST) + 1) * 2,
timeout: int = 7, timeout: int = 7,
logger: logging.Logger = logging.getLogger("connection"),
header_values: Dict[str, str] = None, header_values: Dict[str, str] = None,
session: requests.Session = None, session: requests.Session = None,
accepted_response_codes: Set[int] = None, accepted_response_codes: Set[int] = None,
@ -27,7 +26,7 @@ class Connection:
if header_values is None: if header_values is None:
header_values = dict() header_values = dict()
self.LOGGER = LOGGER self.LOGGER = logger
self.HOST = urlparse(host) self.HOST = urlparse(host)
self.TRIES = tries self.TRIES = tries
self.TIMEOUT = timeout self.TIMEOUT = timeout
@ -74,14 +73,18 @@ class Connection:
try_count: int, try_count: int,
accepted_response_code: set, accepted_response_code: set,
url: str, url: str,
timeout: float,
**kwargs **kwargs
) -> Optional[requests.Response]: ) -> Optional[requests.Response]:
if try_count >= self.TRIES: if try_count >= self.TRIES:
return return
if timeout is None:
timeout = self.TIMEOUT
retry = False retry = False
try: try:
r = request(url=url, **kwargs) r = request(url=url, timeout=timeout, **kwargs)
except requests.exceptions.Timeout: except requests.exceptions.Timeout:
self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})") self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})")
retry = True retry = True
@ -116,6 +119,7 @@ class Connection:
url: str, url: str,
stream: bool = False, stream: bool = False,
accepted_response_codes: set = None, accepted_response_codes: set = None,
timeout: float = None,
**kwargs **kwargs
) -> Optional[requests.Response]: ) -> Optional[requests.Response]:
r = self._request( r = self._request(
@ -124,6 +128,7 @@ class Connection:
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
url=url, url=url,
stream=stream, stream=stream,
timeout=timeout,
**kwargs **kwargs
) )
if r is None: if r is None:
@ -136,6 +141,7 @@ class Connection:
json: dict, json: dict,
stream: bool = False, stream: bool = False,
accepted_response_codes: set = None, accepted_response_codes: set = None,
timeout: float = None,
**kwargs **kwargs
) -> Optional[requests.Response]: ) -> Optional[requests.Response]:
r = self._request( r = self._request(
@ -143,6 +149,7 @@ class Connection:
try_count=0, try_count=0,
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
url=url, url=url,
timeout=timeout,
json=json, json=json,
stream=stream, stream=stream,
**kwargs **kwargs