theoretically implemented the request partially
This commit is contained in:
parent
d6d20aaf1b
commit
e3b1a866a1
@ -0,0 +1 @@
|
|||||||
|
from .connection import Connection
|
@ -7,8 +7,6 @@ import requests
|
|||||||
from .rotating import RotatingProxy
|
from .rotating import RotatingProxy
|
||||||
from ..utils.shared import PROXIES_LIST
|
from ..utils.shared import PROXIES_LIST
|
||||||
|
|
||||||
LOGGER = logging.getLogger("connection")
|
|
||||||
|
|
||||||
|
|
||||||
class Connection:
|
class Connection:
|
||||||
def __init__(
|
def __init__(
|
||||||
@ -17,6 +15,7 @@ class Connection:
|
|||||||
proxies: List[dict] = None,
|
proxies: List[dict] = None,
|
||||||
tries: int = (len(PROXIES_LIST) + 1) * 2,
|
tries: int = (len(PROXIES_LIST) + 1) * 2,
|
||||||
timeout: int = 7,
|
timeout: int = 7,
|
||||||
|
logger: logging.Logger = logging.getLogger("connection"),
|
||||||
header_values: Dict[str, str] = None,
|
header_values: Dict[str, str] = None,
|
||||||
session: requests.Session = None,
|
session: requests.Session = None,
|
||||||
accepted_response_codes: Set[int] = None,
|
accepted_response_codes: Set[int] = None,
|
||||||
@ -27,7 +26,7 @@ class Connection:
|
|||||||
if header_values is None:
|
if header_values is None:
|
||||||
header_values = dict()
|
header_values = dict()
|
||||||
|
|
||||||
self.LOGGER = LOGGER
|
self.LOGGER = logger
|
||||||
self.HOST = urlparse(host)
|
self.HOST = urlparse(host)
|
||||||
self.TRIES = tries
|
self.TRIES = tries
|
||||||
self.TIMEOUT = timeout
|
self.TIMEOUT = timeout
|
||||||
@ -74,14 +73,18 @@ class Connection:
|
|||||||
try_count: int,
|
try_count: int,
|
||||||
accepted_response_code: set,
|
accepted_response_code: set,
|
||||||
url: str,
|
url: str,
|
||||||
|
timeout: float,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> Optional[requests.Response]:
|
) -> Optional[requests.Response]:
|
||||||
if try_count >= self.TRIES:
|
if try_count >= self.TRIES:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if timeout is None:
|
||||||
|
timeout = self.TIMEOUT
|
||||||
|
|
||||||
retry = False
|
retry = False
|
||||||
try:
|
try:
|
||||||
r = request(url=url, **kwargs)
|
r = request(url=url, timeout=timeout, **kwargs)
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})")
|
self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})")
|
||||||
retry = True
|
retry = True
|
||||||
@ -116,6 +119,7 @@ class Connection:
|
|||||||
url: str,
|
url: str,
|
||||||
stream: bool = False,
|
stream: bool = False,
|
||||||
accepted_response_codes: set = None,
|
accepted_response_codes: set = None,
|
||||||
|
timeout: float = None,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> Optional[requests.Response]:
|
) -> Optional[requests.Response]:
|
||||||
r = self._request(
|
r = self._request(
|
||||||
@ -124,6 +128,7 @@ class Connection:
|
|||||||
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
|
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
|
||||||
url=url,
|
url=url,
|
||||||
stream=stream,
|
stream=stream,
|
||||||
|
timeout=timeout,
|
||||||
**kwargs
|
**kwargs
|
||||||
)
|
)
|
||||||
if r is None:
|
if r is None:
|
||||||
@ -136,6 +141,7 @@ class Connection:
|
|||||||
json: dict,
|
json: dict,
|
||||||
stream: bool = False,
|
stream: bool = False,
|
||||||
accepted_response_codes: set = None,
|
accepted_response_codes: set = None,
|
||||||
|
timeout: float = None,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> Optional[requests.Response]:
|
) -> Optional[requests.Response]:
|
||||||
r = self._request(
|
r = self._request(
|
||||||
@ -143,6 +149,7 @@ class Connection:
|
|||||||
try_count=0,
|
try_count=0,
|
||||||
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
|
accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES,
|
||||||
url=url,
|
url=url,
|
||||||
|
timeout=timeout,
|
||||||
json=json,
|
json=json,
|
||||||
stream=stream,
|
stream=stream,
|
||||||
**kwargs
|
**kwargs
|
||||||
|
Loading…
Reference in New Issue
Block a user