generated from Hazel/python-project
implemented cache
This commit is contained in:
parent
e27827047e
commit
bee6fb40a3
@ -2,10 +2,12 @@
|
|||||||
|
|
||||||
A Python library for simplified HTTP requests, featuring rate limiting, browser-like headers, and automatic retries. Built on the official `requests` library for reliability.
|
A Python library for simplified HTTP requests, featuring rate limiting, browser-like headers, and automatic retries. Built on the official `requests` library for reliability.
|
||||||
|
|
||||||
|
## Features
|
||||||
|
|
||||||
## ToDo
|
## ToDo
|
||||||
|
|
||||||
- [ ] basic structure
|
- [ ] basic structure
|
||||||
- [ ] caching
|
- [-] caching
|
||||||
- [ ] add cloudscraper
|
- [ ] add cloudscraper
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
@ -6,7 +6,7 @@ from . import cache
|
|||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
c = Connection()
|
c = Connection(cache_enable=False)
|
||||||
c.generate_headers()
|
c.generate_headers()
|
||||||
print(c.session.headers)
|
print(c.session.headers)
|
||||||
|
|
||||||
|
@ -1,14 +1,23 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
from typing import Optional
|
from typing import Optional
|
||||||
import requests
|
import requests
|
||||||
|
from datetime import timedelta
|
||||||
from urllib.parse import urlparse, urlunsplit, ParseResult
|
from urllib.parse import urlparse, urlunsplit, ParseResult
|
||||||
|
|
||||||
from . import cache
|
from . import cache
|
||||||
|
|
||||||
class Connection:
|
class Connection:
|
||||||
def __init__(self, session: Optional[requests.Session] = None) -> None:
|
def __init__(
|
||||||
|
self,
|
||||||
|
session: Optional[requests.Session] = None,
|
||||||
|
cache_enable: bool = True,
|
||||||
|
cache_expires_after: Optional[timedelta] = None
|
||||||
|
) -> None:
|
||||||
self.session = session if session is not None else requests.Session()
|
self.session = session if session is not None else requests.Session()
|
||||||
|
|
||||||
|
self.cache_enable = cache_enable
|
||||||
|
self.cache_expires_after = cache_expires_after if cache_expires_after is not None else timedelta(hours=1)
|
||||||
|
|
||||||
def generate_headers(self, referer: Optional[str] = None):
|
def generate_headers(self, referer: Optional[str] = None):
|
||||||
headers = {
|
headers = {
|
||||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||||
@ -21,10 +30,32 @@ class Connection:
|
|||||||
|
|
||||||
self.session.headers.update(**headers)
|
self.session.headers.update(**headers)
|
||||||
|
|
||||||
def get(self, url: str):
|
def send_request(self, request: requests.Request) -> requests.Response:
|
||||||
if cache.has_cache(url):
|
url = request.url
|
||||||
|
if url is None:
|
||||||
|
raise ValueError("can't send a request without url")
|
||||||
|
if self.cache_enable and cache.has_cache(url):
|
||||||
return cache.get_cache(url)
|
return cache.get_cache(url)
|
||||||
|
|
||||||
|
response = self.session.send(request.prepare())
|
||||||
|
if self.cache_enable:
|
||||||
|
cache.write_cache(url, response)
|
||||||
|
return response
|
||||||
|
|
||||||
resp = self.session.get(url)
|
|
||||||
cache.write_cache(url, resp)
|
def get(self, url: str, headers: Optional[dict] = None, **kwargs) -> requests.Response:
|
||||||
return resp
|
return self.send_request(requests.Request(
|
||||||
|
'GET',
|
||||||
|
url=url,
|
||||||
|
headers=headers,
|
||||||
|
**kwargs
|
||||||
|
))
|
||||||
|
|
||||||
|
def post(self, url: str, headers: Optional[dict] = None, json: Optional[dict] = None, **kwargs) -> requests.Response:
|
||||||
|
return self.send_request(requests.Request(
|
||||||
|
'POST',
|
||||||
|
url=url,
|
||||||
|
headers=headers,
|
||||||
|
json=json,
|
||||||
|
**kwargs,
|
||||||
|
))
|
||||||
|
Loading…
x
Reference in New Issue
Block a user