generated from Hazel/python-project
implemented cache
This commit is contained in:
parent
e27827047e
commit
bee6fb40a3
@ -2,10 +2,12 @@
|
||||
|
||||
A Python library for simplified HTTP requests, featuring rate limiting, browser-like headers, and automatic retries. Built on the official `requests` library for reliability.
|
||||
|
||||
## Features
|
||||
|
||||
## ToDo
|
||||
|
||||
- [ ] basic structure
|
||||
- [ ] caching
|
||||
- [-] caching
|
||||
- [ ] add cloudscraper
|
||||
|
||||
## License
|
||||
|
@ -6,7 +6,7 @@ from . import cache
|
||||
|
||||
|
||||
def main():
|
||||
c = Connection()
|
||||
c = Connection(cache_enable=False)
|
||||
c.generate_headers()
|
||||
print(c.session.headers)
|
||||
|
||||
|
@ -1,14 +1,23 @@
|
||||
from __future__ import annotations
|
||||
from typing import Optional
|
||||
import requests
|
||||
from datetime import timedelta
|
||||
from urllib.parse import urlparse, urlunsplit, ParseResult
|
||||
|
||||
from . import cache
|
||||
|
||||
class Connection:
|
||||
def __init__(self, session: Optional[requests.Session] = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
session: Optional[requests.Session] = None,
|
||||
cache_enable: bool = True,
|
||||
cache_expires_after: Optional[timedelta] = None
|
||||
) -> None:
|
||||
self.session = session if session is not None else requests.Session()
|
||||
|
||||
self.cache_enable = cache_enable
|
||||
self.cache_expires_after = cache_expires_after if cache_expires_after is not None else timedelta(hours=1)
|
||||
|
||||
def generate_headers(self, referer: Optional[str] = None):
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
|
||||
@ -21,10 +30,32 @@ class Connection:
|
||||
|
||||
self.session.headers.update(**headers)
|
||||
|
||||
def get(self, url: str):
|
||||
if cache.has_cache(url):
|
||||
def send_request(self, request: requests.Request) -> requests.Response:
|
||||
url = request.url
|
||||
if url is None:
|
||||
raise ValueError("can't send a request without url")
|
||||
if self.cache_enable and cache.has_cache(url):
|
||||
return cache.get_cache(url)
|
||||
|
||||
resp = self.session.get(url)
|
||||
cache.write_cache(url, resp)
|
||||
return resp
|
||||
response = self.session.send(request.prepare())
|
||||
if self.cache_enable:
|
||||
cache.write_cache(url, response)
|
||||
return response
|
||||
|
||||
|
||||
def get(self, url: str, headers: Optional[dict] = None, **kwargs) -> requests.Response:
|
||||
return self.send_request(requests.Request(
|
||||
'GET',
|
||||
url=url,
|
||||
headers=headers,
|
||||
**kwargs
|
||||
))
|
||||
|
||||
def post(self, url: str, headers: Optional[dict] = None, json: Optional[dict] = None, **kwargs) -> requests.Response:
|
||||
return self.send_request(requests.Request(
|
||||
'POST',
|
||||
url=url,
|
||||
headers=headers,
|
||||
json=json,
|
||||
**kwargs,
|
||||
))
|
||||
|
Loading…
x
Reference in New Issue
Block a user