implemented cache

This commit is contained in:
Hazel Noack 2025-06-11 12:44:57 +02:00
parent e27827047e
commit bee6fb40a3
3 changed files with 41 additions and 8 deletions

View File

@ -2,10 +2,12 @@
A Python library for simplified HTTP requests, featuring rate limiting, browser-like headers, and automatic retries. Built on the official `requests` library for reliability.
## Features
## ToDo
- [ ] basic structure
- [ ] caching
- [-] caching
- [ ] add cloudscraper
## License

View File

@ -6,7 +6,7 @@ from . import cache
def main():
c = Connection()
c = Connection(cache_enable=False)
c.generate_headers()
print(c.session.headers)

View File

@ -1,14 +1,23 @@
from __future__ import annotations
from typing import Optional
import requests
from datetime import timedelta
from urllib.parse import urlparse, urlunsplit, ParseResult
from . import cache
class Connection:
def __init__(self, session: Optional[requests.Session] = None) -> None:
def __init__(
self,
session: Optional[requests.Session] = None,
cache_enable: bool = True,
cache_expires_after: Optional[timedelta] = None
) -> None:
self.session = session if session is not None else requests.Session()
self.cache_enable = cache_enable
self.cache_expires_after = cache_expires_after if cache_expires_after is not None else timedelta(hours=1)
def generate_headers(self, referer: Optional[str] = None):
headers = {
"User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:139.0) Gecko/20100101 Firefox/139.0",
@ -21,10 +30,32 @@ class Connection:
self.session.headers.update(**headers)
def get(self, url: str):
if cache.has_cache(url):
def send_request(self, request: requests.Request) -> requests.Response:
url = request.url
if url is None:
raise ValueError("can't send a request without url")
if self.cache_enable and cache.has_cache(url):
return cache.get_cache(url)
response = self.session.send(request.prepare())
if self.cache_enable:
cache.write_cache(url, response)
return response
resp = self.session.get(url)
cache.write_cache(url, resp)
return resp
def get(self, url: str, headers: Optional[dict] = None, **kwargs) -> requests.Response:
return self.send_request(requests.Request(
'GET',
url=url,
headers=headers,
**kwargs
))
def post(self, url: str, headers: Optional[dict] = None, json: Optional[dict] = None, **kwargs) -> requests.Response:
return self.send_request(requests.Request(
'POST',
url=url,
headers=headers,
json=json,
**kwargs,
))