fix: improved debug for requests
This commit is contained in:
parent
fa9fd5d72b
commit
71d582a17c
@ -101,7 +101,7 @@ class Connection:
|
|||||||
}
|
}
|
||||||
|
|
||||||
if self.HOST is not None:
|
if self.HOST is not None:
|
||||||
headers["Host"] = self.HOST.netloc
|
# headers["Host"] = self.HOST.netloc
|
||||||
headers["Referer"] = self.base_url(url=self.HOST)
|
headers["Referer"] = self.base_url(url=self.HOST)
|
||||||
|
|
||||||
headers.update(header_values)
|
headers.update(header_values)
|
||||||
@ -215,10 +215,6 @@ class Connection:
|
|||||||
self.save(r, name, **kwargs)
|
self.save(r, name, **kwargs)
|
||||||
return r
|
return r
|
||||||
|
|
||||||
if self.SEMANTIC_NOT_FOUND and r.status_code == 404:
|
|
||||||
self.LOGGER.warning(f"Couldn't find url (404): {request_url}")
|
|
||||||
return None
|
|
||||||
|
|
||||||
# the server rejected the request, or the internet is lacking
|
# the server rejected the request, or the internet is lacking
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
self.LOGGER.warning(f"Request timed out at \"{request_url}\": ({try_count}-{self.TRIES})")
|
self.LOGGER.warning(f"Request timed out at \"{request_url}\": ({try_count}-{self.TRIES})")
|
||||||
@ -231,15 +227,20 @@ class Connection:
|
|||||||
finally:
|
finally:
|
||||||
self.lock = False
|
self.lock = False
|
||||||
|
|
||||||
if not connection_failed:
|
if r is None:
|
||||||
|
self.LOGGER.warning(f"{self.HOST.netloc} didn't respond at {url}. ({try_count}-{self.TRIES})")
|
||||||
|
self.LOGGER.debug("request headers:\n\t"+ "\n\t".join(f"{k}\t=\t{v}" for k, v in headers.items()))
|
||||||
|
else:
|
||||||
self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} at {url}. ({try_count}-{self.TRIES})")
|
self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} at {url}. ({try_count}-{self.TRIES})")
|
||||||
if r is not None:
|
self.LOGGER.debug("request headers:\n\t"+ "\n\t".join(f"{k}\t=\t{v}" for k, v in r.request.headers.items()))
|
||||||
self.LOGGER.debug("request headers:\n\t"+ "\n\t".join(f"{k}\t=\t{v}" for k, v in r.request.headers.items()))
|
self.LOGGER.debug("response headers:\n\t"+ "\n\t".join(f"{k}\t=\t{v}" for k, v in r.headers.items()))
|
||||||
self.LOGGER.debug("response headers:\n\t"+ "\n\t".join(f"{k}\t=\t{v}" for k, v in r.headers.items()))
|
self.LOGGER.debug(r.content)
|
||||||
self.LOGGER.debug(r.content)
|
|
||||||
|
|
||||||
if name != "":
|
if name != "":
|
||||||
self.save(r, name, error=True, **kwargs)
|
self.save(r, name, error=True, **kwargs)
|
||||||
|
|
||||||
|
if self.SEMANTIC_NOT_FOUND and r.status_code == 404:
|
||||||
|
return None
|
||||||
|
|
||||||
if sleep_after_404 != 0:
|
if sleep_after_404 != 0:
|
||||||
self.LOGGER.warning(f"Waiting for {sleep_after_404} seconds.")
|
self.LOGGER.warning(f"Waiting for {sleep_after_404} seconds.")
|
||||||
|
@ -13,7 +13,7 @@ if not load_dotenv(Path(__file__).parent.parent.parent / ".env"):
|
|||||||
__stage__ = os.getenv("STAGE", "prod")
|
__stage__ = os.getenv("STAGE", "prod")
|
||||||
|
|
||||||
DEBUG = (__stage__ == "dev") and True
|
DEBUG = (__stage__ == "dev") and True
|
||||||
DEBUG_LOGGING = DEBUG and False
|
DEBUG_LOGGING = DEBUG and True
|
||||||
DEBUG_TRACE = DEBUG and True
|
DEBUG_TRACE = DEBUG and True
|
||||||
DEBUG_OBJECT_TRACE = DEBUG and False
|
DEBUG_OBJECT_TRACE = DEBUG and False
|
||||||
DEBUG_YOUTUBE_INITIALIZING = DEBUG and False
|
DEBUG_YOUTUBE_INITIALIZING = DEBUG and False
|
||||||
|
Loading…
Reference in New Issue
Block a user