added the config option how long to wait after yt returns 404

This commit is contained in:
Hellow 2023-06-22 14:30:26 +02:00
parent d08abe42cb
commit 400b7d3d6e
4 changed files with 22 additions and 11 deletions

View File

@ -22,7 +22,8 @@ class Connection:
logger: logging.Logger = logging.getLogger("connection"), logger: logging.Logger = logging.getLogger("connection"),
header_values: Dict[str, str] = None, header_values: Dict[str, str] = None,
accepted_response_codes: Set[int] = None, accepted_response_codes: Set[int] = None,
semantic_not_found: bool = True semantic_not_found: bool = True,
sleep_after_404: float = 0.0
): ):
if proxies is None: if proxies is None:
proxies = PROXIES_LIST proxies = PROXIES_LIST
@ -39,6 +40,7 @@ class Connection:
self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200} self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200}
self.SEMANTIC_NOT_FOUND = semantic_not_found self.SEMANTIC_NOT_FOUND = semantic_not_found
self.sleep_after_404 = sleep_after_404
self.session = requests.Session() self.session = requests.Session()
self.session.headers = self.get_header(**self.HEADER_VALUES) self.session.headers = self.get_header(**self.HEADER_VALUES)
@ -86,9 +88,11 @@ class Connection:
headers: dict, headers: dict,
refer_from_origin: bool = True, refer_from_origin: bool = True,
raw_url: bool = False, raw_url: bool = False,
wait_on_403: bool = True, sleep_after_404: float = None,
**kwargs **kwargs
) -> Optional[requests.Response]: ) -> Optional[requests.Response]:
if sleep_after_404 is None:
sleep_after_404 = self.sleep_after_404
if try_count >= self.TRIES: if try_count >= self.TRIES:
return return
@ -127,9 +131,9 @@ class Connection:
self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} "
f"at {url}. ({try_count}-{self.TRIES})") f"at {url}. ({try_count}-{self.TRIES})")
self.LOGGER.debug(r.content) self.LOGGER.debug(r.content)
if wait_on_403: if sleep_after_404 != 0:
self.LOGGER.warning(f"Waiting for 5 seconds.") self.LOGGER.warning(f"Waiting for {sleep_after_404} seconds.")
time.sleep(5) time.sleep(sleep_after_404)
self.rotate() self.rotate()
@ -140,6 +144,7 @@ class Connection:
url=url, url=url,
timeout=timeout, timeout=timeout,
headers=headers, headers=headers,
sleep_after_404=sleep_after_404,
**kwargs **kwargs
) )

View File

@ -20,7 +20,7 @@ from ..objects import (
) )
from ..connection import Connection from ..connection import Connection
from ..utils.support_classes import DownloadResult from ..utils.support_classes import DownloadResult
from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK, PIPED_INSTANCE from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK, PIPED_INSTANCE, SLEEP_AFTER_YOUTUBE_403
""" """
@ -148,7 +148,8 @@ class YouTube(Page):
self.download_connection: Connection = Connection( self.download_connection: Connection = Connection(
host="https://www.youtube.com/", host="https://www.youtube.com/",
logger=self.LOGGER logger=self.LOGGER,
sleep_after_404=SLEEP_AFTER_YOUTUBE_403
) )
# the stuff with the connection is, to ensure sponsorblock uses the proxies, my programm does # the stuff with the connection is, to ensure sponsorblock uses the proxies, my programm does
@ -422,11 +423,8 @@ class YouTube(Page):
endpoint = audio_format["url"] endpoint = audio_format["url"]
self.download_connection.stream_into(endpoint, target, description=desc, raw_url=True) return self.download_connection.stream_into(endpoint, target, description=desc, raw_url=True)
if self.download_connection.get(endpoint, stream=True, raw_url=True):
return DownloadResult(total=1)
return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}")
def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]: def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]:
if not ENABLE_SPONSOR_BLOCK: if not ENABLE_SPONSOR_BLOCK:

View File

@ -100,6 +100,12 @@ class ConnectionSection(Section):
"Hidden instances (.onion) will only work, when setting 'tor=true'.", "Hidden instances (.onion) will only work, when setting 'tor=true'.",
value="https://pipedapi.kavin.rocks" value="https://pipedapi.kavin.rocks"
) )
self.SLEEP_AFTER_YOUTUBE_403 = FloatAttribute(
name="sleep_after_youtube_403",
description="The time to wait, after youtube returned 403 (in seconds)",
value="20"
)
self.ALL_YOUTUBE_URLS = UrlListAttribute( self.ALL_YOUTUBE_URLS = UrlListAttribute(
name="youtube_url", name="youtube_url",
@ -126,6 +132,7 @@ class ConnectionSection(Section):
self.SHOW_DOWNLOAD_ERRORS_THRESHOLD, self.SHOW_DOWNLOAD_ERRORS_THRESHOLD,
self.INVIDIOUS_INSTANCE, self.INVIDIOUS_INSTANCE,
self.PIPED_INSTANCE, self.PIPED_INSTANCE,
self.SLEEP_AFTER_YOUTUBE_403,
self.ALL_YOUTUBE_URLS, self.ALL_YOUTUBE_URLS,
self.SPONSOR_BLOCK self.SPONSOR_BLOCK
] ]

View File

@ -124,3 +124,4 @@ have fun :3
FFMPEG_BINARY: Path = PATHS_SECTION.FFMPEG_BINARY.object_from_value FFMPEG_BINARY: Path = PATHS_SECTION.FFMPEG_BINARY.object_from_value
HASNT_YET_STARTED: bool = MISC_SECTION.HASNT_YET_STARTED.object_from_value HASNT_YET_STARTED: bool = MISC_SECTION.HASNT_YET_STARTED.object_from_value
SLEEP_AFTER_YOUTUBE_403: float = CONNECTION_SECTION.SLEEP_AFTER_YOUTUBE_403.object_from_value