feat: caching the ytdl responses

This commit is contained in:
Hazel 2024-04-10 09:28:28 +02:00
parent b3cdd5e94d
commit 0e2de49831
4 changed files with 51 additions and 15 deletions

View File

@ -127,7 +127,8 @@ class Song(Base):
id3Mapping.ISRC: [self.isrc], id3Mapping.ISRC: [self.isrc],
id3Mapping.LENGTH: [self.length], id3Mapping.LENGTH: [self.length],
id3Mapping.GENRE: [self.genre], id3Mapping.GENRE: [self.genre],
id3Mapping.TRACKNUMBER: [self.tracksort_str] id3Mapping.TRACKNUMBER: [self.tracksort_str],
id3Mapping.COMMENT: [self.note.markdown],
}) })
# metadata.merge_many([s.get_song_metadata() for s in self.source_collection]) album sources have no relevant metadata for id3 # metadata.merge_many([s.get_song_metadata() for s in self.source_collection]) album sources have no relevant metadata for id3

View File

@ -15,11 +15,12 @@ from youtube_dl.extractor.youtube import YoutubeIE
from ...utils.exception.config import SettingValueError from ...utils.exception.config import SettingValueError
from ...utils.config import main_settings, youtube_settings, logging_settings from ...utils.config import main_settings, youtube_settings, logging_settings
from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING
from ...utils.string_processing import clean_song_title
from ...utils import get_current_millis from ...utils import get_current_millis
from ...utils import dump_to_file from ...utils import dump_to_file
from ...objects import Source, DatabaseObject from ...objects import Source, DatabaseObject, ID3Timestamp
from ..abstract import Page from ..abstract import Page
from ...objects import ( from ...objects import (
Artist, Artist,
@ -199,6 +200,8 @@ class YoutubeMusic(SuperYouTube):
self.ydl = MusicKrakenYoutubeDL(self, ydl_opts) self.ydl = MusicKrakenYoutubeDL(self, ydl_opts)
self.yt_ie = MusicKrakenYoutubeIE(downloader=self.ydl, main_instance=self) self.yt_ie = MusicKrakenYoutubeIE(downloader=self.ydl, main_instance=self)
self.download_values_by_url: dict = {}
def _fetch_from_main_page(self): def _fetch_from_main_page(self):
""" """
===API=KEY=== ===API=KEY===
@ -480,12 +483,39 @@ class YoutubeMusic(SuperYouTube):
def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song:
song = Song() ydl_res: dict = self.ydl.extract_info(url=source.url, download=False)
return song self.fetch_media_url(source=source, ydl_res=ydl_res)
artist_name = ydl_res.get("artist", ydl_res.get("uploader", "")).rstrip(" - Topic")
album_list = []
if "album" in ydl_res:
album_list.append(Album(
title=ydl_res.get("album"),
date=ID3Timestamp.strptime(ydl_res.get("upload_date"), "%Y%m%d"),
))
return Song(
title=ydl_res.get("track", clean_song_title(ydl_res.get("title"), artist_name=artist_name)),
note=ydl_res.get("descriptions"),
album_list=album_list,
length=int(ydl_res.get("duration", 0)) * 1000,
main_artist_list=[Artist(
name=artist_name,
source_list=[Source(
SourcePages.YOUTUBE_MUSIC,
f"https://music.youtube.com/channel/{ydl_res.get('channel_id', ydl_res.get('uploader_id', ''))}"
)]
)],
source_list=[Source(
SourcePages.YOUTUBE_MUSIC,
f"https://music.youtube.com/watch?v={ydl_res.get('id')}"
), source],
)
def fetch_media_url(self, source: Source) -> dict: def fetch_media_url(self, source: Source, ydl_res: dict = None) -> dict:
def _get_best_format(format_list: List[Dict]) -> dict: def _get_best_format(format_list: List[Dict]) -> dict:
def _calc_score(_f: dict): def _calc_score(_f: dict):
s = 0 s = 0
@ -506,17 +536,21 @@ class YoutubeMusic(SuperYouTube):
return best_format return best_format
ydl_res = self.ydl.extract_info(url=source.url, download=False) if source.url in self.download_values_by_url:
return self.download_values_by_url[source.url]
if ydl_res is None:
ydl_res = self.ydl.extract_info(url=source.url, download=False)
_best_format = _get_best_format(ydl_res.get("formats", [{}])) _best_format = _get_best_format(ydl_res.get("formats", [{}]))
print(_best_format) self.download_values_by_url[source.url] = {
return {
"url": _best_format.get("url"), "url": _best_format.get("url"),
"chunk_size": _best_format.get("downloader_options", {}).get("http_chunk_size", main_settings["chunk_size"]), "chunk_size": _best_format.get("downloader_options", {}).get("http_chunk_size", main_settings["chunk_size"]),
"headers": _best_format.get("http_headers", {}), "headers": _best_format.get("http_headers", {}),
} }
return self.download_values_by_url[source.url]
def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult:
media = self.fetch_media_url(source) media = self.fetch_media_url(source)

View File

@ -3,7 +3,7 @@ from pathlib import Path
import json import json
import logging import logging
from .shared import DEBUG, DEBUG_LOGGING, DEBUG_PAGES from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP
from .config import config, read_config, write_config from .config import config, read_config, write_config
from .enums.colors import BColors from .enums.colors import BColors
from .path_manager import LOCATIONS from .path_manager import LOCATIONS
@ -29,14 +29,14 @@ def user_input(msg: str, color: BColors = BColors.ENDC):
def dump_to_file(file_name: str, payload: str, is_json: bool = False, exit_after_dump: bool = False): def dump_to_file(file_name: str, payload: str, is_json: bool = False, exit_after_dump: bool = False):
if not DEBUG_PAGES: if not DEBUG_DUMP:
return return
path = Path(LOCATIONS.TEMP_DIRECTORY, file_name) path = Path(LOCATIONS.TEMP_DIRECTORY, file_name)
logging.warning(f"dumping {file_name} to: \"{path}\"") logging.warning(f"dumping {file_name} to: \"{path}\"")
if is_json: if is_json and isinstance(payload, str):
payload = json.dumps(json.loads(payload), indent=4) payload = json.loads(payload)
if isinstance(payload, dict): if isinstance(payload, dict):
payload = json.dumps(payload, indent=4) payload = json.dumps(payload, indent=4)

View File

@ -3,10 +3,11 @@ import random
from .path_manager import LOCATIONS from .path_manager import LOCATIONS
from .config import main_settings from .config import main_settings
DEBUG = False DEBUG = True
DEBUG_LOGGING = DEBUG and True DEBUG_LOGGING = DEBUG and False
DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False
DEBUG_PAGES = DEBUG and False DEBUG_PAGES = DEBUG and False
DEBUG_DUMP = DEBUG and True
if DEBUG: if DEBUG:
print("DEBUG ACTIVE") print("DEBUG ACTIVE")