From 0e2de498317d92fce8cff4fbddc670365a348a99 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 10 Apr 2024 09:28:28 +0200 Subject: [PATCH] feat: caching the ytdl responses --- music_kraken/objects/song.py | 3 +- .../pages/youtube_music/youtube_music.py | 50 ++++++++++++++++--- music_kraken/utils/__init__.py | 8 +-- music_kraken/utils/shared.py | 5 +- 4 files changed, 51 insertions(+), 15 deletions(-) diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index eea36dc..ef38191 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -127,7 +127,8 @@ class Song(Base): id3Mapping.ISRC: [self.isrc], id3Mapping.LENGTH: [self.length], id3Mapping.GENRE: [self.genre], - id3Mapping.TRACKNUMBER: [self.tracksort_str] + id3Mapping.TRACKNUMBER: [self.tracksort_str], + id3Mapping.COMMENT: [self.note.markdown], }) # metadata.merge_many([s.get_song_metadata() for s in self.source_collection]) album sources have no relevant metadata for id3 diff --git a/music_kraken/pages/youtube_music/youtube_music.py b/music_kraken/pages/youtube_music/youtube_music.py index a946a59..31dae1e 100644 --- a/music_kraken/pages/youtube_music/youtube_music.py +++ b/music_kraken/pages/youtube_music/youtube_music.py @@ -15,11 +15,12 @@ from youtube_dl.extractor.youtube import YoutubeIE from ...utils.exception.config import SettingValueError from ...utils.config import main_settings, youtube_settings, logging_settings from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING +from ...utils.string_processing import clean_song_title from ...utils import get_current_millis from ...utils import dump_to_file -from ...objects import Source, DatabaseObject +from ...objects import Source, DatabaseObject, ID3Timestamp from ..abstract import Page from ...objects import ( Artist, @@ -199,6 +200,8 @@ class YoutubeMusic(SuperYouTube): self.ydl = MusicKrakenYoutubeDL(self, ydl_opts) self.yt_ie = MusicKrakenYoutubeIE(downloader=self.ydl, main_instance=self) + self.download_values_by_url: dict = {} + def _fetch_from_main_page(self): """ ===API=KEY=== @@ -480,12 +483,39 @@ class YoutubeMusic(SuperYouTube): def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: - song = Song() + ydl_res: dict = self.ydl.extract_info(url=source.url, download=False) - return song + self.fetch_media_url(source=source, ydl_res=ydl_res) + + artist_name = ydl_res.get("artist", ydl_res.get("uploader", "")).rstrip(" - Topic") + + album_list = [] + if "album" in ydl_res: + album_list.append(Album( + title=ydl_res.get("album"), + date=ID3Timestamp.strptime(ydl_res.get("upload_date"), "%Y%m%d"), + )) + + return Song( + title=ydl_res.get("track", clean_song_title(ydl_res.get("title"), artist_name=artist_name)), + note=ydl_res.get("descriptions"), + album_list=album_list, + length=int(ydl_res.get("duration", 0)) * 1000, + main_artist_list=[Artist( + name=artist_name, + source_list=[Source( + SourcePages.YOUTUBE_MUSIC, + f"https://music.youtube.com/channel/{ydl_res.get('channel_id', ydl_res.get('uploader_id', ''))}" + )] + )], + source_list=[Source( + SourcePages.YOUTUBE_MUSIC, + f"https://music.youtube.com/watch?v={ydl_res.get('id')}" + ), source], + ) - def fetch_media_url(self, source: Source) -> dict: + def fetch_media_url(self, source: Source, ydl_res: dict = None) -> dict: def _get_best_format(format_list: List[Dict]) -> dict: def _calc_score(_f: dict): s = 0 @@ -506,17 +536,21 @@ class YoutubeMusic(SuperYouTube): return best_format - ydl_res = self.ydl.extract_info(url=source.url, download=False) + if source.url in self.download_values_by_url: + return self.download_values_by_url[source.url] + + if ydl_res is None: + ydl_res = self.ydl.extract_info(url=source.url, download=False) _best_format = _get_best_format(ydl_res.get("formats", [{}])) - print(_best_format) - - return { + self.download_values_by_url[source.url] = { "url": _best_format.get("url"), "chunk_size": _best_format.get("downloader_options", {}).get("http_chunk_size", main_settings["chunk_size"]), "headers": _best_format.get("http_headers", {}), } + return self.download_values_by_url[source.url] + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: media = self.fetch_media_url(source) diff --git a/music_kraken/utils/__init__.py b/music_kraken/utils/__init__.py index 9d2eb5b..ed707b8 100644 --- a/music_kraken/utils/__init__.py +++ b/music_kraken/utils/__init__.py @@ -3,7 +3,7 @@ from pathlib import Path import json import logging -from .shared import DEBUG, DEBUG_LOGGING, DEBUG_PAGES +from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP from .config import config, read_config, write_config from .enums.colors import BColors from .path_manager import LOCATIONS @@ -29,14 +29,14 @@ def user_input(msg: str, color: BColors = BColors.ENDC): def dump_to_file(file_name: str, payload: str, is_json: bool = False, exit_after_dump: bool = False): - if not DEBUG_PAGES: + if not DEBUG_DUMP: return path = Path(LOCATIONS.TEMP_DIRECTORY, file_name) logging.warning(f"dumping {file_name} to: \"{path}\"") - if is_json: - payload = json.dumps(json.loads(payload), indent=4) + if is_json and isinstance(payload, str): + payload = json.loads(payload) if isinstance(payload, dict): payload = json.dumps(payload, indent=4) diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index d7be692..413eec9 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -3,10 +3,11 @@ import random from .path_manager import LOCATIONS from .config import main_settings -DEBUG = False -DEBUG_LOGGING = DEBUG and True +DEBUG = True +DEBUG_LOGGING = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_PAGES = DEBUG and False +DEBUG_DUMP = DEBUG and True if DEBUG: print("DEBUG ACTIVE")