fixed crash on search

This commit is contained in:
Hellow 2023-04-03 19:59:31 +02:00
parent 9b9f3ec8f4
commit 98fb2c7bf9
3 changed files with 146 additions and 78 deletions

View File

@ -26,6 +26,7 @@ from ..utils.string_processing import fit_to_file_system
LOGGER = logging.getLogger("this shouldn't be used") LOGGER = logging.getLogger("this shouldn't be used")
@dataclass @dataclass
class DefaultTarget: class DefaultTarget:
genre: str = DEFAULT_VALUES["genre"] genre: str = DEFAULT_VALUES["genre"]
@ -46,8 +47,10 @@ class DefaultTarget:
def target(self) -> Target: def target(self) -> Target:
return Target( return Target(
relative_to_music_dir=True, relative_to_music_dir=True,
path=DOWNLOAD_PATH.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album, song=self.song), path=DOWNLOAD_PATH.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album,
file=DOWNLOAD_FILE.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album, song=self.song) song=self.song),
file=DOWNLOAD_FILE.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album,
song=self.song)
) )
@ -65,7 +68,8 @@ class Page:
SOURCE_TYPE: SourcePages SOURCE_TYPE: SourcePages
@classmethod @classmethod
def get_request(cls, url: str, stream: bool = False, accepted_response_codes: set = set((200,)), trie: int = 0) -> Optional[ def get_request(cls, url: str, stream: bool = False, accepted_response_codes: set = set((200,)), trie: int = 0) -> \
Optional[
requests.Response]: requests.Response]:
retry = False retry = False
try: try:
@ -207,7 +211,8 @@ class Page:
source: Source source: Source
for source in music_object.source_collection.get_sources_from_page(cls.SOURCE_TYPE): for source in music_object.source_collection.get_sources_from_page(cls.SOURCE_TYPE):
new_music_object.merge(cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) new_music_object.merge(
cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level))
had_sources = True had_sources = True
if not had_sources: if not had_sources:
@ -248,9 +253,10 @@ class Page:
music_object.compile(merge_into=True) music_object.compile(merge_into=True)
return music_object return music_object
@classmethod @classmethod
def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1) -> Union[Song, Album, Artist, Label]: def _fetch_object_from_source(cls, source: Source,
obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]],
stop_at_level: int = 1) -> Union[Song, Album, Artist, Label]:
if obj_type == Artist: if obj_type == Artist:
return cls._fetch_artist_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_artist_from_source(source=source, stop_at_level=stop_at_level)
@ -264,7 +270,8 @@ class Page:
return cls._fetch_label_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_label_from_source(source=source, stop_at_level=stop_at_level)
@classmethod @classmethod
def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song],
collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
if type(music_object) == Label: if type(music_object) == Label:
return cls._clean_label(label=music_object, collections=collections) return cls._clean_label(label=music_object, collections=collections)
if type(music_object) == Artist: if type(music_object) == Artist:
@ -275,7 +282,8 @@ class Page:
return cls._clean_song(song=music_object, collections=collections) return cls._clean_song(song=music_object, collections=collections)
@classmethod @classmethod
def _clean_collection(cls, collection: Collection, collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_collection(cls, collection: Collection,
collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
if collection.element_type not in collection_dict: if collection.element_type not in collection_dict:
return return
@ -287,35 +295,39 @@ class Page:
cls._clean_music_object(r.current_element, collection_dict) cls._clean_music_object(r.current_element, collection_dict)
@classmethod @classmethod
def _clean_label(cls, label: Label, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_label(cls, label: Label,
collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(label.current_artist_collection, collections) cls._clean_collection(label.current_artist_collection, collections)
cls._clean_collection(label.album_collection, collections) cls._clean_collection(label.album_collection, collections)
@classmethod @classmethod
def _clean_artist(cls, artist: Artist, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_artist(cls, artist: Artist,
collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(artist.main_album_collection, collections) cls._clean_collection(artist.main_album_collection, collections)
cls._clean_collection(artist.feature_song_collection, collections) cls._clean_collection(artist.feature_song_collection, collections)
cls._clean_collection(artist.label_collection, collections) cls._clean_collection(artist.label_collection, collections)
@classmethod @classmethod
def _clean_album(cls, album: Album, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_album(cls, album: Album,
collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(album.label_collection, collections) cls._clean_collection(album.label_collection, collections)
cls._clean_collection(album.song_collection, collections) cls._clean_collection(album.song_collection, collections)
cls._clean_collection(album.artist_collection, collections) cls._clean_collection(album.artist_collection, collections)
@classmethod @classmethod
def _clean_song(cls, song: Song, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_song(cls, song: Song,
collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(song.album_collection, collections) cls._clean_collection(song.album_collection, collections)
cls._clean_collection(song.feature_artist_collection, collections) cls._clean_collection(song.feature_artist_collection, collections)
cls._clean_collection(song.main_artist_collection, collections) cls._clean_collection(song.main_artist_collection, collections)
@classmethod @classmethod
def download( def download(
cls, cls,
music_object: Union[Song, Album, Artist, Label], music_object: Union[Song, Album, Artist, Label],
download_features: bool = True, download_features: bool = True,
default_target: DefaultTarget = None default_target: DefaultTarget = None
) -> bool: ) -> bool:
if default_target is None: if default_target is None:
default_target = DefaultTarget() default_target = DefaultTarget()
@ -331,7 +343,8 @@ class Page:
return False return False
@classmethod @classmethod
def download_label(cls, label: Label, download_features: bool = True, override_existing: bool = False, default_target: DefaultTarget = None): def download_label(cls, label: Label, download_features: bool = True, override_existing: bool = False,
default_target: DefaultTarget = None):
if default_target is None: if default_target is None:
default_target = DefaultTarget() default_target = DefaultTarget()
else: else:
@ -340,13 +353,15 @@ class Page:
cls.fetch_details(label) cls.fetch_details(label)
for artist in label.current_artist_collection: for artist in label.current_artist_collection:
cls.download_artist(artist, download_features=download_features, override_existing=override_existing, default_target=default_target) cls.download_artist(artist, download_features=download_features, override_existing=override_existing,
default_target=default_target)
for album in label.album_collection: for album in label.album_collection:
cls.download_album(album, override_existing=override_existing, default_target=default_target) cls.download_album(album, override_existing=override_existing, default_target=default_target)
@classmethod @classmethod
def download_artist(cls, artist: Artist, download_features: bool = True, override_existing: bool = False, default_target: DefaultTarget = None): def download_artist(cls, artist: Artist, download_features: bool = True, override_existing: bool = False,
default_target: DefaultTarget = None):
if default_target is None: if default_target is None:
default_target = DefaultTarget() default_target = DefaultTarget()
else: else:
@ -382,7 +397,8 @@ class Page:
cls.download_song(song, override_existing=override_existing, default_target=default_target) cls.download_song(song, override_existing=override_existing, default_target=default_target)
@classmethod @classmethod
def download_song(cls, song: Song, override_existing: bool = False, create_target_on_demand: bool = True, default_target: DefaultTarget = None): def download_song(cls, song: Song, override_existing: bool = False, create_target_on_demand: bool = True,
default_target: DefaultTarget = None):
if default_target is None: if default_target is None:
default_target = DefaultTarget() default_target = DefaultTarget()
else: else:
@ -446,7 +462,6 @@ class Page:
for target in song.target_collection: for target in song.target_collection:
temp_target.copy_content(target) temp_target.copy_content(target)
@classmethod @classmethod
def _fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song: def _fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song:
return Song() return Song()
@ -468,5 +483,5 @@ class Page:
return None return None
@classmethod @classmethod
def _download_song_to_targets(cls, source: Source) -> Target: def _download_song_to_targets(cls, source: Source, target: Target) -> Target:
return Target() return Target()

View File

@ -29,7 +29,7 @@ class Search(Download):
self._option_history: List[MultiPageOptions] = [] self._option_history: List[MultiPageOptions] = []
self._current_option: MultiPageOptions = self.next_options self._current_option: MultiPageOptions = self.next_options()
def __repr__(self): def __repr__(self):

View File

@ -847,6 +847,61 @@ class Musify(Page):
source_list=source_list source_list=source_list
) )
@classmethod
def _parse_album(cls, soup: BeautifulSoup) -> Album:
name: str = None
source_list: List[Source] = []
artist_list: List[Artist] = []
"""
if breadcrumb list has 4 elements, then
the -2 is the artist link,
the -1 is the album
"""
breadcrumb_soup: BeautifulSoup = soup.find("ol", {"class", "breadcrumb"})
breadcrumb_elements: List[BeautifulSoup] = breadcrumb_soup.find_all("li", {"class": "breadcrumb-item"})
if len(breadcrumb_elements) == 4:
# album
album_crumb: BeautifulSoup = breadcrumb_elements[-1]
name = album_crumb.text.strip()
# artist
artist_crumb: BeautifulSoup = breadcrumb_elements[-2]
anchor: BeautifulSoup = artist_crumb.find("a")
if anchor is not None:
href = anchor.get("href")
artist_source_list: List[Source] = []
if href is not None:
artist_source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href.strip()))
span: BeautifulSoup = anchor.find("span")
if span is not None:
artist_list.append(Artist(
name=span.get_text(strip=True),
source_list=artist_source_list
))
else:
cls.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case")
meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"})
if meta_url is not None:
url = meta_url.get("content")
if url is not None:
source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + url))
meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"})
if meta_name is not None:
_name = meta_name.get("content")
if _name is not None:
name = _name
return Album(
title=name,
source_list=source_list,
artist_list=artist_list
)
@classmethod @classmethod
def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album:
""" """
@ -862,17 +917,19 @@ class Musify(Page):
:param source: :param source:
:return: :return:
""" """
album = Album(title="Hi :)", source_list=[source])
url = cls.parse_url(source.url) url = cls.parse_url(source.url)
endpoint = cls.HOST + "/release/" + url.name_with_id endpoint = cls.HOST + "/release/" + url.name_with_id
r = cls.get_request(endpoint) r = cls.get_request(endpoint)
if r is None: if r is None:
return album return Album()
soup = BeautifulSoup(r.content, "html.parser") soup = BeautifulSoup(r.content, "html.parser")
album = cls._parse_album(soup)
print(album)
# <div class="card"><div class="card-body">...</div></div> # <div class="card"><div class="card-body">...</div></div>
cards_soup: BeautifulSoup = soup.find("div", {"class": "card-body"}) cards_soup: BeautifulSoup = soup.find("div", {"class": "card-body"})
if cards_soup is not None: if cards_soup is not None:
@ -890,9 +947,6 @@ class Musify(Page):
album.update_tracksort() album.update_tracksort()
print(album.song_collection)
print()
return album return album
@classmethod @classmethod
@ -908,17 +962,16 @@ class Musify(Page):
return None return None
@classmethod @classmethod
def _download_song_to_targets(cls, source: Source, target: Target) -> Path: def _download_song_to_targets(cls, source: Source, target: Target) -> bool:
""" """
https://musify.club/track/im-in-a-coffin-life-never-was-waste-of-skin-16360302 https://musify.club/track/im-in-a-coffin-life-never-was-waste-of-skin-16360302
https://musify.club/track/dl/16360302/im-in-a-coffin-life-never-was-waste-of-skin.mp3 https://musify.club/track/dl/16360302/im-in-a-coffin-life-never-was-waste-of-skin.mp3
""" """
url: MusifyUrl = cls.parse_url(source.url) url: MusifyUrl = cls.parse_url(source.url)
if url.source_type != MusifyTypes.SONG: if url.source_type != MusifyTypes.SONG:
return return False
endpoint = f"https://musify.club/track/dl/{url.musify_id}/{url.name_without_id}.mp3" endpoint = f"https://musify.club/track/dl/{url.musify_id}/{url.name_without_id}.mp3"
print(endpoint) print(endpoint)
target.stream_into(cls.get_request(endpoint, stream=True)) return target.stream_into(cls.get_request(endpoint, stream=True))