Merge pull request 'fix/musify_artist_spam' (#27) from fix/musify_artist_spam into experimental
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

Reviewed-on: #27
This commit is contained in:
Hazel 2024-05-08 10:31:23 +00:00
commit e3d7ed8837
8 changed files with 256 additions and 263 deletions

View File

@ -7,7 +7,8 @@ logging.getLogger().setLevel(logging.DEBUG)
if __name__ == "__main__": if __name__ == "__main__":
commands = [ commands = [
"s: #a Crystal F", "s: #a Crystal F",
"dm: 10, 20" "10",
"2",
] ]

View File

@ -304,10 +304,8 @@ class Downloader:
def goto(self, data_object: DatabaseObject): def goto(self, data_object: DatabaseObject):
page: Type[Page] page: Type[Page]
self.pages.fetch_details(data_object) self.pages.fetch_details(data_object, stop_at_level=1)
print(data_object)
print(data_object.options)
self.set_current_options(GoToResults(data_object.options, max_items_per_page=self.max_displayed_options)) self.set_current_options(GoToResults(data_object.options, max_items_per_page=self.max_displayed_options))
self.print_current_options() self.print_current_options()
@ -380,13 +378,13 @@ class Downloader:
continue continue
i = 0 i = 0
if possible_index.isdigit(): try:
i = int(possible_index) i = int(possible_index)
else: except ValueError:
raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not a number.") raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not a number.")
if i < 0 and i >= len(self.current_results): if i < 0 or i >= len(self.current_results):
raise MKInvalidInputException(message=f"The index \"{i}\" is not within the bounds of 0-{len(self.current_results)}.") raise MKInvalidInputException(message=f"The index \"{i}\" is not within the bounds of 0-{len(self.current_results) - 1}.")
indices.append(i) indices.append(i)

View File

@ -28,6 +28,9 @@ class Results:
self._by_index = dict() self._by_index = dict()
self._page_by_index = dict() self._page_by_index = dict()
def __len__(self) -> int:
return max(self._by_index.keys())
def __getitem__(self, index: int): def __getitem__(self, index: int):
return self._by_index[index] return self._by_index[index]

View File

@ -2,6 +2,8 @@ from __future__ import annotations
from collections import defaultdict from collections import defaultdict
from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union, Any, Set from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union, Any, Set
import copy
from .parents import OuterProxy from .parents import OuterProxy
from ..utils import object_trace from ..utils import object_trace
from ..utils import output, BColors from ..utils import output, BColors
@ -47,8 +49,15 @@ class Collection(Generic[T]):
self.extend(data) self.extend(data)
def __hash__(self) -> int:
return id(self)
@property
def collection_names(self) -> List[str]:
return list(set(self._collection_for.values()))
def __repr__(self) -> str: def __repr__(self) -> str:
return f"Collection({' | '.join(self._collection_for.values())} {id(self)})" return f"Collection({' | '.join(self.collection_names)} {id(self)})"
def _map_element(self, __object: T, no_unmap: bool = False, **kwargs): def _map_element(self, __object: T, no_unmap: bool = False, **kwargs):
if not no_unmap: if not no_unmap:
@ -104,8 +113,9 @@ class Collection(Generic[T]):
""" """
self._data.append(other) self._data.append(other)
other._inner._is_in_collection.add(self)
# all of the existing hooks to get the defined datastructure # all of the existing hooks to get the defined datastructures
for collection_attribute, generator in self.extend_object_to_attribute.items(): for collection_attribute, generator in self.extend_object_to_attribute.items():
other.__getattribute__(collection_attribute).extend(generator, **kwargs) other.__getattribute__(collection_attribute).extend(generator, **kwargs)
@ -148,30 +158,28 @@ class Collection(Generic[T]):
object_trace(f"Appending {other.option_string} to {self}") object_trace(f"Appending {other.option_string} to {self}")
for c in self.pull_from:
r = c._find_object(other)
if r is not None:
output("found pull from", r, other, self, color=BColors.RED, sep="\t")
other.merge(r, **kwargs)
c.remove(r, existing=r, **kwargs)
break
existing_object = self._find_object(other)
# switching collection in the case of push to # switching collection in the case of push to
for c in self.push_to: for c in self.push_to:
r = c._find_object(other) r = c._find_object(other)
if r is not None: if r is not None:
output("found push to", r, other, self, color=BColors.RED, sep="\t") # output("found push to", r, other, c, self, color=BColors.RED, sep="\t")
return c.append(other, **kwargs) return c.append(other, **kwargs)
if existing_object is None: for c in self.pull_from:
r = c._find_object(other)
if r is not None:
# output("found pull from", r, other, c, self, color=BColors.RED, sep="\t")
c.remove(r, existing=r, **kwargs)
existing = self._find_object(other)
if existing is None:
self._append_new_object(other, **kwargs) self._append_new_object(other, **kwargs)
else: else:
existing_object.merge(other, **kwargs) existing.merge(other, **kwargs)
def remove(self, *other_list: List[T], silent: bool = False, existing: Optional[T] = None, **kwargs): def remove(self, *other_list: List[T], silent: bool = False, existing: Optional[T] = None, remove_from_other_collection=True, **kwargs):
other: T
for other in other_list: for other in other_list:
existing: Optional[T] = existing or self._indexed_values["id"].get(other.id, None) existing: Optional[T] = existing or self._indexed_values["id"].get(other.id, None)
if existing is None: if existing is None:
@ -179,14 +187,11 @@ class Collection(Generic[T]):
raise ValueError(f"Object {other} not found in {self}") raise ValueError(f"Object {other} not found in {self}")
return other return other
""" if remove_from_other_collection:
for collection_attribute, generator in self.extend_object_to_attribute.items(): for c in copy.copy(other._inner._is_in_collection):
other.__getattribute__(collection_attribute).remove(*generator, silent=silent, **kwargs) c.remove(other, silent=True, remove_from_other_collection=False, **kwargs)
other._inner._is_in_collection = set()
for attribute, new_object in self.append_object_to_attribute.items(): else:
other.__getattribute__(attribute).remove(new_object, silent=silent, **kwargs)
"""
self._data.remove(existing) self._data.remove(existing)
self._unmap_element(existing) self._unmap_element(existing)

View File

@ -29,12 +29,15 @@ class InnerData:
""" """
_refers_to_instances: set = None _refers_to_instances: set = None
_is_in_collection: set = None
""" """
Attribute versions keep track, of if the attribute has been changed. Attribute versions keep track, of if the attribute has been changed.
""" """
def __init__(self, object_type, **kwargs): def __init__(self, object_type, **kwargs):
self._refers_to_instances = set() self._refers_to_instances = set()
self._is_in_collection = set()
self._fetched_from: dict = {} self._fetched_from: dict = {}
# initialize the default values # initialize the default values
@ -58,6 +61,7 @@ class InnerData:
""" """
self._fetched_from.update(__other._fetched_from) self._fetched_from.update(__other._fetched_from)
self._is_in_collection.update(__other._is_in_collection)
for key, value in __other.__dict__.copy().items(): for key, value in __other.__dict__.copy().items():
if key.startswith("_"): if key.startswith("_"):

View File

@ -3,6 +3,7 @@ from __future__ import annotations
import random import random
from collections import defaultdict from collections import defaultdict
from typing import List, Optional, Dict, Tuple, Type, Union from typing import List, Optional, Dict, Tuple, Type, Union
import copy
import pycountry import pycountry
@ -118,13 +119,27 @@ class Song(Base):
"tracksort": lambda: 0, "tracksort": lambda: 0,
} }
def __init__(self, title: str = "", unified_title: str = None, isrc: str = None, length: int = None, def __init__(
genre: str = None, note: FormattedText = None, source_list: List[Source] = None, self,
target_list: List[Target] = None, lyrics_list: List[Lyrics] = None, title: str = None,
main_artist_list: List[Artist] = None, feature_artist_list: List[Artist] = None, isrc: str = None,
album_list: List[Album] = None, tracksort: int = 0, artwork: Optional[Artwork] = None, **kwargs) -> None: length: int = None,
genre: str = None,
note: FormattedText = None,
source_list: List[Source] = None,
target_list: List[Target] = None,
lyrics_list: List[Lyrics] = None,
main_artist_list: List[Artist] = None,
feature_artist_list: List[Artist] = None,
album_list: List[Album] = None,
tracksort: int = 0,
artwork: Optional[Artwork] = None,
**kwargs
) -> None:
real_kwargs = copy.copy(locals())
real_kwargs.update(real_kwargs.pop("kwargs", {}))
Base.__init__(**locals()) Base.__init__(**real_kwargs)
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_artist_collection", "feature_artist_collection", "album_collection") UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_artist_collection", "feature_artist_collection", "album_collection")
TITEL = "title" TITEL = "title"
@ -210,14 +225,6 @@ class Song(Base):
r += get_collection_string(self.feature_artist_collection, " feat. {}") r += get_collection_string(self.feature_artist_collection, " feat. {}")
return r return r
@property
def options(self) -> List[P]:
options = self.main_artist_collection.shallow_list
options.extend(self.feature_artist_collection)
options.extend(self.album_collection)
options.append(self)
return options
@property @property
def tracksort_str(self) -> str: def tracksort_str(self) -> str:
""" """
@ -273,15 +280,27 @@ class Album(Base):
TITEL = "title" TITEL = "title"
# This is automatically generated # This is automatically generated
def __init__(self, title: str = None, unified_title: str = None, album_status: AlbumStatus = None, def __init__(
album_type: AlbumType = None, language: Language = None, date: ID3Timestamp = None, self,
barcode: str = None, albumsort: int = None, notes: FormattedText = None, title: str = None,
source_list: List[Source] = None, artist_list: List[Artist] = None, song_list: List[Song] = None, unified_title: str = None,
label_list: List[Label] = None, **kwargs) -> None: album_status: AlbumStatus = None,
super().__init__(title=title, unified_title=unified_title, album_status=album_status, album_type=album_type, album_type: AlbumType = None,
language=language, date=date, barcode=barcode, albumsort=albumsort, notes=notes, language: Language = None,
source_list=source_list, artist_list=artist_list, song_list=song_list, label_list=label_list, date: ID3Timestamp = None,
**kwargs) barcode: str = None,
albumsort: int = None,
notes: FormattedText = None,
source_list: List[Source] = None,
artist_list: List[Artist] = None,
song_list: List[Song] = None,
label_list: List[Label] = None,
**kwargs
) -> None:
real_kwargs = copy.copy(locals())
real_kwargs.update(real_kwargs.pop("kwargs", {}))
Base.__init__(**real_kwargs)
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("song_collection",) DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("song_collection",)
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection", "artist_collection") UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection", "artist_collection")
@ -413,14 +432,8 @@ class Album(Base):
return self.album_type.value return self.album_type.value
"""
All objects dependent on Artist
"""
class Artist(Base): class Artist(Base):
name: str name: str
unified_name: str
country: Country country: Country
formed_in: ID3Timestamp formed_in: ID3Timestamp
notes: FormattedText notes: FormattedText
@ -437,8 +450,7 @@ class Artist(Base):
label_collection: Collection[Label] label_collection: Collection[Label]
_default_factories = { _default_factories = {
"name": str, "name": lambda: None,
"unified_name": lambda: None,
"country": lambda: None, "country": lambda: None,
"unformatted_location": lambda: None, "unformatted_location": lambda: None,
@ -457,17 +469,28 @@ class Artist(Base):
TITEL = "name" TITEL = "name"
# This is automatically generated # This is automatically generated
def __init__(self, name: str = "", unified_name: str = None, country: Country = None, def __init__(
formed_in: ID3Timestamp = None, notes: FormattedText = None, lyrical_themes: List[str] = None, self,
general_genre: str = None, unformatted_location: str = None, source_list: List[Source] = None, name: str = None,
contact_list: List[Contact] = None, feature_song_list: List[Song] = None, unified_name: str = None,
main_album_list: List[Album] = None, label_list: List[Label] = None, **kwargs) -> None: country: Country = None,
formed_in: ID3Timestamp = None,
notes: FormattedText = None,
lyrical_themes: List[str] = None,
general_genre: str = None,
unformatted_location: str = None,
source_list: List[Source] = None,
contact_list: List[Contact] = None,
feature_song_list: List[Song] = None,
main_album_list: List[Album] = None,
label_list: List[Label] = None,
**kwargs
) -> None:
real_kwargs = copy.copy(locals())
real_kwargs.update(real_kwargs.pop("kwargs", {}))
Base.__init__(**real_kwargs)
super().__init__(name=name, unified_name=unified_name, country=country, formed_in=formed_in, notes=notes,
lyrical_themes=lyrical_themes, general_genre=general_genre,
unformatted_location=unformatted_location, source_list=source_list, contact_list=contact_list,
feature_song_list=feature_song_list, main_album_list=main_album_list, label_list=label_list,
**kwargs)
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_album_collection", "feature_song_collection") DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_album_collection", "feature_song_collection")
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection",) UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection",)
@ -593,11 +616,6 @@ class Artist(Base):
return r return r
"""
Label
"""
class Label(Base): class Label(Base):
COLLECTION_STRING_ATTRIBUTES = ("album_collection", "current_artist_collection") COLLECTION_STRING_ATTRIBUTES = ("album_collection", "current_artist_collection")
@ -625,12 +643,21 @@ class Label(Base):
TITEL = "name" TITEL = "name"
def __init__(self, name: str = None, unified_name: str = None, notes: FormattedText = None, def __init__(
source_list: List[Source] = None, contact_list: List[Contact] = None, self,
album_list: List[Album] = None, current_artist_list: List[Artist] = None, **kwargs) -> None: name: str = None,
super().__init__(name=name, unified_name=unified_name, notes=notes, source_list=source_list, unified_name: str = None,
contact_list=contact_list, album_list=album_list, current_artist_list=current_artist_list, notes: FormattedText = None,
**kwargs) source_list: List[Source] = None,
contact_list: List[Contact] = None,
album_list: List[Album] = None,
current_artist_list: List[Artist] = None,
**kwargs
) -> None:
real_kwargs = copy.copy(locals())
real_kwargs.update(real_kwargs.pop("kwargs", {}))
Base.__init__(**real_kwargs)
def __init_collections__(self): def __init_collections__(self):
self.album_collection.append_object_to_attribute = { self.album_collection.append_object_to_attribute = {

View File

@ -254,7 +254,7 @@ class Page:
} }
if obj_type in fetch_map: if obj_type in fetch_map:
music_object = fetch_map[obj_type](source, stop_at_level) music_object = fetch_map[obj_type](source, stop_at_level=stop_at_level)
else: else:
self.LOGGER.warning(f"Can't fetch details of type: {obj_type}") self.LOGGER.warning(f"Can't fetch details of type: {obj_type}")
return None return None

View File

@ -1,7 +1,7 @@
from collections import defaultdict from collections import defaultdict
from dataclasses import dataclass from dataclasses import dataclass
from enum import Enum from enum import Enum
from typing import List, Optional, Type, Union, Generator from typing import List, Optional, Type, Union, Generator, Dict, Any
from urllib.parse import urlparse from urllib.parse import urlparse
import pycountry import pycountry
@ -24,7 +24,7 @@ from ..objects import (
Lyrics, Lyrics,
Artwork Artwork
) )
from ..utils.config import logging_settings from ..utils.config import logging_settings, main_settings
from ..utils import string_processing, shared from ..utils import string_processing, shared
from ..utils.string_processing import clean_song_title from ..utils.string_processing import clean_song_title
from ..utils.support_classes.query import Query from ..utils.support_classes.query import Query
@ -361,7 +361,7 @@ class Musify(Page):
return Song( return Song(
title=clean_song_title(song_title, artist_name=artist_list[0].name if len(artist_list) > 0 else None), title=clean_song_title(song_title, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
main_artist_list=artist_list, feature_artist_list=artist_list,
source_list=source_list source_list=source_list
) )
@ -510,7 +510,7 @@ class Musify(Page):
title=clean_song_title(track_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None), title=clean_song_title(track_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
source_list=source_list, source_list=source_list,
lyrics_list=lyrics_list, lyrics_list=lyrics_list,
main_artist_list=artist_list, feature_artist_list=artist_list,
album_list=album_list, album_list=album_list,
artwork=artwork, artwork=artwork,
) )
@ -652,10 +652,101 @@ class Musify(Page):
return Song( return Song(
title=clean_song_title(song_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None), title=clean_song_title(song_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
tracksort=tracksort, tracksort=tracksort,
main_artist_list=artist_list, feature_artist_list=artist_list,
source_list=source_list source_list=source_list
) )
def _parse_album(self, soup: BeautifulSoup) -> Album:
name: str = None
source_list: List[Source] = []
artist_list: List[Artist] = []
date: ID3Timestamp = None
"""
if breadcrumb list has 4 elements, then
the -2 is the artist link,
the -1 is the album
"""
# breadcrumb
breadcrumb_soup: BeautifulSoup = soup.find("ol", {"class", "breadcrumb"})
breadcrumb_elements: List[BeautifulSoup] = breadcrumb_soup.find_all("li", {"class": "breadcrumb-item"})
if len(breadcrumb_elements) == 4:
# album
album_crumb: BeautifulSoup = breadcrumb_elements[-1]
name = album_crumb.text.strip()
# artist
artist_crumb: BeautifulSoup = breadcrumb_elements[-2]
anchor: BeautifulSoup = artist_crumb.find("a")
if anchor is not None:
href = anchor.get("href")
artist_source_list: List[Source] = []
if href is not None:
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href.strip()))
span: BeautifulSoup = anchor.find("span")
if span is not None:
artist_list.append(Artist(
name=span.get_text(strip=True),
source_list=artist_source_list
))
else:
self.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case")
# meta
meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"})
if meta_url is not None:
url = meta_url.get("content")
if url is not None:
source_list.append(Source(self.SOURCE_TYPE, self.HOST + url))
meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"})
if meta_name is not None:
_name = meta_name.get("content")
if _name is not None:
name = _name
# album info
album_info_ul: BeautifulSoup = soup.find("ul", {"class": "album-info"})
if album_info_ul is not None:
artist_anchor: BeautifulSoup
for artist_anchor in album_info_ul.find_all("a", {"itemprop": "byArtist"}):
# line 98
artist_source_list: List[Source] = []
artist_url_meta = artist_anchor.find("meta", {"itemprop": "url"})
if artist_url_meta is not None:
artist_href = artist_url_meta.get("content")
if artist_href is not None:
artist_source_list.append(Source(self.SOURCE_TYPE, url=self.HOST + artist_href))
artist_meta_name = artist_anchor.find("meta", {"itemprop": "name"})
if artist_meta_name is not None:
artist_name = artist_meta_name.get("content")
if artist_name is not None:
artist_list.append(Artist(
name=artist_name,
source_list=artist_source_list
))
time_soup: BeautifulSoup = album_info_ul.find("time", {"itemprop": "datePublished"})
if time_soup is not None:
raw_datetime = time_soup.get("datetime")
if raw_datetime is not None:
try:
date = ID3Timestamp.strptime(raw_datetime, "%Y-%m-%d")
except ValueError:
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
return Album(
title=name,
source_list=source_list,
artist_list=artist_list,
date=date
)
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
""" """
fetches album from source: fetches album from source:
@ -694,19 +785,14 @@ class Musify(Page):
return album return album
def _get_artist_attributes(self, url: MusifyUrl) -> Artist: def _fetch_initial_artist(self, url: MusifyUrl, source: Source, **kwargs) -> Artist:
""" """
fetches the main Artist attributes from this endpoint
https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent
it needs to parse html
:param url:
:return:
""" """
r = self.connection.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent", name="artist_attributes_" + url.name_with_id) r = self.connection.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent", name="artist_attributes_" + url.name_with_id)
if r is None: if r is None:
return Artist() return Artist(source_list=[source])
soup = self.get_soup_from_response(r) soup = self.get_soup_from_response(r)
@ -821,7 +907,7 @@ class Musify(Page):
notes=notes notes=notes
) )
def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None) -> Album: def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None, **kwargs) -> Album:
""" """
<div class="card release-thumbnail" data-type="2"> <div class="card release-thumbnail" data-type="2">
<a href="/release/ghost-bath-self-loather-2021-1554266"> <a href="/release/ghost-bath-self-loather-2021-1554266">
@ -845,33 +931,9 @@ class Musify(Page):
</div> </div>
""" """
_id: Optional[str] = None album_kwargs: Dict[str, Any] = {
name: str = None "source_list": [],
source_list: List[Source] = [] }
timestamp: Optional[ID3Timestamp] = None
album_status = None
def set_name(new_name: str):
nonlocal name
nonlocal artist_name
# example of just setting not working:
# https://musify.club/release/unjoy-eurythmie-psychonaut-4-tired-numb-still-alive-2012-324067
if new_name.count(" - ") != 1:
name = new_name
return
potential_artist_list, potential_name = new_name.split(" - ")
unified_artist_list = string_processing.unify(potential_artist_list)
if artist_name is not None:
if string_processing.unify(artist_name) not in unified_artist_list:
name = new_name
return
name = potential_name
return
name = new_name
album_status_id = album_card.get("data-type") album_status_id = album_card.get("data-type")
if album_status_id.isdigit(): if album_status_id.isdigit():
@ -882,9 +944,7 @@ class Musify(Page):
album_status = AlbumStatus.BOOTLEG album_status = AlbumStatus.BOOTLEG
def parse_release_anchor(_anchor: BeautifulSoup, text_is_name=False): def parse_release_anchor(_anchor: BeautifulSoup, text_is_name=False):
nonlocal _id nonlocal album_kwargs
nonlocal name
nonlocal source_list
if _anchor is None: if _anchor is None:
return return
@ -892,20 +952,13 @@ class Musify(Page):
href = _anchor.get("href") href = _anchor.get("href")
if href is not None: if href is not None:
# add url to sources # add url to sources
source_list.append(Source( album_kwargs["source_list"].append(Source(
self.SOURCE_TYPE, self.SOURCE_TYPE,
self.HOST + href self.HOST + href
)) ))
# split id from url if text_is_name:
split_href = href.split("-") album_kwargs["title"] = clean_song_title(_anchor.text, artist_name)
if len(split_href) > 1:
_id = split_href[-1]
if not text_is_name:
return
set_name(_anchor.text)
anchor_list = album_card.find_all("a", recursive=False) anchor_list = album_card.find_all("a", recursive=False)
if len(anchor_list) > 0: if len(anchor_list) > 0:
@ -916,7 +969,7 @@ class Musify(Page):
if thumbnail is not None: if thumbnail is not None:
alt = thumbnail.get("alt") alt = thumbnail.get("alt")
if alt is not None: if alt is not None:
set_name(alt) album_kwargs["title"] = clean_song_title(alt, artist_name)
image_url = thumbnail.get("src") image_url = thumbnail.get("src")
else: else:
@ -933,7 +986,7 @@ class Musify(Page):
13.11.2021 13.11.2021
</small> </small>
""" """
nonlocal timestamp nonlocal album_kwargs
italic_tagging_soup: BeautifulSoup = small_soup.find("i") italic_tagging_soup: BeautifulSoup = small_soup.find("i")
if italic_tagging_soup is None: if italic_tagging_soup is None:
@ -943,7 +996,7 @@ class Musify(Page):
return return
raw_time = small_soup.text.strip() raw_time = small_soup.text.strip()
timestamp = ID3Timestamp.strptime(raw_time, "%d.%m.%Y") album_kwargs["date"] = ID3Timestamp.strptime(raw_time, "%d.%m.%Y")
# parse small date # parse small date
card_footer_list = album_card.find_all("div", {"class": "card-footer"}) card_footer_list = album_card.find_all("div", {"class": "card-footer"})
@ -956,105 +1009,9 @@ class Musify(Page):
else: else:
self.LOGGER.debug("there is not even 1 footer in the album card") self.LOGGER.debug("there is not even 1 footer in the album card")
return Album( return Album(**album_kwargs)
title=name,
source_list=source_list,
date=timestamp,
album_type=album_type,
album_status=album_status
)
def _parse_album(self, soup: BeautifulSoup) -> Album: def _fetch_artist_discography(self, artist: Artist, url: MusifyUrl, artist_name: str = None, **kwargs):
name: str = None
source_list: List[Source] = []
artist_list: List[Artist] = []
date: ID3Timestamp = None
"""
if breadcrumb list has 4 elements, then
the -2 is the artist link,
the -1 is the album
"""
# breadcrumb
breadcrumb_soup: BeautifulSoup = soup.find("ol", {"class", "breadcrumb"})
breadcrumb_elements: List[BeautifulSoup] = breadcrumb_soup.find_all("li", {"class": "breadcrumb-item"})
if len(breadcrumb_elements) == 4:
# album
album_crumb: BeautifulSoup = breadcrumb_elements[-1]
name = album_crumb.text.strip()
# artist
artist_crumb: BeautifulSoup = breadcrumb_elements[-2]
anchor: BeautifulSoup = artist_crumb.find("a")
if anchor is not None:
href = anchor.get("href")
artist_source_list: List[Source] = []
if href is not None:
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href.strip()))
span: BeautifulSoup = anchor.find("span")
if span is not None:
artist_list.append(Artist(
name=span.get_text(strip=True),
source_list=artist_source_list
))
else:
self.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case")
# meta
meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"})
if meta_url is not None:
url = meta_url.get("content")
if url is not None:
source_list.append(Source(self.SOURCE_TYPE, self.HOST + url))
meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"})
if meta_name is not None:
_name = meta_name.get("content")
if _name is not None:
name = _name
# album info
album_info_ul: BeautifulSoup = soup.find("ul", {"class": "album-info"})
if album_info_ul is not None:
artist_anchor: BeautifulSoup
for artist_anchor in album_info_ul.find_all("a", {"itemprop": "byArtist"}):
# line 98
artist_source_list: List[Source] = []
artist_url_meta = artist_anchor.find("meta", {"itemprop": "url"})
if artist_url_meta is not None:
artist_href = artist_url_meta.get("content")
if artist_href is not None:
artist_source_list.append(Source(self.SOURCE_TYPE, url=self.HOST + artist_href))
artist_meta_name = artist_anchor.find("meta", {"itemprop": "name"})
if artist_meta_name is not None:
artist_name = artist_meta_name.get("content")
if artist_name is not None:
artist_list.append(Artist(
name=artist_name,
source_list=artist_source_list
))
time_soup: BeautifulSoup = album_info_ul.find("time", {"itemprop": "datePublished"})
if time_soup is not None:
raw_datetime = time_soup.get("datetime")
if raw_datetime is not None:
try:
date = ID3Timestamp.strptime(raw_datetime, "%Y-%m-%d")
except ValueError:
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
return Album(
title=name,
source_list=source_list,
artist_list=artist_list,
date=date
)
def _get_discography(self, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> Generator[Album, None, None]:
""" """
POST https://musify.club/artist/filteralbums POST https://musify.club/artist/filteralbums
ArtistID: 280348 ArtistID: 280348
@ -1062,6 +1019,8 @@ class Musify(Page):
SortOrder.IsAscending: false SortOrder.IsAscending: false
X-Requested-With: XMLHttpRequest X-Requested-With: XMLHttpRequest
""" """
_download_all = kwargs.get("download_all", False)
_album_type_blacklist = kwargs.get("album_type_blacklist", main_settings["album_type_blacklist"])
endpoint = self.HOST + "/" + url.source_type.value + "/filteralbums" endpoint = self.HOST + "/" + url.source_type.value + "/filteralbums"
@ -1072,33 +1031,29 @@ class Musify(Page):
"X-Requested-With": "XMLHttpRequest" "X-Requested-With": "XMLHttpRequest"
}, name="discography_" + url.name_with_id) }, name="discography_" + url.name_with_id)
if r is None: if r is None:
return [] return
soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser")
soup: BeautifulSoup = self.get_soup_from_response(r)
for card_soup in soup.find_all("div", {"class": "card"}): for card_soup in soup.find_all("div", {"class": "card"}):
yield self._parse_album_card(card_soup, artist_name) album = self._parse_album_card(card_soup, artist_name, **kwargs)
if album.album_type in _album_type_blacklist:
continue
def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: artist.main_album_collection.append(album)
def fetch_artist(self, source: Source, **kwargs) -> Artist:
""" """
fetches artist from source TODO
[x] discography [x] discography
[x] attributes [x] attributes
[] picture gallery [] picture gallery
Args:
source (Source): the source to fetch
stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False.
Returns:
Artist: the artist fetched
""" """
url = parse_url(source.url) url = parse_url(source.url)
artist = self._get_artist_attributes(url) artist = self._fetch_initial_artist(url, source=source, **kwargs)
self._fetch_artist_discography(artist, url, artist.name, **kwargs)
artist.main_album_collection.extend(self._get_discography(url, artist.name))
return artist return artist