Merge pull request 'fix/musify_artist_spam' (#27) from fix/musify_artist_spam into experimental
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
Reviewed-on: #27
This commit is contained in:
commit
e3d7ed8837
@ -7,7 +7,8 @@ logging.getLogger().setLevel(logging.DEBUG)
|
||||
if __name__ == "__main__":
|
||||
commands = [
|
||||
"s: #a Crystal F",
|
||||
"dm: 10, 20"
|
||||
"10",
|
||||
"2",
|
||||
]
|
||||
|
||||
|
||||
|
@ -304,10 +304,8 @@ class Downloader:
|
||||
def goto(self, data_object: DatabaseObject):
|
||||
page: Type[Page]
|
||||
|
||||
self.pages.fetch_details(data_object)
|
||||
self.pages.fetch_details(data_object, stop_at_level=1)
|
||||
|
||||
print(data_object)
|
||||
print(data_object.options)
|
||||
self.set_current_options(GoToResults(data_object.options, max_items_per_page=self.max_displayed_options))
|
||||
|
||||
self.print_current_options()
|
||||
@ -380,13 +378,13 @@ class Downloader:
|
||||
continue
|
||||
|
||||
i = 0
|
||||
if possible_index.isdigit():
|
||||
try:
|
||||
i = int(possible_index)
|
||||
else:
|
||||
except ValueError:
|
||||
raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not a number.")
|
||||
|
||||
if i < 0 and i >= len(self.current_results):
|
||||
raise MKInvalidInputException(message=f"The index \"{i}\" is not within the bounds of 0-{len(self.current_results)}.")
|
||||
if i < 0 or i >= len(self.current_results):
|
||||
raise MKInvalidInputException(message=f"The index \"{i}\" is not within the bounds of 0-{len(self.current_results) - 1}.")
|
||||
|
||||
indices.append(i)
|
||||
|
||||
|
@ -28,6 +28,9 @@ class Results:
|
||||
self._by_index = dict()
|
||||
self._page_by_index = dict()
|
||||
|
||||
def __len__(self) -> int:
|
||||
return max(self._by_index.keys())
|
||||
|
||||
def __getitem__(self, index: int):
|
||||
return self._by_index[index]
|
||||
|
||||
|
@ -2,6 +2,8 @@ from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union, Any, Set
|
||||
import copy
|
||||
|
||||
from .parents import OuterProxy
|
||||
from ..utils import object_trace
|
||||
from ..utils import output, BColors
|
||||
@ -47,8 +49,15 @@ class Collection(Generic[T]):
|
||||
|
||||
self.extend(data)
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return id(self)
|
||||
|
||||
@property
|
||||
def collection_names(self) -> List[str]:
|
||||
return list(set(self._collection_for.values()))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"Collection({' | '.join(self._collection_for.values())} {id(self)})"
|
||||
return f"Collection({' | '.join(self.collection_names)} {id(self)})"
|
||||
|
||||
def _map_element(self, __object: T, no_unmap: bool = False, **kwargs):
|
||||
if not no_unmap:
|
||||
@ -104,8 +113,9 @@ class Collection(Generic[T]):
|
||||
"""
|
||||
|
||||
self._data.append(other)
|
||||
other._inner._is_in_collection.add(self)
|
||||
|
||||
# all of the existing hooks to get the defined datastructure
|
||||
# all of the existing hooks to get the defined datastructures
|
||||
for collection_attribute, generator in self.extend_object_to_attribute.items():
|
||||
other.__getattribute__(collection_attribute).extend(generator, **kwargs)
|
||||
|
||||
@ -148,30 +158,28 @@ class Collection(Generic[T]):
|
||||
|
||||
object_trace(f"Appending {other.option_string} to {self}")
|
||||
|
||||
|
||||
for c in self.pull_from:
|
||||
r = c._find_object(other)
|
||||
if r is not None:
|
||||
output("found pull from", r, other, self, color=BColors.RED, sep="\t")
|
||||
other.merge(r, **kwargs)
|
||||
c.remove(r, existing=r, **kwargs)
|
||||
break
|
||||
|
||||
existing_object = self._find_object(other)
|
||||
|
||||
# switching collection in the case of push to
|
||||
for c in self.push_to:
|
||||
r = c._find_object(other)
|
||||
if r is not None:
|
||||
output("found push to", r, other, self, color=BColors.RED, sep="\t")
|
||||
# output("found push to", r, other, c, self, color=BColors.RED, sep="\t")
|
||||
return c.append(other, **kwargs)
|
||||
|
||||
for c in self.pull_from:
|
||||
r = c._find_object(other)
|
||||
if r is not None:
|
||||
# output("found pull from", r, other, c, self, color=BColors.RED, sep="\t")
|
||||
c.remove(r, existing=r, **kwargs)
|
||||
|
||||
if existing_object is None:
|
||||
existing = self._find_object(other)
|
||||
|
||||
if existing is None:
|
||||
self._append_new_object(other, **kwargs)
|
||||
else:
|
||||
existing_object.merge(other, **kwargs)
|
||||
existing.merge(other, **kwargs)
|
||||
|
||||
def remove(self, *other_list: List[T], silent: bool = False, existing: Optional[T] = None, **kwargs):
|
||||
def remove(self, *other_list: List[T], silent: bool = False, existing: Optional[T] = None, remove_from_other_collection=True, **kwargs):
|
||||
other: T
|
||||
for other in other_list:
|
||||
existing: Optional[T] = existing or self._indexed_values["id"].get(other.id, None)
|
||||
if existing is None:
|
||||
@ -179,16 +187,13 @@ class Collection(Generic[T]):
|
||||
raise ValueError(f"Object {other} not found in {self}")
|
||||
return other
|
||||
|
||||
"""
|
||||
for collection_attribute, generator in self.extend_object_to_attribute.items():
|
||||
other.__getattribute__(collection_attribute).remove(*generator, silent=silent, **kwargs)
|
||||
|
||||
for attribute, new_object in self.append_object_to_attribute.items():
|
||||
other.__getattribute__(attribute).remove(new_object, silent=silent, **kwargs)
|
||||
"""
|
||||
|
||||
self._data.remove(existing)
|
||||
self._unmap_element(existing)
|
||||
if remove_from_other_collection:
|
||||
for c in copy.copy(other._inner._is_in_collection):
|
||||
c.remove(other, silent=True, remove_from_other_collection=False, **kwargs)
|
||||
other._inner._is_in_collection = set()
|
||||
else:
|
||||
self._data.remove(existing)
|
||||
self._unmap_element(existing)
|
||||
|
||||
def contains(self, __object: T) -> bool:
|
||||
return self._find_object(__object) is not None
|
||||
|
@ -29,12 +29,15 @@ class InnerData:
|
||||
"""
|
||||
|
||||
_refers_to_instances: set = None
|
||||
_is_in_collection: set = None
|
||||
"""
|
||||
Attribute versions keep track, of if the attribute has been changed.
|
||||
"""
|
||||
|
||||
def __init__(self, object_type, **kwargs):
|
||||
self._refers_to_instances = set()
|
||||
self._is_in_collection = set()
|
||||
|
||||
self._fetched_from: dict = {}
|
||||
|
||||
# initialize the default values
|
||||
@ -58,6 +61,7 @@ class InnerData:
|
||||
"""
|
||||
|
||||
self._fetched_from.update(__other._fetched_from)
|
||||
self._is_in_collection.update(__other._is_in_collection)
|
||||
|
||||
for key, value in __other.__dict__.copy().items():
|
||||
if key.startswith("_"):
|
||||
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from typing import List, Optional, Dict, Tuple, Type, Union
|
||||
import copy
|
||||
|
||||
import pycountry
|
||||
|
||||
@ -118,13 +119,27 @@ class Song(Base):
|
||||
"tracksort": lambda: 0,
|
||||
}
|
||||
|
||||
def __init__(self, title: str = "", unified_title: str = None, isrc: str = None, length: int = None,
|
||||
genre: str = None, note: FormattedText = None, source_list: List[Source] = None,
|
||||
target_list: List[Target] = None, lyrics_list: List[Lyrics] = None,
|
||||
main_artist_list: List[Artist] = None, feature_artist_list: List[Artist] = None,
|
||||
album_list: List[Album] = None, tracksort: int = 0, artwork: Optional[Artwork] = None, **kwargs) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
title: str = None,
|
||||
isrc: str = None,
|
||||
length: int = None,
|
||||
genre: str = None,
|
||||
note: FormattedText = None,
|
||||
source_list: List[Source] = None,
|
||||
target_list: List[Target] = None,
|
||||
lyrics_list: List[Lyrics] = None,
|
||||
main_artist_list: List[Artist] = None,
|
||||
feature_artist_list: List[Artist] = None,
|
||||
album_list: List[Album] = None,
|
||||
tracksort: int = 0,
|
||||
artwork: Optional[Artwork] = None,
|
||||
**kwargs
|
||||
) -> None:
|
||||
real_kwargs = copy.copy(locals())
|
||||
real_kwargs.update(real_kwargs.pop("kwargs", {}))
|
||||
|
||||
Base.__init__(**locals())
|
||||
Base.__init__(**real_kwargs)
|
||||
|
||||
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_artist_collection", "feature_artist_collection", "album_collection")
|
||||
TITEL = "title"
|
||||
@ -210,14 +225,6 @@ class Song(Base):
|
||||
r += get_collection_string(self.feature_artist_collection, " feat. {}")
|
||||
return r
|
||||
|
||||
@property
|
||||
def options(self) -> List[P]:
|
||||
options = self.main_artist_collection.shallow_list
|
||||
options.extend(self.feature_artist_collection)
|
||||
options.extend(self.album_collection)
|
||||
options.append(self)
|
||||
return options
|
||||
|
||||
@property
|
||||
def tracksort_str(self) -> str:
|
||||
"""
|
||||
@ -273,15 +280,27 @@ class Album(Base):
|
||||
TITEL = "title"
|
||||
|
||||
# This is automatically generated
|
||||
def __init__(self, title: str = None, unified_title: str = None, album_status: AlbumStatus = None,
|
||||
album_type: AlbumType = None, language: Language = None, date: ID3Timestamp = None,
|
||||
barcode: str = None, albumsort: int = None, notes: FormattedText = None,
|
||||
source_list: List[Source] = None, artist_list: List[Artist] = None, song_list: List[Song] = None,
|
||||
label_list: List[Label] = None, **kwargs) -> None:
|
||||
super().__init__(title=title, unified_title=unified_title, album_status=album_status, album_type=album_type,
|
||||
language=language, date=date, barcode=barcode, albumsort=albumsort, notes=notes,
|
||||
source_list=source_list, artist_list=artist_list, song_list=song_list, label_list=label_list,
|
||||
**kwargs)
|
||||
def __init__(
|
||||
self,
|
||||
title: str = None,
|
||||
unified_title: str = None,
|
||||
album_status: AlbumStatus = None,
|
||||
album_type: AlbumType = None,
|
||||
language: Language = None,
|
||||
date: ID3Timestamp = None,
|
||||
barcode: str = None,
|
||||
albumsort: int = None,
|
||||
notes: FormattedText = None,
|
||||
source_list: List[Source] = None,
|
||||
artist_list: List[Artist] = None,
|
||||
song_list: List[Song] = None,
|
||||
label_list: List[Label] = None,
|
||||
**kwargs
|
||||
) -> None:
|
||||
real_kwargs = copy.copy(locals())
|
||||
real_kwargs.update(real_kwargs.pop("kwargs", {}))
|
||||
|
||||
Base.__init__(**real_kwargs)
|
||||
|
||||
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("song_collection",)
|
||||
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection", "artist_collection")
|
||||
@ -413,14 +432,8 @@ class Album(Base):
|
||||
return self.album_type.value
|
||||
|
||||
|
||||
"""
|
||||
All objects dependent on Artist
|
||||
"""
|
||||
|
||||
|
||||
class Artist(Base):
|
||||
name: str
|
||||
unified_name: str
|
||||
country: Country
|
||||
formed_in: ID3Timestamp
|
||||
notes: FormattedText
|
||||
@ -437,8 +450,7 @@ class Artist(Base):
|
||||
label_collection: Collection[Label]
|
||||
|
||||
_default_factories = {
|
||||
"name": str,
|
||||
"unified_name": lambda: None,
|
||||
"name": lambda: None,
|
||||
"country": lambda: None,
|
||||
"unformatted_location": lambda: None,
|
||||
|
||||
@ -457,17 +469,28 @@ class Artist(Base):
|
||||
TITEL = "name"
|
||||
|
||||
# This is automatically generated
|
||||
def __init__(self, name: str = "", unified_name: str = None, country: Country = None,
|
||||
formed_in: ID3Timestamp = None, notes: FormattedText = None, lyrical_themes: List[str] = None,
|
||||
general_genre: str = None, unformatted_location: str = None, source_list: List[Source] = None,
|
||||
contact_list: List[Contact] = None, feature_song_list: List[Song] = None,
|
||||
main_album_list: List[Album] = None, label_list: List[Label] = None, **kwargs) -> None:
|
||||
|
||||
super().__init__(name=name, unified_name=unified_name, country=country, formed_in=formed_in, notes=notes,
|
||||
lyrical_themes=lyrical_themes, general_genre=general_genre,
|
||||
unformatted_location=unformatted_location, source_list=source_list, contact_list=contact_list,
|
||||
feature_song_list=feature_song_list, main_album_list=main_album_list, label_list=label_list,
|
||||
**kwargs)
|
||||
def __init__(
|
||||
self,
|
||||
name: str = None,
|
||||
unified_name: str = None,
|
||||
country: Country = None,
|
||||
formed_in: ID3Timestamp = None,
|
||||
notes: FormattedText = None,
|
||||
lyrical_themes: List[str] = None,
|
||||
general_genre: str = None,
|
||||
unformatted_location: str = None,
|
||||
source_list: List[Source] = None,
|
||||
contact_list: List[Contact] = None,
|
||||
feature_song_list: List[Song] = None,
|
||||
main_album_list: List[Album] = None,
|
||||
label_list: List[Label] = None,
|
||||
**kwargs
|
||||
) -> None:
|
||||
real_kwargs = copy.copy(locals())
|
||||
real_kwargs.update(real_kwargs.pop("kwargs", {}))
|
||||
|
||||
Base.__init__(**real_kwargs)
|
||||
|
||||
|
||||
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("main_album_collection", "feature_song_collection")
|
||||
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection",)
|
||||
@ -593,11 +616,6 @@ class Artist(Base):
|
||||
return r
|
||||
|
||||
|
||||
"""
|
||||
Label
|
||||
"""
|
||||
|
||||
|
||||
class Label(Base):
|
||||
COLLECTION_STRING_ATTRIBUTES = ("album_collection", "current_artist_collection")
|
||||
|
||||
@ -625,12 +643,21 @@ class Label(Base):
|
||||
|
||||
TITEL = "name"
|
||||
|
||||
def __init__(self, name: str = None, unified_name: str = None, notes: FormattedText = None,
|
||||
source_list: List[Source] = None, contact_list: List[Contact] = None,
|
||||
album_list: List[Album] = None, current_artist_list: List[Artist] = None, **kwargs) -> None:
|
||||
super().__init__(name=name, unified_name=unified_name, notes=notes, source_list=source_list,
|
||||
contact_list=contact_list, album_list=album_list, current_artist_list=current_artist_list,
|
||||
**kwargs)
|
||||
def __init__(
|
||||
self,
|
||||
name: str = None,
|
||||
unified_name: str = None,
|
||||
notes: FormattedText = None,
|
||||
source_list: List[Source] = None,
|
||||
contact_list: List[Contact] = None,
|
||||
album_list: List[Album] = None,
|
||||
current_artist_list: List[Artist] = None,
|
||||
**kwargs
|
||||
) -> None:
|
||||
real_kwargs = copy.copy(locals())
|
||||
real_kwargs.update(real_kwargs.pop("kwargs", {}))
|
||||
|
||||
Base.__init__(**real_kwargs)
|
||||
|
||||
def __init_collections__(self):
|
||||
self.album_collection.append_object_to_attribute = {
|
||||
|
@ -254,7 +254,7 @@ class Page:
|
||||
}
|
||||
|
||||
if obj_type in fetch_map:
|
||||
music_object = fetch_map[obj_type](source, stop_at_level)
|
||||
music_object = fetch_map[obj_type](source, stop_at_level=stop_at_level)
|
||||
else:
|
||||
self.LOGGER.warning(f"Can't fetch details of type: {obj_type}")
|
||||
return None
|
||||
|
@ -1,7 +1,7 @@
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import List, Optional, Type, Union, Generator
|
||||
from typing import List, Optional, Type, Union, Generator, Dict, Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pycountry
|
||||
@ -24,7 +24,7 @@ from ..objects import (
|
||||
Lyrics,
|
||||
Artwork
|
||||
)
|
||||
from ..utils.config import logging_settings
|
||||
from ..utils.config import logging_settings, main_settings
|
||||
from ..utils import string_processing, shared
|
||||
from ..utils.string_processing import clean_song_title
|
||||
from ..utils.support_classes.query import Query
|
||||
@ -361,7 +361,7 @@ class Musify(Page):
|
||||
|
||||
return Song(
|
||||
title=clean_song_title(song_title, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
|
||||
main_artist_list=artist_list,
|
||||
feature_artist_list=artist_list,
|
||||
source_list=source_list
|
||||
)
|
||||
|
||||
@ -510,7 +510,7 @@ class Musify(Page):
|
||||
title=clean_song_title(track_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
|
||||
source_list=source_list,
|
||||
lyrics_list=lyrics_list,
|
||||
main_artist_list=artist_list,
|
||||
feature_artist_list=artist_list,
|
||||
album_list=album_list,
|
||||
artwork=artwork,
|
||||
)
|
||||
@ -652,10 +652,101 @@ class Musify(Page):
|
||||
return Song(
|
||||
title=clean_song_title(song_name, artist_name=artist_list[0].name if len(artist_list) > 0 else None),
|
||||
tracksort=tracksort,
|
||||
main_artist_list=artist_list,
|
||||
feature_artist_list=artist_list,
|
||||
source_list=source_list
|
||||
)
|
||||
|
||||
|
||||
def _parse_album(self, soup: BeautifulSoup) -> Album:
|
||||
name: str = None
|
||||
source_list: List[Source] = []
|
||||
artist_list: List[Artist] = []
|
||||
date: ID3Timestamp = None
|
||||
|
||||
"""
|
||||
if breadcrumb list has 4 elements, then
|
||||
the -2 is the artist link,
|
||||
the -1 is the album
|
||||
"""
|
||||
# breadcrumb
|
||||
breadcrumb_soup: BeautifulSoup = soup.find("ol", {"class", "breadcrumb"})
|
||||
breadcrumb_elements: List[BeautifulSoup] = breadcrumb_soup.find_all("li", {"class": "breadcrumb-item"})
|
||||
if len(breadcrumb_elements) == 4:
|
||||
# album
|
||||
album_crumb: BeautifulSoup = breadcrumb_elements[-1]
|
||||
name = album_crumb.text.strip()
|
||||
|
||||
# artist
|
||||
artist_crumb: BeautifulSoup = breadcrumb_elements[-2]
|
||||
anchor: BeautifulSoup = artist_crumb.find("a")
|
||||
if anchor is not None:
|
||||
href = anchor.get("href")
|
||||
artist_source_list: List[Source] = []
|
||||
|
||||
if href is not None:
|
||||
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href.strip()))
|
||||
|
||||
span: BeautifulSoup = anchor.find("span")
|
||||
if span is not None:
|
||||
artist_list.append(Artist(
|
||||
name=span.get_text(strip=True),
|
||||
source_list=artist_source_list
|
||||
))
|
||||
else:
|
||||
self.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case")
|
||||
|
||||
# meta
|
||||
meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"})
|
||||
if meta_url is not None:
|
||||
url = meta_url.get("content")
|
||||
if url is not None:
|
||||
source_list.append(Source(self.SOURCE_TYPE, self.HOST + url))
|
||||
|
||||
meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"})
|
||||
if meta_name is not None:
|
||||
_name = meta_name.get("content")
|
||||
if _name is not None:
|
||||
name = _name
|
||||
|
||||
# album info
|
||||
album_info_ul: BeautifulSoup = soup.find("ul", {"class": "album-info"})
|
||||
if album_info_ul is not None:
|
||||
artist_anchor: BeautifulSoup
|
||||
for artist_anchor in album_info_ul.find_all("a", {"itemprop": "byArtist"}):
|
||||
# line 98
|
||||
artist_source_list: List[Source] = []
|
||||
|
||||
artist_url_meta = artist_anchor.find("meta", {"itemprop": "url"})
|
||||
if artist_url_meta is not None:
|
||||
artist_href = artist_url_meta.get("content")
|
||||
if artist_href is not None:
|
||||
artist_source_list.append(Source(self.SOURCE_TYPE, url=self.HOST + artist_href))
|
||||
|
||||
artist_meta_name = artist_anchor.find("meta", {"itemprop": "name"})
|
||||
if artist_meta_name is not None:
|
||||
artist_name = artist_meta_name.get("content")
|
||||
if artist_name is not None:
|
||||
artist_list.append(Artist(
|
||||
name=artist_name,
|
||||
source_list=artist_source_list
|
||||
))
|
||||
|
||||
time_soup: BeautifulSoup = album_info_ul.find("time", {"itemprop": "datePublished"})
|
||||
if time_soup is not None:
|
||||
raw_datetime = time_soup.get("datetime")
|
||||
if raw_datetime is not None:
|
||||
try:
|
||||
date = ID3Timestamp.strptime(raw_datetime, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
|
||||
|
||||
return Album(
|
||||
title=name,
|
||||
source_list=source_list,
|
||||
artist_list=artist_list,
|
||||
date=date
|
||||
)
|
||||
|
||||
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
|
||||
"""
|
||||
fetches album from source:
|
||||
@ -694,19 +785,14 @@ class Musify(Page):
|
||||
|
||||
return album
|
||||
|
||||
def _get_artist_attributes(self, url: MusifyUrl) -> Artist:
|
||||
def _fetch_initial_artist(self, url: MusifyUrl, source: Source, **kwargs) -> Artist:
|
||||
"""
|
||||
fetches the main Artist attributes from this endpoint
|
||||
https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent
|
||||
it needs to parse html
|
||||
|
||||
:param url:
|
||||
:return:
|
||||
"""
|
||||
|
||||
r = self.connection.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent", name="artist_attributes_" + url.name_with_id)
|
||||
if r is None:
|
||||
return Artist()
|
||||
return Artist(source_list=[source])
|
||||
|
||||
soup = self.get_soup_from_response(r)
|
||||
|
||||
@ -821,7 +907,7 @@ class Musify(Page):
|
||||
notes=notes
|
||||
)
|
||||
|
||||
def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None) -> Album:
|
||||
def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None, **kwargs) -> Album:
|
||||
"""
|
||||
<div class="card release-thumbnail" data-type="2">
|
||||
<a href="/release/ghost-bath-self-loather-2021-1554266">
|
||||
@ -845,33 +931,9 @@ class Musify(Page):
|
||||
</div>
|
||||
"""
|
||||
|
||||
_id: Optional[str] = None
|
||||
name: str = None
|
||||
source_list: List[Source] = []
|
||||
timestamp: Optional[ID3Timestamp] = None
|
||||
album_status = None
|
||||
|
||||
def set_name(new_name: str):
|
||||
nonlocal name
|
||||
nonlocal artist_name
|
||||
|
||||
# example of just setting not working:
|
||||
# https://musify.club/release/unjoy-eurythmie-psychonaut-4-tired-numb-still-alive-2012-324067
|
||||
if new_name.count(" - ") != 1:
|
||||
name = new_name
|
||||
return
|
||||
|
||||
potential_artist_list, potential_name = new_name.split(" - ")
|
||||
unified_artist_list = string_processing.unify(potential_artist_list)
|
||||
if artist_name is not None:
|
||||
if string_processing.unify(artist_name) not in unified_artist_list:
|
||||
name = new_name
|
||||
return
|
||||
|
||||
name = potential_name
|
||||
return
|
||||
|
||||
name = new_name
|
||||
album_kwargs: Dict[str, Any] = {
|
||||
"source_list": [],
|
||||
}
|
||||
|
||||
album_status_id = album_card.get("data-type")
|
||||
if album_status_id.isdigit():
|
||||
@ -882,9 +944,7 @@ class Musify(Page):
|
||||
album_status = AlbumStatus.BOOTLEG
|
||||
|
||||
def parse_release_anchor(_anchor: BeautifulSoup, text_is_name=False):
|
||||
nonlocal _id
|
||||
nonlocal name
|
||||
nonlocal source_list
|
||||
nonlocal album_kwargs
|
||||
|
||||
if _anchor is None:
|
||||
return
|
||||
@ -892,20 +952,13 @@ class Musify(Page):
|
||||
href = _anchor.get("href")
|
||||
if href is not None:
|
||||
# add url to sources
|
||||
source_list.append(Source(
|
||||
album_kwargs["source_list"].append(Source(
|
||||
self.SOURCE_TYPE,
|
||||
self.HOST + href
|
||||
))
|
||||
|
||||
# split id from url
|
||||
split_href = href.split("-")
|
||||
if len(split_href) > 1:
|
||||
_id = split_href[-1]
|
||||
|
||||
if not text_is_name:
|
||||
return
|
||||
|
||||
set_name(_anchor.text)
|
||||
if text_is_name:
|
||||
album_kwargs["title"] = clean_song_title(_anchor.text, artist_name)
|
||||
|
||||
anchor_list = album_card.find_all("a", recursive=False)
|
||||
if len(anchor_list) > 0:
|
||||
@ -916,7 +969,7 @@ class Musify(Page):
|
||||
if thumbnail is not None:
|
||||
alt = thumbnail.get("alt")
|
||||
if alt is not None:
|
||||
set_name(alt)
|
||||
album_kwargs["title"] = clean_song_title(alt, artist_name)
|
||||
|
||||
image_url = thumbnail.get("src")
|
||||
else:
|
||||
@ -933,7 +986,7 @@ class Musify(Page):
|
||||
13.11.2021
|
||||
</small>
|
||||
"""
|
||||
nonlocal timestamp
|
||||
nonlocal album_kwargs
|
||||
|
||||
italic_tagging_soup: BeautifulSoup = small_soup.find("i")
|
||||
if italic_tagging_soup is None:
|
||||
@ -943,7 +996,7 @@ class Musify(Page):
|
||||
return
|
||||
|
||||
raw_time = small_soup.text.strip()
|
||||
timestamp = ID3Timestamp.strptime(raw_time, "%d.%m.%Y")
|
||||
album_kwargs["date"] = ID3Timestamp.strptime(raw_time, "%d.%m.%Y")
|
||||
|
||||
# parse small date
|
||||
card_footer_list = album_card.find_all("div", {"class": "card-footer"})
|
||||
@ -956,112 +1009,18 @@ class Musify(Page):
|
||||
else:
|
||||
self.LOGGER.debug("there is not even 1 footer in the album card")
|
||||
|
||||
return Album(
|
||||
title=name,
|
||||
source_list=source_list,
|
||||
date=timestamp,
|
||||
album_type=album_type,
|
||||
album_status=album_status
|
||||
)
|
||||
return Album(**album_kwargs)
|
||||
|
||||
def _parse_album(self, soup: BeautifulSoup) -> Album:
|
||||
name: str = None
|
||||
source_list: List[Source] = []
|
||||
artist_list: List[Artist] = []
|
||||
date: ID3Timestamp = None
|
||||
|
||||
"""
|
||||
if breadcrumb list has 4 elements, then
|
||||
the -2 is the artist link,
|
||||
the -1 is the album
|
||||
"""
|
||||
# breadcrumb
|
||||
breadcrumb_soup: BeautifulSoup = soup.find("ol", {"class", "breadcrumb"})
|
||||
breadcrumb_elements: List[BeautifulSoup] = breadcrumb_soup.find_all("li", {"class": "breadcrumb-item"})
|
||||
if len(breadcrumb_elements) == 4:
|
||||
# album
|
||||
album_crumb: BeautifulSoup = breadcrumb_elements[-1]
|
||||
name = album_crumb.text.strip()
|
||||
|
||||
# artist
|
||||
artist_crumb: BeautifulSoup = breadcrumb_elements[-2]
|
||||
anchor: BeautifulSoup = artist_crumb.find("a")
|
||||
if anchor is not None:
|
||||
href = anchor.get("href")
|
||||
artist_source_list: List[Source] = []
|
||||
|
||||
if href is not None:
|
||||
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href.strip()))
|
||||
|
||||
span: BeautifulSoup = anchor.find("span")
|
||||
if span is not None:
|
||||
artist_list.append(Artist(
|
||||
name=span.get_text(strip=True),
|
||||
source_list=artist_source_list
|
||||
))
|
||||
else:
|
||||
self.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case")
|
||||
|
||||
# meta
|
||||
meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"})
|
||||
if meta_url is not None:
|
||||
url = meta_url.get("content")
|
||||
if url is not None:
|
||||
source_list.append(Source(self.SOURCE_TYPE, self.HOST + url))
|
||||
|
||||
meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"})
|
||||
if meta_name is not None:
|
||||
_name = meta_name.get("content")
|
||||
if _name is not None:
|
||||
name = _name
|
||||
|
||||
# album info
|
||||
album_info_ul: BeautifulSoup = soup.find("ul", {"class": "album-info"})
|
||||
if album_info_ul is not None:
|
||||
artist_anchor: BeautifulSoup
|
||||
for artist_anchor in album_info_ul.find_all("a", {"itemprop": "byArtist"}):
|
||||
# line 98
|
||||
artist_source_list: List[Source] = []
|
||||
|
||||
artist_url_meta = artist_anchor.find("meta", {"itemprop": "url"})
|
||||
if artist_url_meta is not None:
|
||||
artist_href = artist_url_meta.get("content")
|
||||
if artist_href is not None:
|
||||
artist_source_list.append(Source(self.SOURCE_TYPE, url=self.HOST + artist_href))
|
||||
|
||||
artist_meta_name = artist_anchor.find("meta", {"itemprop": "name"})
|
||||
if artist_meta_name is not None:
|
||||
artist_name = artist_meta_name.get("content")
|
||||
if artist_name is not None:
|
||||
artist_list.append(Artist(
|
||||
name=artist_name,
|
||||
source_list=artist_source_list
|
||||
))
|
||||
|
||||
time_soup: BeautifulSoup = album_info_ul.find("time", {"itemprop": "datePublished"})
|
||||
if time_soup is not None:
|
||||
raw_datetime = time_soup.get("datetime")
|
||||
if raw_datetime is not None:
|
||||
try:
|
||||
date = ID3Timestamp.strptime(raw_datetime, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
|
||||
|
||||
return Album(
|
||||
title=name,
|
||||
source_list=source_list,
|
||||
artist_list=artist_list,
|
||||
date=date
|
||||
)
|
||||
|
||||
def _get_discography(self, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> Generator[Album, None, None]:
|
||||
def _fetch_artist_discography(self, artist: Artist, url: MusifyUrl, artist_name: str = None, **kwargs):
|
||||
"""
|
||||
POST https://musify.club/artist/filteralbums
|
||||
ArtistID: 280348
|
||||
SortOrder.Property: dateCreated
|
||||
SortOrder.IsAscending: false
|
||||
X-Requested-With: XMLHttpRequest
|
||||
ArtistID: 280348
|
||||
SortOrder.Property: dateCreated
|
||||
SortOrder.IsAscending: false
|
||||
X-Requested-With: XMLHttpRequest
|
||||
"""
|
||||
_download_all = kwargs.get("download_all", False)
|
||||
_album_type_blacklist = kwargs.get("album_type_blacklist", main_settings["album_type_blacklist"])
|
||||
|
||||
endpoint = self.HOST + "/" + url.source_type.value + "/filteralbums"
|
||||
|
||||
@ -1072,33 +1031,29 @@ class Musify(Page):
|
||||
"X-Requested-With": "XMLHttpRequest"
|
||||
}, name="discography_" + url.name_with_id)
|
||||
if r is None:
|
||||
return []
|
||||
soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser")
|
||||
return
|
||||
|
||||
soup: BeautifulSoup = self.get_soup_from_response(r)
|
||||
|
||||
for card_soup in soup.find_all("div", {"class": "card"}):
|
||||
yield self._parse_album_card(card_soup, artist_name)
|
||||
album = self._parse_album_card(card_soup, artist_name, **kwargs)
|
||||
if album.album_type in _album_type_blacklist:
|
||||
continue
|
||||
|
||||
def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist:
|
||||
artist.main_album_collection.append(album)
|
||||
|
||||
def fetch_artist(self, source: Source, **kwargs) -> Artist:
|
||||
"""
|
||||
fetches artist from source
|
||||
|
||||
TODO
|
||||
[x] discography
|
||||
[x] attributes
|
||||
[] picture gallery
|
||||
|
||||
Args:
|
||||
source (Source): the source to fetch
|
||||
stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False.
|
||||
|
||||
Returns:
|
||||
Artist: the artist fetched
|
||||
"""
|
||||
|
||||
url = parse_url(source.url)
|
||||
|
||||
artist = self._get_artist_attributes(url)
|
||||
|
||||
artist.main_album_collection.extend(self._get_discography(url, artist.name))
|
||||
artist = self._fetch_initial_artist(url, source=source, **kwargs)
|
||||
self._fetch_artist_discography(artist, url, artist.name, **kwargs)
|
||||
|
||||
return artist
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user