From 267bf5284715947b48ce270dd79b8fdf207744ff Mon Sep 17 00:00:00 2001 From: Hellow Date: Fri, 24 Mar 2023 15:58:21 +0100 Subject: [PATCH] fixed funny bug --- .../html/musify/album_overview.html | 0 src/music_kraken/__init__.py | 6 +- src/music_kraken/database/__init__.py | 19 ----- src/music_kraken/database/database.py | 4 +- src/music_kraken/database/object_cache.py | 14 ++-- src/music_kraken/objects/__init__.py | 2 +- src/music_kraken/objects/collection.py | 71 +++++++++------- src/music_kraken/objects/parents.py | 1 + src/music_kraken/pages/abstract.py | 81 ++++++++++++++++--- .../pages/encyclopaedia_metallum.py | 2 +- src/music_kraken/pages/musify.py | 16 ++-- src/musify_search.py | 8 +- 12 files changed, 142 insertions(+), 82 deletions(-) rename {src/music_kraken/pages => documentation}/html/musify/album_overview.html (100%) diff --git a/src/music_kraken/pages/html/musify/album_overview.html b/documentation/html/musify/album_overview.html similarity index 100% rename from src/music_kraken/pages/html/musify/album_overview.html rename to documentation/html/musify/album_overview.html diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index 03e31c6..d00eb2b 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -38,18 +38,18 @@ logging.getLogger("musicbrainzngs").setLevel(logging.WARNING) musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeIIow2/music-downloader") -def get_options_from_query(query: str) -> List[objects.MusicObject]: +def get_options_from_query(query: str) -> List[objects.DatabaseObject]: options = [] for MetadataPage in pages.MetadataPages: options.extend(MetadataPage.search_by_query(query=query)) return options -def get_options_from_option(option: objects.MusicObject) -> List[objects.MusicObject]: +def get_options_from_option(option: objects.DatabaseObject) -> List[objects.DatabaseObject]: for MetadataPage in pages.MetadataPages: option = MetadataPage.fetch_details(option, flat=False) return option.get_options() -def print_options(options: List[objects.MusicObject]): +def print_options(options: List[objects.DatabaseObject]): print("\n".join([f"{str(j).zfill(2)}: {i.get_option_string()}" for j, i in enumerate(options)])) def cli(): diff --git a/src/music_kraken/database/__init__.py b/src/music_kraken/database/__init__.py index 5a22e6a..e69de29 100644 --- a/src/music_kraken/database/__init__.py +++ b/src/music_kraken/database/__init__.py @@ -1,19 +0,0 @@ -from . import database -from .. import objects - -MusicObject = objects.MusicObject - -ID3Timestamp = objects.ID3Timestamp -SourceTypes = objects.SourceTypes -SourcePages = objects.SourcePages -Song = objects.Song -Source = objects.Source -Target = objects.Target -Lyrics = objects.Lyrics -Album = objects.Album -Artist = objects.Artist - -FormattedText = objects.FormattedText - -Database = database.Database -# cache = temp_database.TempDatabase() diff --git a/src/music_kraken/database/database.py b/src/music_kraken/database/database.py index d20e793..0120a89 100644 --- a/src/music_kraken/database/database.py +++ b/src/music_kraken/database/database.py @@ -130,7 +130,7 @@ class Database: print(model._meta.fields) - def push(self, database_object: objects.MusicObject): + def push(self, database_object: objects.DatabaseObject): """ Adds a new music object to the database using the corresponding method from the `write` session. When possible, rather use the `push_many` function. @@ -153,7 +153,7 @@ class Database: if isinstance(database_object, objects.Artist): return writing_session.add_artist(database_object) - def push_many(self, database_objects: List[objects.MusicObject]) -> None: + def push_many(self, database_objects: List[objects.DatabaseObject]) -> None: """ Adds a list of MusicObject instances to the database. This function sends only needs one querry for each type of table added. diff --git a/src/music_kraken/database/object_cache.py b/src/music_kraken/database/object_cache.py index a9c924c..9251dcd 100644 --- a/src/music_kraken/database/object_cache.py +++ b/src/music_kraken/database/object_cache.py @@ -2,7 +2,7 @@ from collections import defaultdict from typing import Dict, List, Optional import weakref -from src.music_kraken.objects import MusicObject +from src.music_kraken.objects import DatabaseObject """ This is a cache for the objects, that et pulled out of the database. @@ -32,14 +32,14 @@ class ObjectCache: :method extent: Add a list of MusicObjects to the cache. :method remove: Remove a MusicObject from the cache by its id. :method get: Retrieve a MusicObject from the cache by its id. """ - object_to_id: Dict[str, MusicObject] + object_to_id: Dict[str, DatabaseObject] weakref_map: Dict[weakref.ref, str] def __init__(self) -> None: self.object_to_id = dict() self.weakref_map = defaultdict() - def exists(self, music_object: MusicObject) -> bool: + def exists(self, music_object: DatabaseObject) -> bool: """ Check if a MusicObject with the same id already exists in the cache. @@ -60,7 +60,7 @@ class ObjectCache: data_id = self.weakref_map.pop(weakref_) self.object_to_id.pop(data_id) - def append(self, music_object: MusicObject) -> bool: + def append(self, music_object: DatabaseObject) -> bool: """ Add a MusicObject to the cache. @@ -75,7 +75,7 @@ class ObjectCache: return False - def extent(self, music_object_list: List[MusicObject]): + def extent(self, music_object_list: List[DatabaseObject]): """ adjacent to the extent method of list, this appends n Object """ @@ -93,7 +93,7 @@ class ObjectCache: self.weakref_map.pop(weakref.ref(data)) self.object_to_id.pop(_id) - def __getitem__(self, item) -> Optional[MusicObject]: + def __getitem__(self, item) -> Optional[DatabaseObject]: """ this returns the data obj :param item: the id of the music object @@ -102,5 +102,5 @@ class ObjectCache: return self.object_to_id.get(item) - def get(self, _id: str) -> Optional[MusicObject]: + def get(self, _id: str) -> Optional[DatabaseObject]: return self.__getitem__(_id) diff --git a/src/music_kraken/objects/__init__.py b/src/music_kraken/objects/__init__.py index 80e4064..fa7ba07 100644 --- a/src/music_kraken/objects/__init__.py +++ b/src/music_kraken/objects/__init__.py @@ -9,7 +9,7 @@ from . import ( collection ) -MusicObject = parents.DatabaseObject +DatabaseObject = parents.DatabaseObject ID3Mapping = metadata.Mapping ID3Timestamp = metadata.ID3Timestamp diff --git a/src/music_kraken/objects/collection.py b/src/music_kraken/objects/collection.py index bfb5b90..6009200 100644 --- a/src/music_kraken/objects/collection.py +++ b/src/music_kraken/objects/collection.py @@ -1,9 +1,16 @@ from typing import List, Iterable, Dict from collections import defaultdict +from dataclasses import dataclass from .parents import DatabaseObject +@dataclass +class AppendResult: + was_in_collection: bool + current_element: DatabaseObject + + class Collection: """ This a class for the iterables @@ -14,12 +21,12 @@ class Collection: _by_url: dict _by_attribute: dict - def __init__(self, data: List[DatabaseObject] = None, element_type = None, *args, **kwargs) -> None: + def __init__(self, data: List[DatabaseObject] = None, element_type=None, *args, **kwargs) -> None: # Attribute needs to point to self.element_type = element_type - + self._data: List[DatabaseObject] = list() - + """ example of attribute_to_object_map the song objects are references pointing to objects @@ -34,7 +41,7 @@ class Collection: """ self._attribute_to_object_map: Dict[str, Dict[object, DatabaseObject]] = defaultdict(dict) self._used_ids: set = set() - + if data is not None: self.extend(data, merge_on_conflict=True) @@ -47,14 +54,14 @@ class Collection: continue self._attribute_to_object_map[name][value] = element - + self._used_ids.add(element.id) - + def unmap_element(self, element: DatabaseObject): for name, value in element.indexing_values: if value is None: continue - + if value in self._attribute_to_object_map[name]: if element is self._attribute_to_object_map[name][value]: try: @@ -62,7 +69,8 @@ class Collection: except KeyError: pass - def append(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> DatabaseObject: + def append(self, element: DatabaseObject, merge_on_conflict: bool = True, + merge_into_existing: bool = True) -> AppendResult: """ :param element: :param merge_on_conflict: @@ -77,41 +85,39 @@ class Collection: for name, value in element.indexing_values: if value in self._attribute_to_object_map[name]: existing_object = self._attribute_to_object_map[name][value] - + if not merge_on_conflict: - return existing_object - + return AppendResult(True, existing_object) + # if the object does already exist # thus merging and don't add it afterwards if merge_into_existing: existing_object.merge(element) # in case any relevant data has been added (e.g. it remaps the old object) self.map_element(existing_object) - return existing_object - + return AppendResult(True, existing_object) + element.merge(existing_object) - + exists_at = self._data.index(existing_object) self._data[exists_at] = element - + self.unmap_element(existing_object) self.map_element(element) - return element + return AppendResult(True, existing_object) self._data.append(element) self.map_element(element) - - return element - - def append_is_already_in_collection(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> bool: - object_representing_the_data = self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing) - def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True): + return AppendResult(False, element) + + def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True, + merge_into_existing: bool = True): for element in element_list: - self.append(element, merge_on_conflict=merge_on_conflict) + self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing) def __iter__(self): - for element in self._data: + for element in self.shallow_list: yield element def __str__(self) -> str: @@ -120,11 +126,22 @@ class Collection: def __len__(self) -> int: return len(self._data) - def __getitem__(self, item): - if type(item) != int: + def __getitem__(self, key): + if type(key) != int: return ValueError("key needs to be an integer") - return self._data[item] + return self._data[key] + + def __setitem__(self, key, value: DatabaseObject): + print(key, value) + if type(key) != int: + return ValueError("key needs to be an integer") + + old_item = self._data[key] + self.unmap_element(old_item) + self.map_element(value) + + self._data[key] = value @property def shallow_list(self) -> List[DatabaseObject]: diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index 855ba71..2152677 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -99,6 +99,7 @@ class DatabaseObject: pass + class MainObject(DatabaseObject): """ This is the parent class for all "main" data objects: diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index a5cae98..dc916e3 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -1,11 +1,8 @@ -from typing import Optional, Union, Type +from typing import Optional, Union, Type, Dict import requests import logging -LOGGER = logging.getLogger("this shouldn't be used") - from ..utils import shared - from ..objects import ( Song, Source, @@ -13,13 +10,15 @@ from ..objects import ( Artist, Lyrics, Target, - MusicObject, + DatabaseObject, Options, SourcePages, Collection, Label ) +LOGGER = logging.getLogger("this shouldn't be used") + class Page: """ @@ -139,7 +138,7 @@ class Page: return Options() @classmethod - def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> MusicObject: + def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: """ when a music object with laccing data is passed in, it returns the SAME object **(no copy)** with more detailed data. @@ -156,21 +155,28 @@ class Page: :return detailed_music_object: IT MODIFIES THE INPUT OBJ """ - new_music_object: MusicObject = type(music_object).__init__() - + new_music_object: DatabaseObject = type(music_object)() + source: Source for source in music_object.source_collection: - new_music_object.merge(cls.fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) + new_music_object.merge(cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) + collections = { + Label: Collection(element_type=Label), + Artist: Collection(element_type=Artist), + Album: Collection(element_type=Album), + Song: Collection(element_type=Song) + } + cls._clean_music_object(new_music_object, collections) music_object.merge(new_music_object) - music_object.compile() + # music_object.compile() return music_object @classmethod - def fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1): + def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1): if obj_type == Artist: return cls.fetch_artist_from_source(source=source, stop_at_level=stop_at_level) @@ -183,6 +189,54 @@ class Page: if obj_type == Label: return cls.fetch_label_from_source(source=source, stop_at_level=stop_at_level) + @classmethod + def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + if type(music_object) == Label: + return cls._clean_label(label=music_object, collections=collections) + if type(music_object) == Artist: + return cls._clean_artist(artist=music_object, collections=collections) + if type(music_object) == Album: + return cls._clean_album(album=music_object, collections=collections) + if type(music_object) == Song: + return cls._clean_song(song=music_object, collections=collections) + + @classmethod + def _clean_collection(cls, collection: Collection, collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + if collection.element_type not in collection_dict: + return + + for i, element in enumerate(collection): + r = collection_dict[collection.element_type].append(element) + if not r.was_in_collection: + cls._clean_music_object(r.current_element, collection_dict) + continue + + collection[i] = r.current_element + cls._clean_music_object(r.current_element, collection_dict) + + @classmethod + def _clean_label(cls, label: Label, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + cls._clean_collection(label.current_artist_collection, collections) + cls._clean_collection(label.album_collection, collections) + + @classmethod + def _clean_artist(cls, artist: Artist, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + cls._clean_collection(artist.main_album_collection, collections) + cls._clean_collection(artist.feature_song_collection, collections) + cls._clean_collection(artist.label_collection, collections) + + @classmethod + def _clean_album(cls, album: Album, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + cls._clean_collection(album.label_collection, collections) + cls._clean_collection(album.song_collection, collections) + cls._clean_collection(album.artist_collection, collections) + + @classmethod + def _clean_song(cls, song: Song, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): + cls._clean_collection(song.album_collection, collections) + cls._clean_collection(song.feature_artist_collection, collections) + cls._clean_collection(song.main_artist_collection, collections) + @classmethod def fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song: return Song() @@ -195,6 +249,7 @@ class Page: @classmethod def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: return Artist() - - def fetch_label_from_source(source: Source, stop_at_level: int = 1) -> Label: + + @classmethod + def fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label: return Label() diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index f226303..987d6c9 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -9,7 +9,7 @@ from ..utils.shared import ( from .abstract import Page from ..objects import ( - MusicObject, + DatabaseObject, Artist, Source, SourcePages, diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index e874ca7..a78ed93 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -14,7 +14,7 @@ from ..utils.shared import ( from .abstract import Page from ..objects import ( - MusicObject, + DatabaseObject, Artist, Source, SourcePages, @@ -545,7 +545,7 @@ class Musify(Page): )) @classmethod - def get_discography(cls, url: MusifyUrl, artist_name: str = None, flat=False) -> List[Album]: + def get_discography(cls, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> List[Album]: """ POST https://musify.club/artist/filteralbums ArtistID: 280348 @@ -570,9 +570,9 @@ class Musify(Page): for card_soup in soup.find_all("div", {"class": "card"}): new_album: Album = cls.parse_album_card(card_soup, artist_name) album_source: Source - if not flat: + if stop_at_level > 1: for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE): - new_album.merge(cls.fetch_album_from_source(album_source)) + new_album.merge(cls.fetch_album_from_source(album_source, stop_at_level=stop_at_level-1)) discography.append(new_album) @@ -709,7 +709,7 @@ class Musify(Page): ) @classmethod - def fetch_artist_from_source(cls, source: Source, flat: bool = False) -> Artist: + def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: """ fetches artist from source @@ -719,7 +719,7 @@ class Musify(Page): Args: source (Source): the source to fetch - flat (bool, optional): if it is false, every album from discograohy will be fetched. Defaults to False. + stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False. Returns: Artist: the artist fetched @@ -851,7 +851,7 @@ class Musify(Page): ) @classmethod - def fetch_album_from_source(cls, source: Source, flat: bool = False) -> Album: + def fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: """ fetches album from source: eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188' @@ -861,8 +861,8 @@ class Musify(Page): [] attributes [] ratings + :param stop_at_level: :param source: - :param flat: :return: """ album = Album(title="Hi :)") diff --git a/src/musify_search.py b/src/musify_search.py index 1dbdb68..63a23bd 100644 --- a/src/musify_search.py +++ b/src/musify_search.py @@ -22,9 +22,15 @@ def fetch_album(): "https://musify.club/release/linkin-park-hybrid-theory-2000-188")] ) - album = Musify.fetch_details(album) + album: objects.Album = Musify.fetch_details(album) print(album.options) + song: objects.Song + for artist in album.artist_collection: + print(artist.id, artist.name) + for song in album.song_collection: + for artist in song.main_artist_collection: + print(artist.id, artist.name) if __name__ == "__main__": fetch_album()