fixed funny bug

This commit is contained in:
Hellow 2023-03-24 15:58:21 +01:00
parent 5387301ed2
commit 267bf52847
12 changed files with 142 additions and 82 deletions

View File

@ -38,18 +38,18 @@ logging.getLogger("musicbrainzngs").setLevel(logging.WARNING)
musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeIIow2/music-downloader") musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeIIow2/music-downloader")
def get_options_from_query(query: str) -> List[objects.MusicObject]: def get_options_from_query(query: str) -> List[objects.DatabaseObject]:
options = [] options = []
for MetadataPage in pages.MetadataPages: for MetadataPage in pages.MetadataPages:
options.extend(MetadataPage.search_by_query(query=query)) options.extend(MetadataPage.search_by_query(query=query))
return options return options
def get_options_from_option(option: objects.MusicObject) -> List[objects.MusicObject]: def get_options_from_option(option: objects.DatabaseObject) -> List[objects.DatabaseObject]:
for MetadataPage in pages.MetadataPages: for MetadataPage in pages.MetadataPages:
option = MetadataPage.fetch_details(option, flat=False) option = MetadataPage.fetch_details(option, flat=False)
return option.get_options() return option.get_options()
def print_options(options: List[objects.MusicObject]): def print_options(options: List[objects.DatabaseObject]):
print("\n".join([f"{str(j).zfill(2)}: {i.get_option_string()}" for j, i in enumerate(options)])) print("\n".join([f"{str(j).zfill(2)}: {i.get_option_string()}" for j, i in enumerate(options)]))
def cli(): def cli():

View File

@ -1,19 +0,0 @@
from . import database
from .. import objects
MusicObject = objects.MusicObject
ID3Timestamp = objects.ID3Timestamp
SourceTypes = objects.SourceTypes
SourcePages = objects.SourcePages
Song = objects.Song
Source = objects.Source
Target = objects.Target
Lyrics = objects.Lyrics
Album = objects.Album
Artist = objects.Artist
FormattedText = objects.FormattedText
Database = database.Database
# cache = temp_database.TempDatabase()

View File

@ -130,7 +130,7 @@ class Database:
print(model._meta.fields) print(model._meta.fields)
def push(self, database_object: objects.MusicObject): def push(self, database_object: objects.DatabaseObject):
""" """
Adds a new music object to the database using the corresponding method from the `write` session. Adds a new music object to the database using the corresponding method from the `write` session.
When possible, rather use the `push_many` function. When possible, rather use the `push_many` function.
@ -153,7 +153,7 @@ class Database:
if isinstance(database_object, objects.Artist): if isinstance(database_object, objects.Artist):
return writing_session.add_artist(database_object) return writing_session.add_artist(database_object)
def push_many(self, database_objects: List[objects.MusicObject]) -> None: def push_many(self, database_objects: List[objects.DatabaseObject]) -> None:
""" """
Adds a list of MusicObject instances to the database. Adds a list of MusicObject instances to the database.
This function sends only needs one querry for each type of table added. This function sends only needs one querry for each type of table added.

View File

@ -2,7 +2,7 @@ from collections import defaultdict
from typing import Dict, List, Optional from typing import Dict, List, Optional
import weakref import weakref
from src.music_kraken.objects import MusicObject from src.music_kraken.objects import DatabaseObject
""" """
This is a cache for the objects, that et pulled out of the database. This is a cache for the objects, that et pulled out of the database.
@ -32,14 +32,14 @@ class ObjectCache:
:method extent: Add a list of MusicObjects to the cache. :method extent: Add a list of MusicObjects to the cache.
:method remove: Remove a MusicObject from the cache by its id. :method remove: Remove a MusicObject from the cache by its id.
:method get: Retrieve a MusicObject from the cache by its id. """ :method get: Retrieve a MusicObject from the cache by its id. """
object_to_id: Dict[str, MusicObject] object_to_id: Dict[str, DatabaseObject]
weakref_map: Dict[weakref.ref, str] weakref_map: Dict[weakref.ref, str]
def __init__(self) -> None: def __init__(self) -> None:
self.object_to_id = dict() self.object_to_id = dict()
self.weakref_map = defaultdict() self.weakref_map = defaultdict()
def exists(self, music_object: MusicObject) -> bool: def exists(self, music_object: DatabaseObject) -> bool:
""" """
Check if a MusicObject with the same id already exists in the cache. Check if a MusicObject with the same id already exists in the cache.
@ -60,7 +60,7 @@ class ObjectCache:
data_id = self.weakref_map.pop(weakref_) data_id = self.weakref_map.pop(weakref_)
self.object_to_id.pop(data_id) self.object_to_id.pop(data_id)
def append(self, music_object: MusicObject) -> bool: def append(self, music_object: DatabaseObject) -> bool:
""" """
Add a MusicObject to the cache. Add a MusicObject to the cache.
@ -75,7 +75,7 @@ class ObjectCache:
return False return False
def extent(self, music_object_list: List[MusicObject]): def extent(self, music_object_list: List[DatabaseObject]):
""" """
adjacent to the extent method of list, this appends n Object adjacent to the extent method of list, this appends n Object
""" """
@ -93,7 +93,7 @@ class ObjectCache:
self.weakref_map.pop(weakref.ref(data)) self.weakref_map.pop(weakref.ref(data))
self.object_to_id.pop(_id) self.object_to_id.pop(_id)
def __getitem__(self, item) -> Optional[MusicObject]: def __getitem__(self, item) -> Optional[DatabaseObject]:
""" """
this returns the data obj this returns the data obj
:param item: the id of the music object :param item: the id of the music object
@ -102,5 +102,5 @@ class ObjectCache:
return self.object_to_id.get(item) return self.object_to_id.get(item)
def get(self, _id: str) -> Optional[MusicObject]: def get(self, _id: str) -> Optional[DatabaseObject]:
return self.__getitem__(_id) return self.__getitem__(_id)

View File

@ -9,7 +9,7 @@ from . import (
collection collection
) )
MusicObject = parents.DatabaseObject DatabaseObject = parents.DatabaseObject
ID3Mapping = metadata.Mapping ID3Mapping = metadata.Mapping
ID3Timestamp = metadata.ID3Timestamp ID3Timestamp = metadata.ID3Timestamp

View File

@ -1,9 +1,16 @@
from typing import List, Iterable, Dict from typing import List, Iterable, Dict
from collections import defaultdict from collections import defaultdict
from dataclasses import dataclass
from .parents import DatabaseObject from .parents import DatabaseObject
@dataclass
class AppendResult:
was_in_collection: bool
current_element: DatabaseObject
class Collection: class Collection:
""" """
This a class for the iterables This a class for the iterables
@ -14,12 +21,12 @@ class Collection:
_by_url: dict _by_url: dict
_by_attribute: dict _by_attribute: dict
def __init__(self, data: List[DatabaseObject] = None, element_type = None, *args, **kwargs) -> None: def __init__(self, data: List[DatabaseObject] = None, element_type=None, *args, **kwargs) -> None:
# Attribute needs to point to # Attribute needs to point to
self.element_type = element_type self.element_type = element_type
self._data: List[DatabaseObject] = list() self._data: List[DatabaseObject] = list()
""" """
example of attribute_to_object_map example of attribute_to_object_map
the song objects are references pointing to objects the song objects are references pointing to objects
@ -34,7 +41,7 @@ class Collection:
""" """
self._attribute_to_object_map: Dict[str, Dict[object, DatabaseObject]] = defaultdict(dict) self._attribute_to_object_map: Dict[str, Dict[object, DatabaseObject]] = defaultdict(dict)
self._used_ids: set = set() self._used_ids: set = set()
if data is not None: if data is not None:
self.extend(data, merge_on_conflict=True) self.extend(data, merge_on_conflict=True)
@ -47,14 +54,14 @@ class Collection:
continue continue
self._attribute_to_object_map[name][value] = element self._attribute_to_object_map[name][value] = element
self._used_ids.add(element.id) self._used_ids.add(element.id)
def unmap_element(self, element: DatabaseObject): def unmap_element(self, element: DatabaseObject):
for name, value in element.indexing_values: for name, value in element.indexing_values:
if value is None: if value is None:
continue continue
if value in self._attribute_to_object_map[name]: if value in self._attribute_to_object_map[name]:
if element is self._attribute_to_object_map[name][value]: if element is self._attribute_to_object_map[name][value]:
try: try:
@ -62,7 +69,8 @@ class Collection:
except KeyError: except KeyError:
pass pass
def append(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> DatabaseObject: def append(self, element: DatabaseObject, merge_on_conflict: bool = True,
merge_into_existing: bool = True) -> AppendResult:
""" """
:param element: :param element:
:param merge_on_conflict: :param merge_on_conflict:
@ -77,41 +85,39 @@ class Collection:
for name, value in element.indexing_values: for name, value in element.indexing_values:
if value in self._attribute_to_object_map[name]: if value in self._attribute_to_object_map[name]:
existing_object = self._attribute_to_object_map[name][value] existing_object = self._attribute_to_object_map[name][value]
if not merge_on_conflict: if not merge_on_conflict:
return existing_object return AppendResult(True, existing_object)
# if the object does already exist # if the object does already exist
# thus merging and don't add it afterwards # thus merging and don't add it afterwards
if merge_into_existing: if merge_into_existing:
existing_object.merge(element) existing_object.merge(element)
# in case any relevant data has been added (e.g. it remaps the old object) # in case any relevant data has been added (e.g. it remaps the old object)
self.map_element(existing_object) self.map_element(existing_object)
return existing_object return AppendResult(True, existing_object)
element.merge(existing_object) element.merge(existing_object)
exists_at = self._data.index(existing_object) exists_at = self._data.index(existing_object)
self._data[exists_at] = element self._data[exists_at] = element
self.unmap_element(existing_object) self.unmap_element(existing_object)
self.map_element(element) self.map_element(element)
return element return AppendResult(True, existing_object)
self._data.append(element) self._data.append(element)
self.map_element(element) self.map_element(element)
return element
def append_is_already_in_collection(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> bool:
object_representing_the_data = self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing)
def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True): return AppendResult(False, element)
def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True,
merge_into_existing: bool = True):
for element in element_list: for element in element_list:
self.append(element, merge_on_conflict=merge_on_conflict) self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing)
def __iter__(self): def __iter__(self):
for element in self._data: for element in self.shallow_list:
yield element yield element
def __str__(self) -> str: def __str__(self) -> str:
@ -120,11 +126,22 @@ class Collection:
def __len__(self) -> int: def __len__(self) -> int:
return len(self._data) return len(self._data)
def __getitem__(self, item): def __getitem__(self, key):
if type(item) != int: if type(key) != int:
return ValueError("key needs to be an integer") return ValueError("key needs to be an integer")
return self._data[item] return self._data[key]
def __setitem__(self, key, value: DatabaseObject):
print(key, value)
if type(key) != int:
return ValueError("key needs to be an integer")
old_item = self._data[key]
self.unmap_element(old_item)
self.map_element(value)
self._data[key] = value
@property @property
def shallow_list(self) -> List[DatabaseObject]: def shallow_list(self) -> List[DatabaseObject]:

View File

@ -99,6 +99,7 @@ class DatabaseObject:
pass pass
class MainObject(DatabaseObject): class MainObject(DatabaseObject):
""" """
This is the parent class for all "main" data objects: This is the parent class for all "main" data objects:

View File

@ -1,11 +1,8 @@
from typing import Optional, Union, Type from typing import Optional, Union, Type, Dict
import requests import requests
import logging import logging
LOGGER = logging.getLogger("this shouldn't be used")
from ..utils import shared from ..utils import shared
from ..objects import ( from ..objects import (
Song, Song,
Source, Source,
@ -13,13 +10,15 @@ from ..objects import (
Artist, Artist,
Lyrics, Lyrics,
Target, Target,
MusicObject, DatabaseObject,
Options, Options,
SourcePages, SourcePages,
Collection, Collection,
Label Label
) )
LOGGER = logging.getLogger("this shouldn't be used")
class Page: class Page:
""" """
@ -139,7 +138,7 @@ class Page:
return Options() return Options()
@classmethod @classmethod
def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> MusicObject: def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject:
""" """
when a music object with laccing data is passed in, it returns when a music object with laccing data is passed in, it returns
the SAME object **(no copy)** with more detailed data. the SAME object **(no copy)** with more detailed data.
@ -156,21 +155,28 @@ class Page:
:return detailed_music_object: IT MODIFIES THE INPUT OBJ :return detailed_music_object: IT MODIFIES THE INPUT OBJ
""" """
new_music_object: MusicObject = type(music_object).__init__() new_music_object: DatabaseObject = type(music_object)()
source: Source source: Source
for source in music_object.source_collection: for source in music_object.source_collection:
new_music_object.merge(cls.fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) new_music_object.merge(cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level))
collections = {
Label: Collection(element_type=Label),
Artist: Collection(element_type=Artist),
Album: Collection(element_type=Album),
Song: Collection(element_type=Song)
}
cls._clean_music_object(new_music_object, collections)
music_object.merge(new_music_object) music_object.merge(new_music_object)
music_object.compile() # music_object.compile()
return music_object return music_object
@classmethod @classmethod
def fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1): def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1):
if obj_type == Artist: if obj_type == Artist:
return cls.fetch_artist_from_source(source=source, stop_at_level=stop_at_level) return cls.fetch_artist_from_source(source=source, stop_at_level=stop_at_level)
@ -183,6 +189,54 @@ class Page:
if obj_type == Label: if obj_type == Label:
return cls.fetch_label_from_source(source=source, stop_at_level=stop_at_level) return cls.fetch_label_from_source(source=source, stop_at_level=stop_at_level)
@classmethod
def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
if type(music_object) == Label:
return cls._clean_label(label=music_object, collections=collections)
if type(music_object) == Artist:
return cls._clean_artist(artist=music_object, collections=collections)
if type(music_object) == Album:
return cls._clean_album(album=music_object, collections=collections)
if type(music_object) == Song:
return cls._clean_song(song=music_object, collections=collections)
@classmethod
def _clean_collection(cls, collection: Collection, collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
if collection.element_type not in collection_dict:
return
for i, element in enumerate(collection):
r = collection_dict[collection.element_type].append(element)
if not r.was_in_collection:
cls._clean_music_object(r.current_element, collection_dict)
continue
collection[i] = r.current_element
cls._clean_music_object(r.current_element, collection_dict)
@classmethod
def _clean_label(cls, label: Label, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(label.current_artist_collection, collections)
cls._clean_collection(label.album_collection, collections)
@classmethod
def _clean_artist(cls, artist: Artist, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(artist.main_album_collection, collections)
cls._clean_collection(artist.feature_song_collection, collections)
cls._clean_collection(artist.label_collection, collections)
@classmethod
def _clean_album(cls, album: Album, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(album.label_collection, collections)
cls._clean_collection(album.song_collection, collections)
cls._clean_collection(album.artist_collection, collections)
@classmethod
def _clean_song(cls, song: Song, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
cls._clean_collection(song.album_collection, collections)
cls._clean_collection(song.feature_artist_collection, collections)
cls._clean_collection(song.main_artist_collection, collections)
@classmethod @classmethod
def fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song: def fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song:
return Song() return Song()
@ -195,6 +249,7 @@ class Page:
@classmethod @classmethod
def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
return Artist() return Artist()
def fetch_label_from_source(source: Source, stop_at_level: int = 1) -> Label: @classmethod
def fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label:
return Label() return Label()

View File

@ -9,7 +9,7 @@ from ..utils.shared import (
from .abstract import Page from .abstract import Page
from ..objects import ( from ..objects import (
MusicObject, DatabaseObject,
Artist, Artist,
Source, Source,
SourcePages, SourcePages,

View File

@ -14,7 +14,7 @@ from ..utils.shared import (
from .abstract import Page from .abstract import Page
from ..objects import ( from ..objects import (
MusicObject, DatabaseObject,
Artist, Artist,
Source, Source,
SourcePages, SourcePages,
@ -545,7 +545,7 @@ class Musify(Page):
)) ))
@classmethod @classmethod
def get_discography(cls, url: MusifyUrl, artist_name: str = None, flat=False) -> List[Album]: def get_discography(cls, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> List[Album]:
""" """
POST https://musify.club/artist/filteralbums POST https://musify.club/artist/filteralbums
ArtistID: 280348 ArtistID: 280348
@ -570,9 +570,9 @@ class Musify(Page):
for card_soup in soup.find_all("div", {"class": "card"}): for card_soup in soup.find_all("div", {"class": "card"}):
new_album: Album = cls.parse_album_card(card_soup, artist_name) new_album: Album = cls.parse_album_card(card_soup, artist_name)
album_source: Source album_source: Source
if not flat: if stop_at_level > 1:
for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE): for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE):
new_album.merge(cls.fetch_album_from_source(album_source)) new_album.merge(cls.fetch_album_from_source(album_source, stop_at_level=stop_at_level-1))
discography.append(new_album) discography.append(new_album)
@ -709,7 +709,7 @@ class Musify(Page):
) )
@classmethod @classmethod
def fetch_artist_from_source(cls, source: Source, flat: bool = False) -> Artist: def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
""" """
fetches artist from source fetches artist from source
@ -719,7 +719,7 @@ class Musify(Page):
Args: Args:
source (Source): the source to fetch source (Source): the source to fetch
flat (bool, optional): if it is false, every album from discograohy will be fetched. Defaults to False. stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False.
Returns: Returns:
Artist: the artist fetched Artist: the artist fetched
@ -851,7 +851,7 @@ class Musify(Page):
) )
@classmethod @classmethod
def fetch_album_from_source(cls, source: Source, flat: bool = False) -> Album: def fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album:
""" """
fetches album from source: fetches album from source:
eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188' eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188'
@ -861,8 +861,8 @@ class Musify(Page):
[] attributes [] attributes
[] ratings [] ratings
:param stop_at_level:
:param source: :param source:
:param flat:
:return: :return:
""" """
album = Album(title="Hi :)") album = Album(title="Hi :)")

View File

@ -22,9 +22,15 @@ def fetch_album():
"https://musify.club/release/linkin-park-hybrid-theory-2000-188")] "https://musify.club/release/linkin-park-hybrid-theory-2000-188")]
) )
album = Musify.fetch_details(album) album: objects.Album = Musify.fetch_details(album)
print(album.options) print(album.options)
song: objects.Song
for artist in album.artist_collection:
print(artist.id, artist.name)
for song in album.song_collection:
for artist in song.main_artist_collection:
print(artist.id, artist.name)
if __name__ == "__main__": if __name__ == "__main__":
fetch_album() fetch_album()