diff --git a/src/music_kraken/pages/html/musify/album_overview.html b/documentation/html/musify/album_overview.html
similarity index 100%
rename from src/music_kraken/pages/html/musify/album_overview.html
rename to documentation/html/musify/album_overview.html
diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py
index 03e31c6..d00eb2b 100644
--- a/src/music_kraken/__init__.py
+++ b/src/music_kraken/__init__.py
@@ -38,18 +38,18 @@ logging.getLogger("musicbrainzngs").setLevel(logging.WARNING)
musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeIIow2/music-downloader")
-def get_options_from_query(query: str) -> List[objects.MusicObject]:
+def get_options_from_query(query: str) -> List[objects.DatabaseObject]:
options = []
for MetadataPage in pages.MetadataPages:
options.extend(MetadataPage.search_by_query(query=query))
return options
-def get_options_from_option(option: objects.MusicObject) -> List[objects.MusicObject]:
+def get_options_from_option(option: objects.DatabaseObject) -> List[objects.DatabaseObject]:
for MetadataPage in pages.MetadataPages:
option = MetadataPage.fetch_details(option, flat=False)
return option.get_options()
-def print_options(options: List[objects.MusicObject]):
+def print_options(options: List[objects.DatabaseObject]):
print("\n".join([f"{str(j).zfill(2)}: {i.get_option_string()}" for j, i in enumerate(options)]))
def cli():
diff --git a/src/music_kraken/database/__init__.py b/src/music_kraken/database/__init__.py
index 5a22e6a..e69de29 100644
--- a/src/music_kraken/database/__init__.py
+++ b/src/music_kraken/database/__init__.py
@@ -1,19 +0,0 @@
-from . import database
-from .. import objects
-
-MusicObject = objects.MusicObject
-
-ID3Timestamp = objects.ID3Timestamp
-SourceTypes = objects.SourceTypes
-SourcePages = objects.SourcePages
-Song = objects.Song
-Source = objects.Source
-Target = objects.Target
-Lyrics = objects.Lyrics
-Album = objects.Album
-Artist = objects.Artist
-
-FormattedText = objects.FormattedText
-
-Database = database.Database
-# cache = temp_database.TempDatabase()
diff --git a/src/music_kraken/database/database.py b/src/music_kraken/database/database.py
index d20e793..0120a89 100644
--- a/src/music_kraken/database/database.py
+++ b/src/music_kraken/database/database.py
@@ -130,7 +130,7 @@ class Database:
print(model._meta.fields)
- def push(self, database_object: objects.MusicObject):
+ def push(self, database_object: objects.DatabaseObject):
"""
Adds a new music object to the database using the corresponding method from the `write` session.
When possible, rather use the `push_many` function.
@@ -153,7 +153,7 @@ class Database:
if isinstance(database_object, objects.Artist):
return writing_session.add_artist(database_object)
- def push_many(self, database_objects: List[objects.MusicObject]) -> None:
+ def push_many(self, database_objects: List[objects.DatabaseObject]) -> None:
"""
Adds a list of MusicObject instances to the database.
This function sends only needs one querry for each type of table added.
diff --git a/src/music_kraken/database/object_cache.py b/src/music_kraken/database/object_cache.py
index a9c924c..9251dcd 100644
--- a/src/music_kraken/database/object_cache.py
+++ b/src/music_kraken/database/object_cache.py
@@ -2,7 +2,7 @@ from collections import defaultdict
from typing import Dict, List, Optional
import weakref
-from src.music_kraken.objects import MusicObject
+from src.music_kraken.objects import DatabaseObject
"""
This is a cache for the objects, that et pulled out of the database.
@@ -32,14 +32,14 @@ class ObjectCache:
:method extent: Add a list of MusicObjects to the cache.
:method remove: Remove a MusicObject from the cache by its id.
:method get: Retrieve a MusicObject from the cache by its id. """
- object_to_id: Dict[str, MusicObject]
+ object_to_id: Dict[str, DatabaseObject]
weakref_map: Dict[weakref.ref, str]
def __init__(self) -> None:
self.object_to_id = dict()
self.weakref_map = defaultdict()
- def exists(self, music_object: MusicObject) -> bool:
+ def exists(self, music_object: DatabaseObject) -> bool:
"""
Check if a MusicObject with the same id already exists in the cache.
@@ -60,7 +60,7 @@ class ObjectCache:
data_id = self.weakref_map.pop(weakref_)
self.object_to_id.pop(data_id)
- def append(self, music_object: MusicObject) -> bool:
+ def append(self, music_object: DatabaseObject) -> bool:
"""
Add a MusicObject to the cache.
@@ -75,7 +75,7 @@ class ObjectCache:
return False
- def extent(self, music_object_list: List[MusicObject]):
+ def extent(self, music_object_list: List[DatabaseObject]):
"""
adjacent to the extent method of list, this appends n Object
"""
@@ -93,7 +93,7 @@ class ObjectCache:
self.weakref_map.pop(weakref.ref(data))
self.object_to_id.pop(_id)
- def __getitem__(self, item) -> Optional[MusicObject]:
+ def __getitem__(self, item) -> Optional[DatabaseObject]:
"""
this returns the data obj
:param item: the id of the music object
@@ -102,5 +102,5 @@ class ObjectCache:
return self.object_to_id.get(item)
- def get(self, _id: str) -> Optional[MusicObject]:
+ def get(self, _id: str) -> Optional[DatabaseObject]:
return self.__getitem__(_id)
diff --git a/src/music_kraken/objects/__init__.py b/src/music_kraken/objects/__init__.py
index 80e4064..fa7ba07 100644
--- a/src/music_kraken/objects/__init__.py
+++ b/src/music_kraken/objects/__init__.py
@@ -9,7 +9,7 @@ from . import (
collection
)
-MusicObject = parents.DatabaseObject
+DatabaseObject = parents.DatabaseObject
ID3Mapping = metadata.Mapping
ID3Timestamp = metadata.ID3Timestamp
diff --git a/src/music_kraken/objects/collection.py b/src/music_kraken/objects/collection.py
index bfb5b90..6009200 100644
--- a/src/music_kraken/objects/collection.py
+++ b/src/music_kraken/objects/collection.py
@@ -1,9 +1,16 @@
from typing import List, Iterable, Dict
from collections import defaultdict
+from dataclasses import dataclass
from .parents import DatabaseObject
+@dataclass
+class AppendResult:
+ was_in_collection: bool
+ current_element: DatabaseObject
+
+
class Collection:
"""
This a class for the iterables
@@ -14,12 +21,12 @@ class Collection:
_by_url: dict
_by_attribute: dict
- def __init__(self, data: List[DatabaseObject] = None, element_type = None, *args, **kwargs) -> None:
+ def __init__(self, data: List[DatabaseObject] = None, element_type=None, *args, **kwargs) -> None:
# Attribute needs to point to
self.element_type = element_type
-
+
self._data: List[DatabaseObject] = list()
-
+
"""
example of attribute_to_object_map
the song objects are references pointing to objects
@@ -34,7 +41,7 @@ class Collection:
"""
self._attribute_to_object_map: Dict[str, Dict[object, DatabaseObject]] = defaultdict(dict)
self._used_ids: set = set()
-
+
if data is not None:
self.extend(data, merge_on_conflict=True)
@@ -47,14 +54,14 @@ class Collection:
continue
self._attribute_to_object_map[name][value] = element
-
+
self._used_ids.add(element.id)
-
+
def unmap_element(self, element: DatabaseObject):
for name, value in element.indexing_values:
if value is None:
continue
-
+
if value in self._attribute_to_object_map[name]:
if element is self._attribute_to_object_map[name][value]:
try:
@@ -62,7 +69,8 @@ class Collection:
except KeyError:
pass
- def append(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> DatabaseObject:
+ def append(self, element: DatabaseObject, merge_on_conflict: bool = True,
+ merge_into_existing: bool = True) -> AppendResult:
"""
:param element:
:param merge_on_conflict:
@@ -77,41 +85,39 @@ class Collection:
for name, value in element.indexing_values:
if value in self._attribute_to_object_map[name]:
existing_object = self._attribute_to_object_map[name][value]
-
+
if not merge_on_conflict:
- return existing_object
-
+ return AppendResult(True, existing_object)
+
# if the object does already exist
# thus merging and don't add it afterwards
if merge_into_existing:
existing_object.merge(element)
# in case any relevant data has been added (e.g. it remaps the old object)
self.map_element(existing_object)
- return existing_object
-
+ return AppendResult(True, existing_object)
+
element.merge(existing_object)
-
+
exists_at = self._data.index(existing_object)
self._data[exists_at] = element
-
+
self.unmap_element(existing_object)
self.map_element(element)
- return element
+ return AppendResult(True, existing_object)
self._data.append(element)
self.map_element(element)
-
- return element
-
- def append_is_already_in_collection(self, element: DatabaseObject, merge_on_conflict: bool = True, merge_into_existing: bool = True) -> bool:
- object_representing_the_data = self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing)
- def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True):
+ return AppendResult(False, element)
+
+ def extend(self, element_list: Iterable[DatabaseObject], merge_on_conflict: bool = True,
+ merge_into_existing: bool = True):
for element in element_list:
- self.append(element, merge_on_conflict=merge_on_conflict)
+ self.append(element, merge_on_conflict=merge_on_conflict, merge_into_existing=merge_into_existing)
def __iter__(self):
- for element in self._data:
+ for element in self.shallow_list:
yield element
def __str__(self) -> str:
@@ -120,11 +126,22 @@ class Collection:
def __len__(self) -> int:
return len(self._data)
- def __getitem__(self, item):
- if type(item) != int:
+ def __getitem__(self, key):
+ if type(key) != int:
return ValueError("key needs to be an integer")
- return self._data[item]
+ return self._data[key]
+
+ def __setitem__(self, key, value: DatabaseObject):
+ print(key, value)
+ if type(key) != int:
+ return ValueError("key needs to be an integer")
+
+ old_item = self._data[key]
+ self.unmap_element(old_item)
+ self.map_element(value)
+
+ self._data[key] = value
@property
def shallow_list(self) -> List[DatabaseObject]:
diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py
index 855ba71..2152677 100644
--- a/src/music_kraken/objects/parents.py
+++ b/src/music_kraken/objects/parents.py
@@ -99,6 +99,7 @@ class DatabaseObject:
pass
+
class MainObject(DatabaseObject):
"""
This is the parent class for all "main" data objects:
diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py
index a5cae98..dc916e3 100644
--- a/src/music_kraken/pages/abstract.py
+++ b/src/music_kraken/pages/abstract.py
@@ -1,11 +1,8 @@
-from typing import Optional, Union, Type
+from typing import Optional, Union, Type, Dict
import requests
import logging
-LOGGER = logging.getLogger("this shouldn't be used")
-
from ..utils import shared
-
from ..objects import (
Song,
Source,
@@ -13,13 +10,15 @@ from ..objects import (
Artist,
Lyrics,
Target,
- MusicObject,
+ DatabaseObject,
Options,
SourcePages,
Collection,
Label
)
+LOGGER = logging.getLogger("this shouldn't be used")
+
class Page:
"""
@@ -139,7 +138,7 @@ class Page:
return Options()
@classmethod
- def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> MusicObject:
+ def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject:
"""
when a music object with laccing data is passed in, it returns
the SAME object **(no copy)** with more detailed data.
@@ -156,21 +155,28 @@ class Page:
:return detailed_music_object: IT MODIFIES THE INPUT OBJ
"""
- new_music_object: MusicObject = type(music_object).__init__()
-
+ new_music_object: DatabaseObject = type(music_object)()
+
source: Source
for source in music_object.source_collection:
- new_music_object.merge(cls.fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level))
+ new_music_object.merge(cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level))
+ collections = {
+ Label: Collection(element_type=Label),
+ Artist: Collection(element_type=Artist),
+ Album: Collection(element_type=Album),
+ Song: Collection(element_type=Song)
+ }
+ cls._clean_music_object(new_music_object, collections)
music_object.merge(new_music_object)
- music_object.compile()
+ # music_object.compile()
return music_object
@classmethod
- def fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1):
+ def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1):
if obj_type == Artist:
return cls.fetch_artist_from_source(source=source, stop_at_level=stop_at_level)
@@ -183,6 +189,54 @@ class Page:
if obj_type == Label:
return cls.fetch_label_from_source(source=source, stop_at_level=stop_at_level)
+ @classmethod
+ def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ if type(music_object) == Label:
+ return cls._clean_label(label=music_object, collections=collections)
+ if type(music_object) == Artist:
+ return cls._clean_artist(artist=music_object, collections=collections)
+ if type(music_object) == Album:
+ return cls._clean_album(album=music_object, collections=collections)
+ if type(music_object) == Song:
+ return cls._clean_song(song=music_object, collections=collections)
+
+ @classmethod
+ def _clean_collection(cls, collection: Collection, collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ if collection.element_type not in collection_dict:
+ return
+
+ for i, element in enumerate(collection):
+ r = collection_dict[collection.element_type].append(element)
+ if not r.was_in_collection:
+ cls._clean_music_object(r.current_element, collection_dict)
+ continue
+
+ collection[i] = r.current_element
+ cls._clean_music_object(r.current_element, collection_dict)
+
+ @classmethod
+ def _clean_label(cls, label: Label, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ cls._clean_collection(label.current_artist_collection, collections)
+ cls._clean_collection(label.album_collection, collections)
+
+ @classmethod
+ def _clean_artist(cls, artist: Artist, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ cls._clean_collection(artist.main_album_collection, collections)
+ cls._clean_collection(artist.feature_song_collection, collections)
+ cls._clean_collection(artist.label_collection, collections)
+
+ @classmethod
+ def _clean_album(cls, album: Album, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ cls._clean_collection(album.label_collection, collections)
+ cls._clean_collection(album.song_collection, collections)
+ cls._clean_collection(album.artist_collection, collections)
+
+ @classmethod
+ def _clean_song(cls, song: Song, collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
+ cls._clean_collection(song.album_collection, collections)
+ cls._clean_collection(song.feature_artist_collection, collections)
+ cls._clean_collection(song.main_artist_collection, collections)
+
@classmethod
def fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song:
return Song()
@@ -195,6 +249,7 @@ class Page:
@classmethod
def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
return Artist()
-
- def fetch_label_from_source(source: Source, stop_at_level: int = 1) -> Label:
+
+ @classmethod
+ def fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label:
return Label()
diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py
index f226303..987d6c9 100644
--- a/src/music_kraken/pages/encyclopaedia_metallum.py
+++ b/src/music_kraken/pages/encyclopaedia_metallum.py
@@ -9,7 +9,7 @@ from ..utils.shared import (
from .abstract import Page
from ..objects import (
- MusicObject,
+ DatabaseObject,
Artist,
Source,
SourcePages,
diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py
index e874ca7..a78ed93 100644
--- a/src/music_kraken/pages/musify.py
+++ b/src/music_kraken/pages/musify.py
@@ -14,7 +14,7 @@ from ..utils.shared import (
from .abstract import Page
from ..objects import (
- MusicObject,
+ DatabaseObject,
Artist,
Source,
SourcePages,
@@ -545,7 +545,7 @@ class Musify(Page):
))
@classmethod
- def get_discography(cls, url: MusifyUrl, artist_name: str = None, flat=False) -> List[Album]:
+ def get_discography(cls, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> List[Album]:
"""
POST https://musify.club/artist/filteralbums
ArtistID: 280348
@@ -570,9 +570,9 @@ class Musify(Page):
for card_soup in soup.find_all("div", {"class": "card"}):
new_album: Album = cls.parse_album_card(card_soup, artist_name)
album_source: Source
- if not flat:
+ if stop_at_level > 1:
for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE):
- new_album.merge(cls.fetch_album_from_source(album_source))
+ new_album.merge(cls.fetch_album_from_source(album_source, stop_at_level=stop_at_level-1))
discography.append(new_album)
@@ -709,7 +709,7 @@ class Musify(Page):
)
@classmethod
- def fetch_artist_from_source(cls, source: Source, flat: bool = False) -> Artist:
+ def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
"""
fetches artist from source
@@ -719,7 +719,7 @@ class Musify(Page):
Args:
source (Source): the source to fetch
- flat (bool, optional): if it is false, every album from discograohy will be fetched. Defaults to False.
+ stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False.
Returns:
Artist: the artist fetched
@@ -851,7 +851,7 @@ class Musify(Page):
)
@classmethod
- def fetch_album_from_source(cls, source: Source, flat: bool = False) -> Album:
+ def fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album:
"""
fetches album from source:
eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188'
@@ -861,8 +861,8 @@ class Musify(Page):
[] attributes
[] ratings
+ :param stop_at_level:
:param source:
- :param flat:
:return:
"""
album = Album(title="Hi :)")
diff --git a/src/musify_search.py b/src/musify_search.py
index 1dbdb68..63a23bd 100644
--- a/src/musify_search.py
+++ b/src/musify_search.py
@@ -22,9 +22,15 @@ def fetch_album():
"https://musify.club/release/linkin-park-hybrid-theory-2000-188")]
)
- album = Musify.fetch_details(album)
+ album: objects.Album = Musify.fetch_details(album)
print(album.options)
+ song: objects.Song
+ for artist in album.artist_collection:
+ print(artist.id, artist.name)
+ for song in album.song_collection:
+ for artist in song.main_artist_collection:
+ print(artist.id, artist.name)
if __name__ == "__main__":
fetch_album()