From d9c711a2f8201e5a45367f0ac75a592c355dde10 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 11:40:00 +0200 Subject: [PATCH 01/22] feat: added lru cache to unify function to speed up indexing --- development/actual_donwload.py | 2 +- music_kraken/download/page_attributes.py | 2 +- music_kraken/utils/shared.py | 2 +- music_kraken/utils/string_processing.py | 3 ++- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/development/actual_donwload.py b/development/actual_donwload.py index ed2a9de..abc5fe6 100644 --- a/development/actual_donwload.py +++ b/development/actual_donwload.py @@ -7,7 +7,7 @@ logging.getLogger().setLevel(logging.DEBUG) if __name__ == "__main__": commands = [ "s: #a Ghost Bath", - "4", + "d: 0", ] diff --git a/music_kraken/download/page_attributes.py b/music_kraken/download/page_attributes.py index e3881f6..ebd7423 100644 --- a/music_kraken/download/page_attributes.py +++ b/music_kraken/download/page_attributes.py @@ -14,7 +14,7 @@ from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, ALL_PAGES: Set[Type[Page]] = { - EncyclopaediaMetallum, + # EncyclopaediaMetallum, Musify, YoutubeMusic, Bandcamp diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index 6676393..d1c0bb7 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -17,7 +17,7 @@ DEBUG_LOGGING = DEBUG and True DEBUG_TRACE = DEBUG and True DEBUG_OBJECT_TRACE = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False -DEBUG_PAGES = DEBUG and False +DEBUG_PAGES = DEBUG and True DEBUG_DUMP = DEBUG and True if DEBUG: diff --git a/music_kraken/utils/string_processing.py b/music_kraken/utils/string_processing.py index 570d18e..15ff683 100644 --- a/music_kraken/utils/string_processing.py +++ b/music_kraken/utils/string_processing.py @@ -1,6 +1,7 @@ from typing import Tuple, Union from pathlib import Path import string +from functools import lru_cache from transliterate.exceptions import LanguageDetectionError from transliterate import translit @@ -11,7 +12,7 @@ COMMON_TITLE_APPENDIX_LIST: Tuple[str, ...] = ( "(official video)", ) - +@lru_cache def unify(string: str) -> str: """ returns a unified str, to make comparisons easy. -- 2.45.2 From 24a90f1cdfbd403145f120f904ea4ce08c92cd0a Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 11:43:21 +0200 Subject: [PATCH 02/22] feat: artist name in clean song title is optional --- music_kraken/utils/string_processing.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/music_kraken/utils/string_processing.py b/music_kraken/utils/string_processing.py index 15ff683..d21b367 100644 --- a/music_kraken/utils/string_processing.py +++ b/music_kraken/utils/string_processing.py @@ -1,4 +1,4 @@ -from typing import Tuple, Union +from typing import Tuple, Union, Optional from pathlib import Path import string from functools import lru_cache @@ -53,7 +53,7 @@ def fit_to_file_system(string: Union[str, Path]) -> Union[str, Path]: return fit_string(string) -def clean_song_title(raw_song_title: str, artist_name: str) -> str: +def clean_song_title(raw_song_title: str, artist_name: Optional[str] = None) -> str: """ This function cleans common naming "conventions" for non clean song titles, like the title of youtube videos @@ -65,19 +65,22 @@ def clean_song_title(raw_song_title: str, artist_name: str) -> str: - `song (prod. some producer)` """ raw_song_title = raw_song_title.strip() - artist_name = artist_name.strip() # Clean official Video appendix for dirty_appendix in COMMON_TITLE_APPENDIX_LIST: if raw_song_title.lower().endswith(dirty_appendix): raw_song_title = raw_song_title[:-len(dirty_appendix)].strip() - # Remove artist from the start of the title - if raw_song_title.lower().startswith(artist_name.lower()): - raw_song_title = raw_song_title[len(artist_name):].strip() + # everything that requires the artist name + if artist_name is not None: + artist_name = artist_name.strip() - if raw_song_title.startswith("-"): - raw_song_title = raw_song_title[1:].strip() + # Remove artist from the start of the title + if raw_song_title.lower().startswith(artist_name.lower()): + raw_song_title = raw_song_title[len(artist_name):].strip() + + if raw_song_title.startswith("-"): + raw_song_title = raw_song_title[1:].strip() return raw_song_title.strip() -- 2.45.2 From 1e62d371cdd7712158ec3a1b731d05bf9793e6ec Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 11:46:56 +0200 Subject: [PATCH 03/22] feat: cleaned bandcamp songs --- music_kraken/pages/bandcamp.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/bandcamp.py index f9a3a45..a583cd9 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/bandcamp.py @@ -22,6 +22,7 @@ from ..objects import ( ) from ..connection import Connection from ..utils.support_classes.download_result import DownloadResult +from ..utils.string_processing import clean_song_title from ..utils.config import main_settings, logging_settings from ..utils.shared import DEBUG @@ -114,7 +115,7 @@ class Bandcamp(Page): if object_type is BandcampTypes.SONG: return Song( - title=name.strip(), + title=clean_song_title(name, artist_name=data["band_name"]), source_list=source_list, main_artist_list=[ Artist( @@ -254,7 +255,7 @@ class Bandcamp(Page): def _parse_track_element(self, track: dict) -> Optional[Song]: return Song( - title=track["item"]["name"].strip(), + title=clean_song_title(track["item"]["name"]), source_list=[Source(self.SOURCE_TYPE, track["item"]["mainEntityOfPage"])], tracksort=int(track["position"]) ) @@ -337,7 +338,7 @@ class Bandcamp(Page): mp3_url = value song = Song( - title=data["name"].strip(), + title=clean_song_title(data["name"], artist_name=artist_data["name"]), source_list=[Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]), audio_url=mp3_url)], album_list=[Album( title=album_data["name"].strip(), -- 2.45.2 From 06acf22abbfb1a1391fede308502ac3a76fe6b8a Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:02:54 +0200 Subject: [PATCH 04/22] feat: improved the cleaning song title function to remove redundand brackets --- music_kraken/utils/string_processing.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/music_kraken/utils/string_processing.py b/music_kraken/utils/string_processing.py index d21b367..b591604 100644 --- a/music_kraken/utils/string_processing.py +++ b/music_kraken/utils/string_processing.py @@ -11,6 +11,9 @@ from pathvalidate import sanitize_filename COMMON_TITLE_APPENDIX_LIST: Tuple[str, ...] = ( "(official video)", ) +OPEN_BRACKETS = "([" +CLOSE_BRACKETS = ")]" +DISALLOWED_SUBSTRING_IN_BRACKETS = ("official", "video", "audio", "lyrics", "prod", "remix", "ft", "feat", "ft.", "feat.") @lru_cache def unify(string: str) -> str: @@ -71,6 +74,26 @@ def clean_song_title(raw_song_title: str, artist_name: Optional[str] = None) -> if raw_song_title.lower().endswith(dirty_appendix): raw_song_title = raw_song_title[:-len(dirty_appendix)].strip() + # remove brackets and their content if they contain disallowed substrings + for open_bracket, close_bracket in zip(OPEN_BRACKETS, CLOSE_BRACKETS): + start = 0 + + while True: + try: + open_bracket_index = raw_song_title.index(open_bracket, start) + except ValueError: + break + try: + close_bracket_index = raw_song_title.index(close_bracket, open_bracket_index + 1) + except ValueError: + break + + substring = raw_song_title[open_bracket_index + 1:close_bracket_index] + if any(disallowed_substring in substring for disallowed_substring in DISALLOWED_SUBSTRING_IN_BRACKETS): + raw_song_title = raw_song_title[:open_bracket_index] + raw_song_title[close_bracket_index + 1:] + else: + start = close_bracket_index + 1 + # everything that requires the artist name if artist_name is not None: artist_name = artist_name.strip() -- 2.45.2 From 3c5bbc19af167a34e7e9067ba04432e271367e92 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:04:13 +0200 Subject: [PATCH 05/22] feat: some slight performance improvements --- music_kraken/utils/string_processing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/music_kraken/utils/string_processing.py b/music_kraken/utils/string_processing.py index b591604..00b6024 100644 --- a/music_kraken/utils/string_processing.py +++ b/music_kraken/utils/string_processing.py @@ -56,6 +56,7 @@ def fit_to_file_system(string: Union[str, Path]) -> Union[str, Path]: return fit_string(string) +@lru_cache(maxsize=128) def clean_song_title(raw_song_title: str, artist_name: Optional[str] = None) -> str: """ This function cleans common naming "conventions" for non clean song titles, like the title of youtube videos @@ -76,6 +77,9 @@ def clean_song_title(raw_song_title: str, artist_name: Optional[str] = None) -> # remove brackets and their content if they contain disallowed substrings for open_bracket, close_bracket in zip(OPEN_BRACKETS, CLOSE_BRACKETS): + if open_bracket not in raw_song_title or close_bracket not in raw_song_title: + continue + start = 0 while True: -- 2.45.2 From b933c6ac1418309a392a608e5af75472acc6f320 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:17:08 +0200 Subject: [PATCH 06/22] feat: improved the lyrics support for bandcamp --- development/actual_donwload.py | 3 ++- music_kraken/objects/formatted_text.py | 10 +++++++++- music_kraken/pages/bandcamp.py | 9 ++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/development/actual_donwload.py b/development/actual_donwload.py index abc5fe6..ff96706 100644 --- a/development/actual_donwload.py +++ b/development/actual_donwload.py @@ -7,7 +7,8 @@ logging.getLogger().setLevel(logging.DEBUG) if __name__ == "__main__": commands = [ "s: #a Ghost Bath", - "d: 0", + "0", + "1", ] diff --git a/music_kraken/objects/formatted_text.py b/music_kraken/objects/formatted_text.py index c7ee042..6d95389 100644 --- a/music_kraken/objects/formatted_text.py +++ b/music_kraken/objects/formatted_text.py @@ -1,18 +1,26 @@ import mistune import html2markdown + +def plain_to_markdown(plain: str) -> str: + return plain.replace("\n", " \n") + + class FormattedText: html = "" def __init__( self, markdown: str = None, - html: str = None + html: str = None, + plain: str = None, ) -> None: if html is not None: self.html = html elif markdown is not None: self.html = mistune.markdown(markdown) + elif plain is not None: + self.html = mistune.markdown(plain_to_markdown(plain)) @property def is_empty(self) -> bool: diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/bandcamp.py index a583cd9..f63e30e 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/bandcamp.py @@ -18,7 +18,8 @@ from ..objects import ( Contact, ID3Timestamp, Lyrics, - FormattedText + FormattedText, + Artwork, ) from ..connection import Connection from ..utils.support_classes.download_result import DownloadResult @@ -254,6 +255,12 @@ class Bandcamp(Page): return artist def _parse_track_element(self, track: dict) -> Optional[Song]: + lyrics_list: List[Lyrics] = [] + + _lyrics: Optional[str] = track.get("item", {}).get("recordingOf", {}).get("lyrics", {}).get("text") + if _lyrics is not None: + lyrics_list.append(Lyrics(text=FormattedText(plain=_lyrics))) + return Song( title=clean_song_title(track["item"]["name"]), source_list=[Source(self.SOURCE_TYPE, track["item"]["mainEntityOfPage"])], -- 2.45.2 From e20b14a9df242cefc390f35d2cccd76f285a8545 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:37:14 +0200 Subject: [PATCH 07/22] feat: added fetching artworks to bandcamp --- development/actual_donwload.py | 2 +- music_kraken/pages/bandcamp.py | 27 ++++++++++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/development/actual_donwload.py b/development/actual_donwload.py index ff96706..333da4f 100644 --- a/development/actual_donwload.py +++ b/development/actual_donwload.py @@ -8,7 +8,7 @@ if __name__ == "__main__": commands = [ "s: #a Ghost Bath", "0", - "1", + "d: 1", ] diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/bandcamp.py index f63e30e..a670026 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/bandcamp.py @@ -254,7 +254,7 @@ class Bandcamp(Page): artist.source_collection.append(source) return artist - def _parse_track_element(self, track: dict) -> Optional[Song]: + def _parse_track_element(self, track: dict, artwork: Artwork) -> Optional[Song]: lyrics_list: List[Lyrics] = [] _lyrics: Optional[str] = track.get("item", {}).get("recordingOf", {}).get("lyrics", {}).get("text") @@ -264,7 +264,8 @@ class Bandcamp(Page): return Song( title=clean_song_title(track["item"]["name"]), source_list=[Source(self.SOURCE_TYPE, track["item"]["mainEntityOfPage"])], - tracksort=int(track["position"]) + tracksort=int(track["position"]), + artwork=artwork, ) def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: @@ -297,12 +298,32 @@ class Bandcamp(Page): )] ) + artwork: Artwork = Artwork() + + def _get_artwork_url(_data: dict) -> Optional[str]: + if "image" in _data: + return _data["image"] + for _property in _data.get("additionalProperty", []): + if _property.get("name") == "art_id": + return f"https://f4.bcbits.com/img/a{_property.get('value')}_2.jpg" + + _artwork_url = _get_artwork_url(data) + if _artwork_url is not None: + artwork.append(url=_artwork_url, width=350, height=350) + else: + for album_release in data.get("albumRelease", []): + _artwork_url = _get_artwork_url(album_release) + if _artwork_url is not None: + artwork.append(url=_artwork_url, width=350, height=350) + break + + for i, track_json in enumerate(data.get("track", {}).get("itemListElement", [])): if DEBUG: dump_to_file(f"album_track_{i}.json", json.dumps(track_json), is_json=True, exit_after_dump=False) try: - album.song_collection.append(self._parse_track_element(track_json)) + album.song_collection.append(self._parse_track_element(track_json, artwork=artwork)) except KeyError: continue -- 2.45.2 From 919a99885c572c52c7f10dcdfaf9829515e8f518 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:47:36 +0200 Subject: [PATCH 08/22] feat: disabled debugging pages --- music_kraken/utils/shared.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index d1c0bb7..6676393 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -17,7 +17,7 @@ DEBUG_LOGGING = DEBUG and True DEBUG_TRACE = DEBUG and True DEBUG_OBJECT_TRACE = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False -DEBUG_PAGES = DEBUG and True +DEBUG_PAGES = DEBUG and False DEBUG_DUMP = DEBUG and True if DEBUG: -- 2.45.2 From 06ffae06a6fc356a41f9f67d534ec3fbd768d095 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 12:57:34 +0200 Subject: [PATCH 09/22] fix: lyrics should be embedded in the markdown format rather than html --- music_kraken/objects/lyrics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music_kraken/objects/lyrics.py b/music_kraken/objects/lyrics.py index 09f7a3a..65d550d 100644 --- a/music_kraken/objects/lyrics.py +++ b/music_kraken/objects/lyrics.py @@ -34,6 +34,6 @@ class Lyrics(OuterProxy): @property def metadata(self) -> Metadata: return Metadata({ - id3Mapping.UNSYNCED_LYRICS: [self.text.html] + id3Mapping.UNSYNCED_LYRICS: [self.text.markdown] }) -- 2.45.2 From 301ff82bcf2efdc176aac1a601a153cd3223b4ae Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 13:37:12 +0200 Subject: [PATCH 10/22] feat: implemented the merging from where it has been fetched from --- music_kraken/objects/parents.py | 8 +++++--- music_kraken/pages/bandcamp.py | 11 ++++------- music_kraken/utils/shared.py | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/music_kraken/objects/parents.py b/music_kraken/objects/parents.py index 7b71269..53bc5bc 100644 --- a/music_kraken/objects/parents.py +++ b/music_kraken/objects/parents.py @@ -32,6 +32,7 @@ class InnerData: def __init__(self, object_type, **kwargs): self._refers_to_instances = set() + self._fetched_from: dict = {} # collection : collection that is a collection of self self._is_collection_child: Dict[Collection, Collection] = {} @@ -52,6 +53,8 @@ class InnerData: :return: """ + self._fetched_from.update(__other._fetched_from) + for key, value in __other.__dict__.copy().items(): # just set the other value if self doesn't already have it if key not in self.__dict__ or (key in self.__dict__ and self.__dict__[key] == self._default_values.get(key)): @@ -109,7 +112,6 @@ class OuterProxy: del kwargs[name] - self._fetched_from: dict = {} self._inner: InnerData = InnerData(type(self), **kwargs) self._inner._refers_to_instances.add(self) @@ -220,13 +222,13 @@ class OuterProxy: def mark_as_fetched(self, *url_hash_list: List[str]): for url_hash in url_hash_list: - self._fetched_from[url_hash] = { + self._inner._fetched_from[url_hash] = { "time": get_unix_time(), "url": url_hash, } def already_fetched_from(self, url_hash: str) -> bool: - res = self._fetched_from.get(url_hash, None) + res = self._inner._fetched_from.get(url_hash, None) if res is None: return False diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/bandcamp.py index a670026..52142eb 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/bandcamp.py @@ -352,10 +352,9 @@ class Bandcamp(Page): if len(other_data_list) > 0: other_data = json.loads(other_data_list[0]["data-tralbum"]) - if DEBUG: - dump_to_file("bandcamp_song_data.json", data_container.text, is_json=True, exit_after_dump=False) - dump_to_file("bandcamp_song_data_other.json", json.dumps(other_data), is_json=True, exit_after_dump=False) - dump_to_file("bandcamp_song_page.html", r.text, exit_after_dump=False) + dump_to_file("bandcamp_song_data.json", data_container.text, is_json=True, exit_after_dump=False) + dump_to_file("bandcamp_song_data_other.json", json.dumps(other_data), is_json=True, exit_after_dump=False) + dump_to_file("bandcamp_song_page.html", r.text, exit_after_dump=False) data = json.loads(data_container.text) album_data = data["inAlbum"] @@ -367,7 +366,7 @@ class Bandcamp(Page): song = Song( title=clean_song_title(data["name"], artist_name=artist_data["name"]), - source_list=[Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]), audio_url=mp3_url)], + source_list=[source, Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]), audio_url=mp3_url)], album_list=[Album( title=album_data["name"].strip(), date=ID3Timestamp.strptime(data["datePublished"], "%d %b %Y %H:%M:%S %Z"), @@ -380,8 +379,6 @@ class Bandcamp(Page): lyrics_list=self._fetch_lyrics(soup=soup) ) - song.source_collection.append(source) - return song def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index 6676393..b3f30e5 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -13,7 +13,7 @@ if not load_dotenv(Path(__file__).parent.parent.parent / ".env"): __stage__ = os.getenv("STAGE", "prod") DEBUG = (__stage__ == "dev") and True -DEBUG_LOGGING = DEBUG and True +DEBUG_LOGGING = DEBUG and False DEBUG_TRACE = DEBUG and True DEBUG_OBJECT_TRACE = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False -- 2.45.2 From 81708ba100d47dedd9c73cbccdde2adf4bbb1083 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 13:51:08 +0200 Subject: [PATCH 11/22] feat: switched to a more readable markdown converter --- music_kraken/objects/formatted_text.py | 4 ++-- music_kraken/pages/bandcamp.py | 1 - pyproject.toml | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/music_kraken/objects/formatted_text.py b/music_kraken/objects/formatted_text.py index 6d95389..8c01d39 100644 --- a/music_kraken/objects/formatted_text.py +++ b/music_kraken/objects/formatted_text.py @@ -1,5 +1,5 @@ import mistune -import html2markdown +from markdownify import markdownify as md def plain_to_markdown(plain: str) -> str: @@ -36,7 +36,7 @@ class FormattedText: @property def markdown(self) -> str: - return html2markdown.convert(self.html) + return md(self.html) def __str__(self) -> str: return self.markdown diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/bandcamp.py index 52142eb..1088be0 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/bandcamp.py @@ -333,7 +333,6 @@ class Bandcamp(Page): def _fetch_lyrics(self, soup: BeautifulSoup) -> List[Lyrics]: track_lyrics = soup.find("div", {"class": "lyricsText"}) if track_lyrics: - self.LOGGER.debug(" Lyrics retrieved..") return [Lyrics(text=FormattedText(html=track_lyrics.prettify()))] return [] diff --git a/pyproject.toml b/pyproject.toml index 5fab835..16fac20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dependencies = [ "rich~=13.7.1", "mistune~=3.0.2", + "markdownify~=0.12.1", "html2markdown~=0.1.7", "jellyfish~=0.9.0", "transliterate~=1.10.2", -- 2.45.2 From 29770825a44ffd42021e5f7770247e457949a538 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 13:54:08 +0200 Subject: [PATCH 12/22] fix: unified wrong attribute in song, causing many duplicates --- music_kraken/objects/song.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index e682fe1..759071c 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -126,7 +126,7 @@ class Song(Base): def indexing_values(self) -> List[Tuple[str, object]]: return [ ('id', self.id), - ('title', unify(self.unified_title)), + ('title', unify(self.title)), ('isrc', self.isrc), *[('url', source.url) for source in self.source_collection] ] -- 2.45.2 From be09562632d3ca4804d49e035a5fcb2c815ad412 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 14:05:05 +0200 Subject: [PATCH 13/22] feat: stripped whitespaces from lyrics --- music_kraken/objects/formatted_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music_kraken/objects/formatted_text.py b/music_kraken/objects/formatted_text.py index 8c01d39..b1891b6 100644 --- a/music_kraken/objects/formatted_text.py +++ b/music_kraken/objects/formatted_text.py @@ -36,7 +36,7 @@ class FormattedText: @property def markdown(self) -> str: - return md(self.html) + return md(self.html).strip() def __str__(self) -> str: return self.markdown -- 2.45.2 From 1735ff4e1d5f99438272bec38b422558d83d9055 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 14:16:31 +0200 Subject: [PATCH 14/22] feat: removed redundand commands from song --- music_kraken/objects/song.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index 759071c..9a068bf 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -86,11 +86,6 @@ class Song(Base): TITEL = "title" def __init_collections__(self) -> None: - """ - self.album_collection.contain_given_in_attribute = { - "artist_collection": self.main_artist_collection, - } - """ self.album_collection.sync_on_append = { "artist_collection": self.main_artist_collection, } @@ -347,7 +342,6 @@ class Album(Base): tracksort_map[i] = existing_list.pop(0) tracksort_map[i].tracksort = i - def compile(self, merge_into: bool = False): """ compiles the recursive structures, -- 2.45.2 From b4c73d56a756747bd13f053bab20c29476ecf8d8 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 14:43:09 +0200 Subject: [PATCH 15/22] feat: improved tracing --- music_kraken/objects/parents.py | 2 +- music_kraken/utils/__init__.py | 5 +++-- music_kraken/utils/shared.py | 5 +++-- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/music_kraken/objects/parents.py b/music_kraken/objects/parents.py index 53bc5bc..4e87cfb 100644 --- a/music_kraken/objects/parents.py +++ b/music_kraken/objects/parents.py @@ -194,7 +194,7 @@ class OuterProxy: if len(b._inner._refers_to_instances) > len(a._inner._refers_to_instances): a, b = b, a - object_trace(f"merging {type(a).__name__} [{a.title_string} | {a.id}] with {type(b).__name__} [{b.title_string} | {b.id}] called by [{' | '.join(f'{s.function} {Path(s.filename).name}:{str(s.lineno)}' for s in inspect.stack()[1:5])}]") + object_trace(f"merging {type(a).__name__} [{a.title_string} | {a.id}] with {type(b).__name__} [{b.title_string} | {b.id}]") for collection, child_collection in b._inner._is_collection_child.items(): try: diff --git a/music_kraken/utils/__init__.py b/music_kraken/utils/__init__.py index 96b4379..b9715e3 100644 --- a/music_kraken/utils/__init__.py +++ b/music_kraken/utils/__init__.py @@ -3,7 +3,7 @@ from pathlib import Path import json import logging -from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP, DEBUG_TRACE, DEBUG_OBJECT_TRACE +from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP, DEBUG_TRACE, DEBUG_OBJECT_TRACE, DEBUG_OBJECT_TRACE_CALLSTACK from .config import config, read_config, write_config from .enums.colors import BColors from .path_manager import LOCATIONS @@ -56,7 +56,8 @@ def object_trace(obj): if not DEBUG_OBJECT_TRACE: return - output("object: " + str(obj), BColors.GREY) + appendix = f" called by [{' | '.join(f'{s.function} {Path(s.filename).name}:{str(s.lineno)}' for s in inspect.stack()[1:5])}]" if DEBUG_OBJECT_TRACE_CALLSTACK else "" + output("object: " + str(obj) + appendix, BColors.GREY) """ diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index b3f30e5..638e6a4 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -15,10 +15,11 @@ __stage__ = os.getenv("STAGE", "prod") DEBUG = (__stage__ == "dev") and True DEBUG_LOGGING = DEBUG and False DEBUG_TRACE = DEBUG and True -DEBUG_OBJECT_TRACE = DEBUG and False +DEBUG_OBJECT_TRACE = DEBUG and True +DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_PAGES = DEBUG and False -DEBUG_DUMP = DEBUG and True +DEBUG_DUMP = DEBUG and False if DEBUG: print("DEBUG ACTIVE") -- 2.45.2 From a998e52cd94b7f13569fdb7260cfb8a065a3206a Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 17:45:49 +0200 Subject: [PATCH 16/22] fix: syncing --- development/objects_collection.py | 97 ++++++------------------------ music_kraken/objects/collection.py | 25 ++++++-- music_kraken/objects/parents.py | 6 ++ music_kraken/objects/song.py | 1 + music_kraken/utils/__init__.py | 1 + music_kraken/utils/shared.py | 2 +- tests/test_collection.py | 46 +++++++++++++- 7 files changed, 91 insertions(+), 87 deletions(-) diff --git a/development/objects_collection.py b/development/objects_collection.py index 9e147f5..c7526dc 100644 --- a/development/objects_collection.py +++ b/development/objects_collection.py @@ -2,91 +2,30 @@ import music_kraken from music_kraken.objects import Song, Album, Artist, Collection if __name__ == "__main__": - artist: Artist = Artist( - name="artist", - main_album_list=[ - Album( - title="album", - song_list=[ - Song( - title="song", - album_list=[ - Album( - title="album", - albumsort=123, - main_artist=Artist(name="artist"), - ), - ], - ), - Song( - title="other_song", - album_list=[ - Album(title="album", albumsort=423), - ], - ), - ] - ), - Album(title="album", barcode="1234567890123"), + album_1 = Album( + title="album", + song_list=[ + Song(title="song", main_artist_list=[Artist(name="artist")]), + ], + artist_list=[ + Artist(name="artist 3"), ] ) - - other_artist: Artist = Artist( - name="artist", - main_album_list=[ - Album( - title="album", - song_list=[ - Song( - title="song", - album_list=[ - Album( - title="album", - albumsort=123, - main_artist=Artist(name="other_artist"), - ), - ], - ), - Song( - title="other_song", - album_list=[ - Album(title="album", albumsort=423), - ], - ), - ] - ), - Album(title="album", barcode="1234567890123"), + album_2 = Album( + title="album", + song_list=[ + Song(title="song", main_artist_list=[Artist(name="artist 2")]), + ], + artist_list=[ + Artist(name="artist"), ] ) - artist.merge(other_artist) + album_1.merge(album_2) - a = artist.main_album_collection[0] - b = a.song_collection[0].album_collection[0] - c = a.song_collection[1].album_collection[0] - d = b.song_collection[0].album_collection[0] - e = d.song_collection[0].album_collection[0] - f = e.song_collection[0].album_collection[0] - g = f.song_collection[0].album_collection[0] - - print(a.id, a.title, a.barcode, a.albumsort) - print(b.id, b.title, b.barcode, b.albumsort) - print(c.id, c.title, c.barcode, c.albumsort) - print(d.id, d.title, d.barcode, d.albumsort) - print(e.id, e.title, e.barcode, e.albumsort) - print(f.id, f.title, f.barcode, f.albumsort) - print(g.id, g.title, g.barcode, g.albumsort) print() + print(album_1.artist_collection.data) - d.title = "new_title" - - print(a.id, a.title, a.barcode, a.albumsort) - print(b.id, b.title, b.barcode, b.albumsort) - print(c.id, c.title, c.barcode, c.albumsort) - print(d.id, d.title, d.barcode, d.albumsort) - print(e.id, e.title, e.barcode, e.albumsort) - print(f.id, f.title, f.barcode, f.albumsort) - print(g.id, g.title, g.barcode, g.albumsort) - print() - - print(artist.main_album_collection._indexed_values) \ No newline at end of file + print(id(album_1.artist_collection), id(album_2.artist_collection)) + print(id(album_1.song_collection[0].main_artist_collection), id(album_2.song_collection[0].main_artist_collection)) \ No newline at end of file diff --git a/music_kraken/objects/collection.py b/music_kraken/objects/collection.py index 1d62116..c13a294 100644 --- a/music_kraken/objects/collection.py +++ b/music_kraken/objects/collection.py @@ -3,6 +3,7 @@ from __future__ import annotations from collections import defaultdict from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union from .parents import OuterProxy +from ..utils import object_trace T = TypeVar('T', bound=OuterProxy) @@ -25,6 +26,8 @@ class Collection(Generic[T]): contain_attribute_in_given: Dict[str, Collection] = None, append_object_to_attribute: Dict[str, T] = None ) -> None: + self._collection_for: dict = dict() + self._contains_ids = set() self._data = [] @@ -44,6 +47,9 @@ class Collection(Generic[T]): self.extend(data) + def __repr__(self) -> str: + return f"Collection({id(self)})" + def _map_element(self, __object: T, from_map: bool = False): self._contains_ids.add(__object.id) @@ -224,16 +230,25 @@ class Collection(Generic[T]): append_to._data.append(__object) append_to._map_element(__object) - # only modify collections if the object actually has been appended for collection_attribute, child_collection in self.contain_given_in_attribute.items(): __object.__getattribute__(collection_attribute).contain_collection_inside(child_collection, __object) for attribute, new_object in self.append_object_to_attribute.items(): __object.__getattribute__(attribute).append(new_object) - - for attribute, collection in self.sync_on_append.items(): - collection.extend(__object.__getattribute__(attribute)) - __object.__setattr__(attribute, collection) + + # only modify collections if the object actually has been appended + for attribute, a in self.sync_on_append.items(): + b = __object.__getattribute__(attribute) + object_trace(f"Syncing [{a}{id(a)}] = [{b}{id(b)}]") + + data_to_extend = b.data + + a._collection_for.update(b._collection_for) + for synced_with, key in b._collection_for.items(): + synced_with.__setattr__(key, a) + + a.extend(data_to_extend) + else: # merge only if the two objects are not the same diff --git a/music_kraken/objects/parents.py b/music_kraken/objects/parents.py index 4e87cfb..a0bad1e 100644 --- a/music_kraken/objects/parents.py +++ b/music_kraken/objects/parents.py @@ -44,8 +44,13 @@ class InnerData: self._default_values[name] = factory() for key, value in kwargs.items(): + if hasattr(value, "__is_collection__"): + value._collection_for[self] = key self.__setattr__(key, value) + def __hash__(self): + return self.id + def __merge__(self, __other: InnerData, override: bool = False): """ :param __other: @@ -116,6 +121,7 @@ class OuterProxy: self._inner._refers_to_instances.add(self) object_trace(f"creating {type(self).__name__} [{self.title_string}]") + self.__init_collections__() for name, data_list in collection_data.items(): diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index 9a068bf..93b39ba 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -204,6 +204,7 @@ class Album(Base): notes: FormattedText source_collection: SourceCollection + artist_collection: Collection[Artist] song_collection: Collection[Song] label_collection: Collection[Label] diff --git a/music_kraken/utils/__init__.py b/music_kraken/utils/__init__.py index b9715e3..2b63305 100644 --- a/music_kraken/utils/__init__.py +++ b/music_kraken/utils/__init__.py @@ -2,6 +2,7 @@ from datetime import datetime from pathlib import Path import json import logging +import inspect from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP, DEBUG_TRACE, DEBUG_OBJECT_TRACE, DEBUG_OBJECT_TRACE_CALLSTACK from .config import config, read_config, write_config diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index 638e6a4..6397fd3 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -16,7 +16,7 @@ DEBUG = (__stage__ == "dev") and True DEBUG_LOGGING = DEBUG and False DEBUG_TRACE = DEBUG and True DEBUG_OBJECT_TRACE = DEBUG and True -DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG and False +DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG and True DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_PAGES = DEBUG and False DEBUG_DUMP = DEBUG and False diff --git a/tests/test_collection.py b/tests/test_collection.py index 85b1941..7a9e3ca 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -70,7 +70,50 @@ class TestCollection(unittest.TestCase): self.assertTrue(a.name == b.name == c.name == d.name == "artist") self.assertTrue(a.country == b.country == c.country == d.country) - """ + def test_artist_artist_relation(self): + artist = Artist( + name="artist", + main_album_list=[ + Album( + title="album", + song_list=[ + Song(title="song"), + ], + artist_list=[ + Artist(name="artist"), + ] + ) + ] + ) + + self.assertTrue(artist.id == artist.main_album_collection[0].song_collection[0].artist_collection[0].id) + + def test_artist_collection_sync(self): + album_1 = Album( + title="album", + song_list=[ + Song(title="song", main_artist_list=[Artist(name="artist")]), + ], + artist_list=[ + Artist(name="artist"), + ] + ) + + album_2 = Album( + title="album", + song_list=[ + Song(title="song", main_artist_list=[Artist(name="artist")]), + ], + artist_list=[ + Artist(name="artist"), + ] + ) + + album_1.merge(album_2) + + self.assertTrue(id(album_1.artist_collection) == id(album_1.artist_collection)) + self.assertTrue(id(album_1.song_collection[0].main_artist_collection) == id(album_1.song_collection[0].main_artist_collection)) + def test_song_artist_relations(self): a = self.complicated_object() b = a.main_album_collection[0].song_collection[0].main_artist_collection[0] @@ -80,7 +123,6 @@ class TestCollection(unittest.TestCase): self.assertTrue(a.id == b.id == c.id == d.id) self.assertTrue(a.name == b.name == c.name == d.name == "artist") self.assertTrue(a.country == b.country == c.country == d.country) - """ if __name__ == "__main__": unittest.main() -- 2.45.2 From 312e57d82ef2e00589098fbe3bd64f9e1bf0c24b Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 19 Apr 2024 17:48:42 +0200 Subject: [PATCH 17/22] feat: progress --- tests/test_collection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_collection.py b/tests/test_collection.py index 7a9e3ca..12fe2f2 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -111,8 +111,7 @@ class TestCollection(unittest.TestCase): album_1.merge(album_2) - self.assertTrue(id(album_1.artist_collection) == id(album_1.artist_collection)) - self.assertTrue(id(album_1.song_collection[0].main_artist_collection) == id(album_1.song_collection[0].main_artist_collection)) + self.assertTrue(id(album_1.artist_collection) == id(album_1.artist_collection) == id(album_1.song_collection[0].main_artist_collection) == id(album_1.song_collection[0].main_artist_collection)) def test_song_artist_relations(self): a = self.complicated_object() -- 2.45.2 From fa723d774705116f78c3a9d94371f9e8cba2def0 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 23 Apr 2024 09:19:06 +0200 Subject: [PATCH 18/22] feat: removed redundand collection functions --- development/objects_collection.py | 2 +- music_kraken/objects/collection.py | 98 +----------------------------- music_kraken/objects/song.py | 1 - 3 files changed, 2 insertions(+), 99 deletions(-) diff --git a/development/objects_collection.py b/development/objects_collection.py index c7526dc..642bb18 100644 --- a/development/objects_collection.py +++ b/development/objects_collection.py @@ -25,7 +25,7 @@ if __name__ == "__main__": album_1.merge(album_2) print() - print(album_1.artist_collection.data) + print(*(f"{a.title_string} ; {a.id}" for a in album_1.artist_collection.data), sep=" | ") print(id(album_1.artist_collection), id(album_2.artist_collection)) print(id(album_1.song_collection[0].main_artist_collection), id(album_2.song_collection[0].main_artist_collection)) \ No newline at end of file diff --git a/music_kraken/objects/collection.py b/music_kraken/objects/collection.py index c13a294..6b59ac9 100644 --- a/music_kraken/objects/collection.py +++ b/music_kraken/objects/collection.py @@ -76,106 +76,10 @@ class Collection(Generic[T]): del self._id_to_index_values[obj_id] - def _contained_in_self(self, __object: T) -> bool: - if __object.id in self._contains_ids: - return True - - for name, value in __object.indexing_values: - if value is None: - continue - if value == self._indexed_values[name]: - return True - return False - - def _contained_in_sub(self, __object: T, break_at_first: bool = True) -> List[Collection]: - """ - Gets the collection this object is found in, if it is found in any. - - :param __object: - :param break_at_first: - :return: - """ - results = [] - - if self._contained_in_self(__object): - return [self] - - for collection in self.children: - results.extend(collection._contained_in_sub(__object, break_at_first=break_at_first)) - - if break_at_first: - return results - - return results - - def _get_root_collections(self) -> List[Collection]: - if not len(self.parents): - return [self] - - root_collections = [] - for upper_collection in self.parents: - root_collections.extend(upper_collection._get_root_collections()) - return root_collections - @property - def _is_root(self) -> bool: + def is_root(self) -> bool: return len(self.parents) <= 0 - def _get_parents_of_multiple_contained_children(self, __object: T): - results = [] - if len(self.children) < 2 or self._contained_in_self(__object): - return results - - count = 0 - - for collection in self.children: - sub_results = collection._get_parents_of_multiple_contained_children(__object) - - if len(sub_results) > 0: - count += 1 - results.extend(sub_results) - - if count >= 2: - results.append(self) - - return results - - def merge_into_self(self, __object: T, from_map: bool = False): - """ - 1. find existing objects - 2. merge into existing object - 3. remap existing object - """ - if __object.id in self._contains_ids: - return - - existing_object: T = None - - for name, value in __object.indexing_values: - if value is None: - continue - - if value == self._indexed_values[name]: - existing_object = self._indexed_to_objects[value] - if existing_object.id == __object.id: - return None - - break - - if existing_object is None: - return None - - existing_object.merge(__object) - - # just a check if it really worked - if existing_object.id != __object.id: - raise ValueError("This should NEVER happen. Merging doesn't work.") - - self._map_element(existing_object, from_map=from_map) - - def contains(self, __object: T) -> bool: - return len(self._contained_in_sub(__object)) > 0 - def _find_object_in_self(self, __object: T) -> Optional[T]: for name, value in __object.indexing_values: if value == self._indexed_values[name]: diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index 93b39ba..2437112 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -93,7 +93,6 @@ class Song(Base): self.album_collection.append_object_to_attribute = { "song_collection": self, } - self.main_artist_collection.contain_given_in_attribute = { "main_album_collection": self.album_collection } -- 2.45.2 From ea5adfbe8a2e54062d3eff51fa80f2f20b544eaa Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 23 Apr 2024 11:37:49 +0200 Subject: [PATCH 19/22] feat: limited complexity of collection by removing child collections --- music_kraken/objects/collection.py | 137 ++++++++--------------------- music_kraken/objects/song.py | 4 +- 2 files changed, 38 insertions(+), 103 deletions(-) diff --git a/music_kraken/objects/collection.py b/music_kraken/objects/collection.py index 6b59ac9..f1a694b 100644 --- a/music_kraken/objects/collection.py +++ b/music_kraken/objects/collection.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections import defaultdict -from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union +from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union, Any from .parents import OuterProxy from ..utils import object_trace @@ -22,28 +22,27 @@ class Collection(Generic[T]): self, data: Optional[Iterable[T]] = None, sync_on_append: Dict[str, Collection] = None, - contain_given_in_attribute: Dict[str, Collection] = None, - contain_attribute_in_given: Dict[str, Collection] = None, - append_object_to_attribute: Dict[str, T] = None + append_object_to_attribute: Dict[str, T] = None, + extend_object_to_attribute: Dict[str, Collection] = None, ) -> None: self._collection_for: dict = dict() self._contains_ids = set() self._data = [] - self.parents: List[Collection[T]] = [] - self.children: List[Collection[T]] = [] - # List of collection attributes that should be modified on append # Key: collection attribute (str) of appended element # Value: main collection to sync to - self.contain_given_in_attribute: Dict[str, Collection] = contain_given_in_attribute or {} self.append_object_to_attribute: Dict[str, T] = append_object_to_attribute or {} + self.extend_object_to_attribute: Dict[str, Collection[T]] = extend_object_to_attribute or {} self.sync_on_append: Dict[str, Collection] = sync_on_append or {} self._id_to_index_values: Dict[int, set] = defaultdict(set) - self._indexed_values = defaultdict(lambda: None) - self._indexed_to_objects = defaultdict(lambda: None) + + # This is to cleanly unmap previously mapped items by their id + self._indexed_from_id: Dict[int, Dict[str, Any]] = defaultdict(dict) + # this is to keep track and look up the actual objects + self._indexed_values: Dict[str, Dict[Any, T]] = defaultdict(dict) self.extend(data) @@ -51,66 +50,34 @@ class Collection(Generic[T]): return f"Collection({id(self)})" def _map_element(self, __object: T, from_map: bool = False): - self._contains_ids.add(__object.id) + self._unmap_element(__object.id) - for name, value in (*__object.indexing_values, ('id', __object.id)): + self._indexed_from_id[__object.id]["id"] = __object.id + self._indexed_values["id"][__object.id] = __object + + for name, value in __object.indexing_values: if value is None or value == __object._inner._default_values.get(name): continue - self._indexed_values[name] = value - self._indexed_to_objects[value] = __object - - self._id_to_index_values[__object.id].add((name, value)) + self._indexed_values[name][value] = __object + self._indexed_from_id[__object.id][name] = value def _unmap_element(self, __object: Union[T, int]): obj_id = __object.id if isinstance(__object, OuterProxy) else __object - if obj_id in self._contains_ids: - self._contains_ids.remove(obj_id) + if obj_id not in self._indexed_from_id: + return - for name, value in self._id_to_index_values[obj_id]: - if name in self._indexed_values: - del self._indexed_values[name] - if value in self._indexed_to_objects: - del self._indexed_to_objects[value] + for name, value in self._indexed_from_id[obj_id].items(): + if value in self._indexed_values[name]: + del self._indexed_values[name][value] - del self._id_to_index_values[obj_id] + del self._indexed_from_id[obj_id] - @property - def is_root(self) -> bool: - return len(self.parents) <= 0 - - def _find_object_in_self(self, __object: T) -> Optional[T]: + def _find_object(self, __object: T) -> Optional[T]: for name, value in __object.indexing_values: - if value == self._indexed_values[name]: - return self._indexed_to_objects[value] - - def _find_object(self, __object: T, no_sibling: bool = False) -> Tuple[Collection[T], Optional[T]]: - other_object = self._find_object_in_self(__object) - if other_object is not None: - return self, other_object - - for c in self.children: - o, other_object = c._find_object(__object) - if other_object is not None: - return o, other_object - - if no_sibling: - return self, None - - """ - # find in siblings and all children of siblings - for parent in self.parents: - for sibling in parent.children: - if sibling is self: - continue - - o, other_object = sibling._find_object(__object, no_sibling=True) - if other_object is not None: - return o, other_object - """ - - return self, None + if value in self._indexed_values[name]: + return self._indexed_values[name][value] def append(self, __object: Optional[T], already_is_parent: bool = False, from_map: bool = False): """ @@ -127,15 +94,15 @@ class Collection(Generic[T]): if __object is None: return - append_to, existing_object = self._find_object(__object) + existing_object = self._find_object(__object) if existing_object is None: # append - append_to._data.append(__object) - append_to._map_element(__object) + self._data.append(__object) + self._map_element(__object) - for collection_attribute, child_collection in self.contain_given_in_attribute.items(): - __object.__getattribute__(collection_attribute).contain_collection_inside(child_collection, __object) + for collection_attribute, child_collection in self.extend_object_to_attribute.items(): + __object.__getattribute__(collection_attribute).extend(child_collection) for attribute, new_object in self.append_object_to_attribute.items(): __object.__getattribute__(attribute).append(new_object) @@ -164,9 +131,9 @@ class Collection(Generic[T]): existing_object.merge(__object) if existing_object.id != old_id: - append_to._unmap_element(old_id) + self._unmap_element(old_id) - append_to._map_element(existing_object) + self._map_element(existing_object) def extend(self, __iterable: Optional[Generator[T, None, None]]): if __iterable is None: @@ -175,54 +142,22 @@ class Collection(Generic[T]): for __object in __iterable: self.append(__object) - def contain_collection_inside(self, sub_collection: Collection, _object: T): - """ - This collection will ALWAYS contain everything from the passed in collection - """ - if self is sub_collection or sub_collection in self.children: - return - - _object._inner._is_collection_child[self] = sub_collection - _object._inner._is_collection_parent[sub_collection] = self - - self.children.append(sub_collection) - sub_collection.parents.append(self) - @property def data(self) -> List[T]: return list(self.__iter__()) def __len__(self) -> int: - return len(self._data) + sum(len(collection) for collection in self.children) + return len(self._data) @property def empty(self) -> bool: return self.__len__() <= 0 - def __iter__(self, finished_ids: set = None) -> Iterator[T]: - _finished_ids = finished_ids or set() - - for element in self._data: - if element.id in _finished_ids: - continue - _finished_ids.add(element.id) - yield element - - for c in self.children: - yield from c.__iter__(finished_ids=finished_ids) + def __iter__(self) -> Iterator[T]: + yield from self._data def __merge__(self, __other: Collection, override: bool = False): self.extend(__other) def __getitem__(self, item: int): - if item < len(self._data): - return self._data[item] - - item = item - len(self._data) - - for c in self.children: - if item < len(c): - return c.__getitem__(item) - item = item - len(c._data) - - raise IndexError + return self._data[item] diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index 2437112..2df348e 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -93,7 +93,7 @@ class Song(Base): self.album_collection.append_object_to_attribute = { "song_collection": self, } - self.main_artist_collection.contain_given_in_attribute = { + self.main_artist_collection.extend_object_to_attribute = { "main_album_collection": self.album_collection } self.feature_artist_collection.append_object_to_attribute = { @@ -253,7 +253,7 @@ class Album(Base): self.artist_collection.append_object_to_attribute = { "main_album_collection": self } - self.artist_collection.contain_given_in_attribute = { + self.artist_collection.extend_object_to_attribute = { "label_collection": self.label_collection } -- 2.45.2 From 0080a48e70811d2a3844cbbb6f4f67340cd2e766 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 23 Apr 2024 11:39:25 +0200 Subject: [PATCH 20/22] feat: removed legacy code --- music_kraken/objects/parents.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/music_kraken/objects/parents.py b/music_kraken/objects/parents.py index a0bad1e..d80cd23 100644 --- a/music_kraken/objects/parents.py +++ b/music_kraken/objects/parents.py @@ -34,10 +34,6 @@ class InnerData: self._refers_to_instances = set() self._fetched_from: dict = {} - # collection : collection that is a collection of self - self._is_collection_child: Dict[Collection, Collection] = {} - self._is_collection_parent: Dict[Collection, Collection] = {} - # initialize the default values self._default_values = {} for name, factory in object_type._default_factories.items(): @@ -201,18 +197,6 @@ class OuterProxy: a, b = b, a object_trace(f"merging {type(a).__name__} [{a.title_string} | {a.id}] with {type(b).__name__} [{b.title_string} | {b.id}]") - - for collection, child_collection in b._inner._is_collection_child.items(): - try: - collection.children.remove(child_collection) - except ValueError: - pass - - for collection, parent_collection in b._inner._is_collection_parent.items(): - try: - collection.parents.remove(parent_collection) - except ValueError: - pass old_inner = b._inner -- 2.45.2 From 3d432cd0d7ede55af69511013bb83a98755720d2 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 23 Apr 2024 11:44:39 +0200 Subject: [PATCH 21/22] fix: test --- music_kraken/utils/shared.py | 2 +- tests/test_collection.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index 6397fd3..401b051 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -16,7 +16,7 @@ DEBUG = (__stage__ == "dev") and True DEBUG_LOGGING = DEBUG and False DEBUG_TRACE = DEBUG and True DEBUG_OBJECT_TRACE = DEBUG and True -DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG and True +DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG_OBJECT_TRACE and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_PAGES = DEBUG and False DEBUG_DUMP = DEBUG and False diff --git a/tests/test_collection.py b/tests/test_collection.py index 12fe2f2..bc3674b 100644 --- a/tests/test_collection.py +++ b/tests/test_collection.py @@ -86,7 +86,7 @@ class TestCollection(unittest.TestCase): ] ) - self.assertTrue(artist.id == artist.main_album_collection[0].song_collection[0].artist_collection[0].id) + self.assertTrue(artist.id == artist.main_album_collection[0].song_collection[0].main_artist_collection[0].id) def test_artist_collection_sync(self): album_1 = Album( -- 2.45.2 From 0179246ec0b84a5c13df948f07bf6b57c38ba6c4 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 23 Apr 2024 11:52:08 +0200 Subject: [PATCH 22/22] feat: dynamic objects now also have ids --- music_kraken/objects/parents.py | 2 +- music_kraken/utils/shared.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/music_kraken/objects/parents.py b/music_kraken/objects/parents.py index d80cd23..fd2a80c 100644 --- a/music_kraken/objects/parents.py +++ b/music_kraken/objects/parents.py @@ -89,7 +89,7 @@ class OuterProxy: def __init__(self, _id: int = None, dynamic: bool = False, **kwargs): _automatic_id: bool = False - if _id is None and not dynamic: + if _id is None: """ generates a random integer id the range is defined in the config diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index 401b051..a2b06b8 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -15,7 +15,7 @@ __stage__ = os.getenv("STAGE", "prod") DEBUG = (__stage__ == "dev") and True DEBUG_LOGGING = DEBUG and False DEBUG_TRACE = DEBUG and True -DEBUG_OBJECT_TRACE = DEBUG and True +DEBUG_OBJECT_TRACE = DEBUG and False DEBUG_OBJECT_TRACE_CALLSTACK = DEBUG_OBJECT_TRACE and False DEBUG_YOUTUBE_INITIALIZING = DEBUG and False DEBUG_PAGES = DEBUG and False -- 2.45.2