From adfce16d2a2654a665a5198c3355e6ae4c714ea3 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 21 May 2024 16:21:58 +0200 Subject: [PATCH] feat: fetched the flat artist details --- .vscode/settings.json | 1 + music_kraken/connection/cache.py | 6 ++-- music_kraken/objects/artwork.py | 2 ++ music_kraken/pages/genius.py | 49 ++++++++++++++++++-------------- 4 files changed, 33 insertions(+), 25 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index bea0c42..fbc21fa 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -27,6 +27,7 @@ "Gitea", "iframe", "isrc", + "itemprop", "levenshtein", "metallum", "MUSICBRAINZ", diff --git a/music_kraken/connection/cache.py b/music_kraken/connection/cache.py index 004a6ba..4c40fe9 100644 --- a/music_kraken/connection/cache.py +++ b/music_kraken/connection/cache.py @@ -136,13 +136,13 @@ class Cache: ) self._write_attribute(cache_attribute) - cache_path = fit_to_file_system(Path(module_path, name), hidden_ok=True) + cache_path = fit_to_file_system(Path(module_path, name.replace("/", "_")), hidden_ok=True) with cache_path.open("wb") as content_file: self.logger.debug(f"writing cache to {cache_path}") content_file.write(content) def get(self, name: str) -> Optional[CacheResult]: - path = fit_to_file_system(Path(self._dir, self.module, name), hidden_ok=True) + path = fit_to_file_system(Path(self._dir, self.module, name.replace("/", "_")), hidden_ok=True) if not path.is_file(): return None @@ -165,7 +165,7 @@ class Cache: if ca.name == "": continue - file = fit_to_file_system(Path(self._dir, ca.module, ca.name), hidden_ok=True) + file = fit_to_file_system(Path(self._dir, ca.module, ca.name.replace("/", "_")), hidden_ok=True) if not ca.is_valid: self.logger.debug(f"deleting cache {ca.id}") diff --git a/music_kraken/objects/artwork.py b/music_kraken/objects/artwork.py index d5ba54b..178edf6 100644 --- a/music_kraken/objects/artwork.py +++ b/music_kraken/objects/artwork.py @@ -59,4 +59,6 @@ class Artwork: self._variant_mapping[key] = value def __eq__(self, other: Artwork) -> bool: + if not isinstance(other, Artwork): + return False return any(a == b for a, b in zip(self._variant_mapping.keys(), other._variant_mapping.keys())) diff --git a/music_kraken/pages/genius.py b/music_kraken/pages/genius.py index e6fa86f..7c3af73 100644 --- a/music_kraken/pages/genius.py +++ b/music_kraken/pages/genius.py @@ -85,20 +85,38 @@ class Genius(Page): self.add_to_artwork(artwork, data.get("header_image_url")) self.add_to_artwork(artwork, data.get("image_url")) + additional_sources: List[Source] = [] source: Source = Source(self.SOURCE_TYPE, data.get("url"), additional_data={ "id": data.get("id"), "slug": data.get("slug"), "api_path": data.get("api_path"), }) + notes = FormattedText() + description = data.get("description", {}) + if "html" in description: + notes.html = description["html"] + elif "markdown" in description: + notes.markdown = description["markdown"] + elif "description_preview" in data: + notes.plain = data["description_preview"] + if source.url is None: return None if object_type == "artist": + if data.get("instagram_name") is not None: + additional_sources.append(Source(ALL_SOURCE_TYPES.INSTAGRAM, f"https://www.instagram.com/{data['instagram_name']}/")) + if data.get("facebook_name") is not None: + additional_sources.append(Source(ALL_SOURCE_TYPES.FACEBOOK, f"https://www.facebook.com/{data['facebook_name']}/")) + if data.get("twitter_name") is not None: + additional_sources.append(Source(ALL_SOURCE_TYPES.TWITTER, f"https://x.com/{data['twitter_name']}/")) + return Artist( name=data.get("name"), source_list=[source], artwork=artwork, + notes=notes, ) if object_type == "album": @@ -165,35 +183,22 @@ class Genius(Page): def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: artist = Artist() + # https://genius.com/api/artists/24527/albums?page=1 - r = self.connection.get(_parse_artist_url(source.url), name=f"artist_{urlparse(source.url).scheme}_{urlparse(source.url).netloc}") + r = self.connection.get(source.url, name=source.url) if r is None: return artist - soup = self.get_soup_from_response(r) - if DEBUG: - dump_to_file("artist_page.html", r.text, exit_after_dump=False) + # find the content attribute in the meta tag which is contained in the head + data_container = soup.find("meta", {"itemprop": "page_data"}) + if data_container is not None: + content = data_container["content"] + dump_to_file("genius_itemprop_artist.json", content, is_json=True, exit_after_dump=False) + data = json.loads(content) - artist = self._parse_artist_details(soup=soup.find("div", {"id": "bio-container"})) + artist = self.parse_api_object(data.get("artist", {})) - html_music_grid = soup.find("ol", {"id": "music-grid"}) - if html_music_grid is not None: - for subsoup in html_music_grid.find_all("li"): - artist.album_collection.append(self._parse_album(soup=subsoup, initial_source=source)) - - for i, data_blob_soup in enumerate(soup.find_all("div", {"id": ["pagedata", "collectors-data"]})): - data_blob = data_blob_soup["data-blob"] - - if DEBUG: - dump_to_file(f"bandcamp_artist_data_blob_{i}.json", data_blob, is_json=True, exit_after_dump=False) - - if data_blob is not None: - artist.album_collection.extend( - self._parse_artist_data_blob(json.loads(data_blob), source.url) - ) - - artist.source_collection.append(source) return artist def _parse_track_element(self, track: dict, artwork: Artwork) -> Optional[Song]: