layed out better methods for fetching data

This commit is contained in:
Hellow2 2023-03-27 16:28:34 +02:00
parent 2ff58d3a30
commit f5646276d4
2 changed files with 33 additions and 13 deletions

View File

@ -1,4 +1,5 @@
from typing import Optional, Union, Type, Dict from typing import Optional, Union, Type, Dict
from bs4 import BeautifulSoup
import requests import requests
import logging import logging
@ -70,6 +71,10 @@ class Page:
return cls.post_request(url, accepted_response_codes, trie + 1) return cls.post_request(url, accepted_response_codes, trie + 1)
@classmethod
def get_soup_from_response(cls, r: requests.Response) -> BeautifulSoup:
return BeautifulSoup(r.content, "html.parser")
class Query: class Query:
def __init__(self, query: str): def __init__(self, query: str):
self.query = query self.query = query
@ -178,16 +183,16 @@ class Page:
@classmethod @classmethod
def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1): def _fetch_object_from_source(cls, source: Source, obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], stop_at_level: int = 1):
if obj_type == Artist: if obj_type == Artist:
return cls.fetch_artist_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_artist_from_source(source=source, stop_at_level=stop_at_level)
if obj_type == Song: if obj_type == Song:
return cls.fetch_song_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_song_from_source(source=source, stop_at_level=stop_at_level)
if obj_type == Album: if obj_type == Album:
return cls.fetch_album_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_album_from_source(source=source, stop_at_level=stop_at_level)
if obj_type == Label: if obj_type == Label:
return cls.fetch_label_from_source(source=source, stop_at_level=stop_at_level) return cls._fetch_label_from_source(source=source, stop_at_level=stop_at_level)
@classmethod @classmethod
def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]):
@ -238,18 +243,33 @@ class Page:
cls._clean_collection(song.main_artist_collection, collections) cls._clean_collection(song.main_artist_collection, collections)
@classmethod @classmethod
def fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song: def _parse_song(cls, song_soup: BeautifulSoup) -> Song:
return Song() return Song()
@classmethod @classmethod
def fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: def _fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song:
return Album() return Song()
@classmethod @classmethod
def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: def _parse_artist(cls, album_soup: BeautifulSoup) -> Album:
return Album()
@classmethod
def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album:
return Album()
@classmethod
def _parse_artist(cls, artist_soup: BeautifulSoup) -> Artist:
return Artist() return Artist()
@classmethod @classmethod
def fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label: def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
return Artist()
@classmethod
def _parse_label(cls, label_soup: BeautifulSoup) -> Label:
return Label()
@classmethod
def _fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label:
return Label() return Label()

View File

@ -572,7 +572,7 @@ class Musify(Page):
album_source: Source album_source: Source
if stop_at_level > 1: if stop_at_level > 1:
for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE): for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE):
new_album.merge(cls.fetch_album_from_source(album_source, stop_at_level=stop_at_level-1)) new_album.merge(cls._fetch_album_from_source(album_source, stop_at_level=stop_at_level-1))
discography.append(new_album) discography.append(new_album)
@ -709,7 +709,7 @@ class Musify(Page):
) )
@classmethod @classmethod
def fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist:
""" """
fetches artist from source fetches artist from source
@ -851,7 +851,7 @@ class Musify(Page):
) )
@classmethod @classmethod
def fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album:
""" """
fetches album from source: fetches album from source:
eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188' eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188'