music-kraken/music-kraken-core
3
2
Fork 0
Code Issues 10 Pull Requests 1 Packages Projects 2 Releases Wiki Activity

sucessfully migrated metal archives

Browse Source
This commit is contained in:
Hellow 2023-04-20 19:45:29 +02:00
parent e3b1a866a1
commit 9445e95ef9
2 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/music_kraken/pages/abstract.py
View File

@ -6,6 +6,7 @@ from typing import Optional, Union, Type, Dict, Set
import requests
from bs4 import BeautifulSoup
from ..connection import Connection
from .support_classes.default_target import DefaultTarget
from .support_classes.download_result import DownloadResult
from ..objects import (
@ -30,6 +31,8 @@ class Page:
This is an abstract class, laying out the
functionality for every other class fetching something
"""
CONNECTION: Connection
API_SESSION: requests.Session = requests.Session()
API_SESSION.proxies = shared.proxies
TIMEOUT = 5

41
src/music_kraken/pages/encyclopaedia_metallum.py
View File

@ -5,6 +5,7 @@ from bs4 import BeautifulSoup
import pycountry
from urllib.parse import urlparse
from ..connection import Connection
from ..utils.shared import ENCYCLOPAEDIA_METALLUM_LOGGER, proxies
from ..utils import string_processing
from .abstract import Page
@ -24,12 +25,10 @@ from ..objects import (
class EncyclopaediaMetallum(Page):
API_SESSION: requests.Session = requests.Session()
API_SESSION.proxies = proxies
API_SESSION.headers = {
"Host": "www.metal-archives.com",
"Connection": "keep-alive"
}
CONNECTION: Connection = Connection(
host="https://www.metal-archives.com/",
logger=ENCYCLOPAEDIA_METALLUM_LOGGER
)
SOURCE_TYPE = SourcePages.ENCYCLOPAEDIA_METALLUM
@ -70,10 +69,10 @@ class EncyclopaediaMetallum(Page):
"&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&mDataProp_4=4&_" \
"=1674550595663"
r = cls.get_request(endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str))
if r.status_code != 200:
cls.LOGGER.warning(
f"code {r.status_code} at {endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)}")
r = cls.CONNECTION.get(
endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)
)
if r is None:
return []
return [cls.get_song_from_json(
@ -92,10 +91,8 @@ class EncyclopaediaMetallum(Page):
"=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=1&iColumns=3&sColumns" \
"=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&_=1674563943747"
r = cls.get_request(endpoint.format(artist=query.artist_str, album=query.album_str))
if r.status_code != 200:
cls.LOGGER.warning(
f"code {r.status_code} at {endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)}")
r = cls.CONNECTION.get(endpoint.format(artist=query.artist_str, album=query.album_str))
if r is None:
return []
return [cls.get_album_from_json(
@ -111,7 +108,7 @@ class EncyclopaediaMetallum(Page):
"=&bandLabelName=&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0" \
"&mDataProp_1=1&mDataProp_2=2&_=1674565459976"
r = cls.get_request(endpoint.format(artist=query.artist))
r = cls.CONNECTION.get(endpoint.format(artist=query.artist))
if r is None:
return []
@ -134,7 +131,7 @@ class EncyclopaediaMetallum(Page):
"""
endpoint = "https://www.metal-archives.com/search/ajax-band-search/?field=name&query={query}&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2"
r = cls.get_request(endpoint.format(query=query))
r = cls.CONNECTION.get(endpoint.format(query=query))
if r is None:
return []
@ -220,7 +217,7 @@ class EncyclopaediaMetallum(Page):
discography_url = "https://www.metal-archives.com/band/discography/id/{}/tab/all"
# make the request
r = cls.get_request(discography_url.format(ma_artist_id))
r = cls.CONNECTION.get(discography_url.format(ma_artist_id))
if r is None:
return []
soup = cls.get_soup_from_response(r)
@ -257,7 +254,7 @@ class EncyclopaediaMetallum(Page):
@classmethod
def _fetch_artist_sources(cls, ma_artist_id: str) -> List[Source]:
sources_url = "https://www.metal-archives.com/link/ajax-list/type/band/id/{}"
r = cls.get_request(sources_url.format(ma_artist_id))
r = cls.CONNECTION.get(sources_url.format(ma_artist_id))
if r is None:
return []
@ -405,7 +402,7 @@ class EncyclopaediaMetallum(Page):
@classmethod
def _fetch_artist_attributes(cls, url: str) -> Artist:
r = cls.get_request(url)
r = cls.CONNECTION.get(url)
if r is None:
return Artist()
soup: BeautifulSoup = cls.get_soup_from_response(r)
@ -417,7 +414,7 @@ class EncyclopaediaMetallum(Page):
endpoint = "https://www.metal-archives.com/band/read-more/id/{}"
# make the request
r = cls.get_request(endpoint.format(ma_artist_id))
r = cls.CONNECTION.get(endpoint.format(ma_artist_id))
if r is None:
return FormattedText()
@ -570,7 +567,7 @@ class EncyclopaediaMetallum(Page):
# <table class="display table_lyrics
r = cls.get_request(source.url)
r = cls.CONNECTION.get(source.url)
if r is None:
return Album()
@ -610,7 +607,7 @@ class EncyclopaediaMetallum(Page):
endpoint = "https://www.metal-archives.com/release/ajax-view-lyrics/id/{id}".format(id=song_id)
r = cls.get_request(endpoint)
r = cls.CONNECTION.get(endpoint)
if r is None:
return None