music-kraken/music-kraken-core
3
2
Fork 0
Code Issues 10 Pull Requests 1 Packages Projects 2 Releases Wiki Activity

layed out page boilerplate for bandcamp

Browse Source
This commit is contained in:
Hazel 2023-09-12 14:17:58 +02:00
parent e25f788b50
commit 8846eacf06
14 changed files with 93 additions and 589 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.vscode/settings.json vendored
View File

@ -14,5 +14,8 @@
"[python]": {
"editor.defaultFormatter": "ms-python.autopep8"
},
"python.formatting.provider": "none"
"python.formatting.provider": "none",
"cSpell.words": [
"Bandcamp"
]
}

3
src/actual_donwload.py
View File

@ -29,8 +29,7 @@ if __name__ == "__main__":
]
youtube_music_test = [
"s: #a Favorite #r Anarcho",
"0"
"s: #a Favorite #r Anarcho"
]
music_kraken.cli.download(genre="test", command_list=youtube_music_test, process_metadata_anyway=True)

22
src/music_kraken/download/page_attributes.py
View File

@ -9,20 +9,14 @@ from ..utils.support_classes import Query, DownloadResult
from ..utils.exception.download import UrlNotFoundException
from ..utils.shared import DEBUG_PAGES
from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, INDEPENDENT_DB_OBJECTS
if DEBUG_PAGES:
DEBUGGING_PAGE = YoutubeMusic
print(f"Only downloading from page {DEBUGGING_PAGE}.")
ALL_PAGES = {DEBUGGING_PAGE}
AUDIO_PAGES = ALL_PAGES.union(AUDIO_PAGES)
from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, Bandcamp, INDEPENDENT_DB_OBJECTS
ALL_PAGES: Set[Type[Page]] = {
EncyclopaediaMetallum,
Musify,
YoutubeMusic
YoutubeMusic,
Bandcamp
}
if youtube_settings["use_youtube_alongside_youtube_music"]:
@ -31,13 +25,21 @@ if youtube_settings["use_youtube_alongside_youtube_music"]:
AUDIO_PAGES: Set[Type[Page]] = {
Musify,
YouTube,
YoutubeMusic
YoutubeMusic,
Bandcamp
}
SHADY_PAGES: Set[Type[Page]] = {
Musify,
}
if DEBUG_PAGES:
DEBUGGING_PAGE = Bandcamp
print(f"Only downloading from page {DEBUGGING_PAGE}.")
ALL_PAGES = {DEBUGGING_PAGE}
AUDIO_PAGES = ALL_PAGES.union(AUDIO_PAGES)
class Pages:
def __init__(self, exclude_pages: Set[Type[Page]] = None, exclude_shady: bool = False) -> None:
# initialize all page instances

1
src/music_kraken/pages/__init__.py
View File

@ -2,5 +2,6 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum
from .musify import Musify
from .youtube import YouTube
from .youtube_music import YoutubeMusic
from .bandcamp import Bandcamp
from .abstract import Page, INDEPENDENT_DB_OBJECTS

2
src/music_kraken/pages/abstract.py
View File

@ -220,7 +220,7 @@ class Page:
if type(music_object) in search_functions:
r = search_functions[type(music_object)](music_object)
if len(r) > 0:
if r is not None and len(r) > 0:
return r
r = []

65
src/music_kraken/pages/bandcamp.py Normal file
View File

@ -0,0 +1,65 @@
from typing import List, Optional, Type
from urllib.parse import urlparse
import logging
from ..objects import Source, DatabaseObject
from .abstract import Page
from ..objects import (
Artist,
Source,
SourcePages,
Song,
Album,
Label,
Target
)
from ..connection import Connection
from ..utils.support_classes import DownloadResult
from ..utils.config import main_settings, logging_settings
class Bandcamp(Page):
# CHANGE
SOURCE_TYPE = SourcePages.BANDCAMP
LOGGER = logging_settings["bandcamp_logger"]
def __init__(self, *args, **kwargs):
self.connection: Connection = Connection(
host="https://bandcamp.com/",
logger=self.LOGGER
)
super().__init__(*args, **kwargs)
def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]:
return super().get_source_type(source)
def general_search(self, search_query: str) -> List[DatabaseObject]:
return []
def label_search(self, label: Label) -> List[Label]:
return []
def artist_search(self, artist: Artist) -> List[Artist]:
return []
def album_search(self, album: Album) -> List[Album]:
return []
def song_search(self, song: Song) -> List[Song]:
return []
def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song:
return Song()
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
return Album()
def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist:
return Artist()
def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label:
return Label()
def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult:
return DownloadResult()

6
src/music_kraken/utils/config/config_files/logging_config.py
View File

@ -79,6 +79,11 @@ Reference for the logging formats: https://docs.python.org/3/library/logging.htm
description="The logger for the genius scraper",
default_value="genius"
),
LoggerAttribute(
name="bandcamp_logger",
description="The logger for the bandcamp scraper",
default_value="bandcamp"
)
], LOCATIONS.get_config_file("logging"))
@ -97,3 +102,4 @@ class SettingsStructure(TypedDict):
youtube_music_logger: Logger
metal_archives_logger: Logger
genius_logger: Logger
bandcamp_logger: Logger

0
src/music_kraken/utils/config/sections/__init__.py
View File

154
src/music_kraken/utils/config/sections/audio.py
View File

@ -1,154 +0,0 @@
import logging
from ..base_classes import (
SingleAttribute,
FloatAttribute,
StringAttribute,
Section,
Description,
EmptyLine,
BoolAttribute,
ListAttribute
)
from ...enums.album import AlbumType
from ...exception.config import SettingValueError
# Only the formats with id3 metadata can be used
# https://www.audioranger.com/audio-formats.php
# https://web.archive.org/web/20230322234434/https://www.audioranger.com/audio-formats.php
ID3_2_FILE_FORMATS = frozenset((
"mp3", "mp2", "mp1", # MPEG-1 ID3.2
"wav", "wave", "rmi", # RIFF (including WAV) ID3.2
"aiff", "aif", "aifc", # AIFF ID3.2
"aac", "aacp", # Raw AAC ID3.2
"tta", # True Audio ID3.2
))
_sorted_id3_2_formats = sorted(ID3_2_FILE_FORMATS)
ID3_1_FILE_FORMATS = frozenset((
"ape", # Monkey's Audio ID3.1
"mpc", "mpp", "mp+", # MusePack ID3.1
"wv", # WavPack ID3.1
"ofr", "ofs" # OptimFrog ID3.1
))
_sorted_id3_1_formats = sorted(ID3_1_FILE_FORMATS)
class AudioFormatAttribute(SingleAttribute):
def validate(self, value: str):
v = self.value.strip().lower()
if v not in ID3_1_FILE_FORMATS and v not in ID3_2_FILE_FORMATS:
raise SettingValueError(
setting_name=self.name,
setting_value=value,
rule="has to be a valid audio format, supporting id3 metadata"
)
@property
def object_from_value(self) -> str:
v = self.value.strip().lower()
if v in ID3_2_FILE_FORMATS:
return v
if v in ID3_1_FILE_FORMATS:
logging.debug(f"setting audio format to a format that only supports ID3.1: {v}")
return v
raise ValueError(f"Invalid Audio Format: {v}")
class AlbumTypeListAttribute(ListAttribute):
def validate(self, value: str):
try:
AlbumType(value.strip())
except ValueError:
raise SettingValueError(
setting_name=self.name,
setting_value=value,
rule="has to be an existing album type"
)
def single_object_from_element(self, value: str) -> AlbumType:
return AlbumType(value)
class AudioSection(Section):
def __init__(self):
self.BITRATE = FloatAttribute(
name="bitrate",
description="Streams the audio with given bitrate [kB/s]. "
"Can't stream with a higher Bitrate, than the audio source provides.",
value="125"
)
self.AUDIO_FORMAT = AudioFormatAttribute(name="audio_format", value="mp3", description=f"""
Music Kraken will stream the audio into this format.
You can use Audio formats which support ID3.2 and ID3.1,
but you will have cleaner Metadata using ID3.2.
ID3.2: {', '.join(_sorted_id3_2_formats)}
ID3.1: {', '.join(_sorted_id3_1_formats)}
""".strip())
self.SORT_BY_DATE = BoolAttribute(
name="sort_by_date",
description="If this is set to true, it will set the albumsort attribute such that,\n"
"the albums are sorted by date.",
value="true"
)
self.SORT_BY_ALBUM_TYPE = BoolAttribute(
name="sort_album_by_type",
description="If this is set to true, it will set the albumsort attribute such that,\n"
"the albums are put into categories before being sorted.\n"
"This means for example, the Studio Albums and EP's are always in front of Singles, "
"and Compilations are in the back.",
value="true"
)
self.DOWNLOAD_PATH = StringAttribute(
name="download_path",
value="{genre}/{artist}/{album}",
description="The folder music kraken should put the songs into."
)
self.DOWNLOAD_FILE = StringAttribute(
name="download_file",
value="{song}.{audio_format}",
description="The filename of the audio file."
)
self.ALBUM_TYPE_BLACKLIST = AlbumTypeListAttribute(
name="album_type_blacklist",
description="Music Kraken ignores all albums of those types.\n"
"Following album types exist in the programm:\n"
f"{', '.join(album.value for album in AlbumType)}",
value=[
AlbumType.COMPILATION_ALBUM.value,
AlbumType.LIVE_ALBUM.value,
AlbumType.MIXTAPE.value
]
)
self.attribute_list = [
self.BITRATE,
self.AUDIO_FORMAT,
EmptyLine(),
self.SORT_BY_DATE,
self.SORT_BY_ALBUM_TYPE,
Description("""
There are multiple fields, you can use for the path and file name:
- genre
- label
- artist
- album
- song
- album_type
""".strip()),
self.DOWNLOAD_PATH,
self.DOWNLOAD_FILE,
self.ALBUM_TYPE_BLACKLIST,
]
super().__init__()
AUDIO_SECTION = AudioSection()

157
src/music_kraken/utils/config/sections/connection.py
View File

@ -1,157 +0,0 @@
from urllib.parse import urlparse, ParseResult
import re
from ..base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute, StringAttribute
from ...regex import URL_PATTERN
from ...exception.config import SettingValueError
class ProxAttribute(ListAttribute):
def single_object_from_element(self, value) -> dict:
return {
'http': value,
'https': value,
'ftp': value
}
class UrlStringAttribute(StringAttribute):
def validate(self, value: str):
v = value.strip()
url = re.match(URL_PATTERN, v)
if url is None:
raise SettingValueError(
setting_name=self.name,
setting_value=v,
rule="has to be a valid url"
)
@property
def object_from_value(self) -> ParseResult:
return urlparse(self.value)
class UrlListAttribute(ListAttribute):
def validate(self, value: str):
v = value.strip()
url = re.match(URL_PATTERN, v)
if url is None:
raise SettingValueError(
setting_name=self.name,
setting_value=v,
rule="has to be a valid url"
)
def single_object_from_element(self, value: str):
return urlparse(value)
class ConnectionSection(Section):
def __init__(self):
self.PROXIES = ProxAttribute(
name="proxies",
description="Set your proxies.\n"
"Must be valid for http, as well as https.",
value=[]
)
self.USE_TOR = BoolAttribute(
name="tor",
description="Route ALL traffic through Tor.\n"
"If you use Tor, make sure the Tor browser is installed, and running."
"I can't guarantee maximum security though!",
value="false"
)
self.TOR_PORT = IntAttribute(
name="tor_port",
description="The port, tor is listening. If tor is already working, don't change it.",
value="9150"
)
self.CHUNK_SIZE = IntAttribute(
name="chunk_size",
description="Size of the chunks that are streamed.",
value="1024"
)
self.SHOW_DOWNLOAD_ERRORS_THRESHOLD = FloatAttribute(
name="show_download_errors_threshold",
description="If the percentage of failed downloads goes over this threshold,\n"
"all the error messages are shown.",
value="0.3"
)
# INVIDIOUS INSTANCES LIST
self.INVIDIOUS_INSTANCE = UrlStringAttribute(
name="invidious_instance",
description="This is an attribute, where you can define the invidious instances,\n"
"the youtube downloader should use.\n"
"Here is a list of active ones: https://docs.invidious.io/instances/\n"
"Instances that use cloudflare or have source code changes could cause issues.\n"
"Hidden instances (.onion) will only work, when setting 'tor=true'.",
value="https://yt.artemislena.eu/"
)
self.PIPED_INSTANCE = UrlStringAttribute(
name="piped_instance",
description="This is an attribute, where you can define the pioed instances,\n"
"the youtube downloader should use.\n"
"Here is a list of active ones: https://github.com/TeamPiped/Piped/wiki/Instances\n"
"Instances that use cloudflare or have source code changes could cause issues.\n"
"Hidden instances (.onion) will only work, when setting 'tor=true'.",
value="https://pipedapi.kavin.rocks"
)
self.SLEEP_AFTER_YOUTUBE_403 = FloatAttribute(
name="sleep_after_youtube_403",
description="The time to wait, after youtube returned 403 (in seconds)",
value="20"
)
self.YOUTUBE_MUSIC_API_KEY = StringAttribute(
name="youtube_music_api_key",
description="This is the API key used by YouTube-Music internally.\nDw. if it is empty, Rachel will fetch it automatically for you <333\n(she will also update outdated api keys/those that don't work)",
value="AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30"
)
self.YOUTUBE_MUSIC_CLEAN_DATA = BoolAttribute(
name="youtube_music_clean_data",
description="If set to true, it exclusively fetches artists/albums/songs, not things like user channels etc.",
value="true"
)
self.ALL_YOUTUBE_URLS = UrlListAttribute(
name="youtube_url",
description="This is used to detect, if an url is from youtube, or any alternativ frontend.\n"
"If any instance seems to be missing, run music kraken with the -f flag.",
value=[
"https://www.youtube.com/",
"https://www.youtu.be/",
"https://redirect.invidious.io/",
"https://piped.kavin.rocks/"
]
)
self.SPONSOR_BLOCK = BoolAttribute(
name="use_sponsor_block",
value="true",
description="Use sponsor block to remove adds or simmilar from the youtube videos."
)
self.attribute_list = [
self.USE_TOR,
self.TOR_PORT,
self.CHUNK_SIZE,
self.SHOW_DOWNLOAD_ERRORS_THRESHOLD,
self.INVIDIOUS_INSTANCE,
self.PIPED_INSTANCE,
self.SLEEP_AFTER_YOUTUBE_403,
self.YOUTUBE_MUSIC_API_KEY,
self.YOUTUBE_MUSIC_CLEAN_DATA,
self.ALL_YOUTUBE_URLS,
self.SPONSOR_BLOCK
]
super().__init__()
CONNECTION_SECTION = ConnectionSection()

130
src/music_kraken/utils/config/sections/logging.py
View File

@ -1,130 +0,0 @@
import logging
from typing import Callable
from ..base_classes import SingleAttribute, StringAttribute, Section, Description, EmptyLine
LOG_LEVELS = {
"CRITICAL": 50,
"ERROR": 40,
"WARNING": 30,
"INFO": 20,
"DEBUG": 10,
"NOTSET": 0
}
class LoggerAttribute(SingleAttribute):
@property
def object_from_value(self) -> logging.Logger:
return logging.getLogger(self.value)
class LogLevelAttribute(SingleAttribute):
@property
def object_from_value(self) -> int:
"""
gets the numeric value of a log level
:return:
"""
if self.value.isnumeric():
return int(self.value)
v = self.value.strip().upper()
if v not in LOG_LEVELS:
raise ValueError(
f"{self.name} can only been either one of the following levels, or an integer:\n"
f"{';'.join(key for key in LOG_LEVELS)}"
)
return LOG_LEVELS[v]
class LoggingSection(Section):
def __init__(self):
self.FORMAT = StringAttribute(
name="logging_format",
description="Reference for the logging formats: "
"https://docs.python.org/3/library/logging.html#logrecord-attributes",
value=logging.BASIC_FORMAT
)
self.LOG_LEVEL = LogLevelAttribute(
name="log_level",
description=f"can only been either one of the following levels, or an integer:\n"
f"{';'.join(key for key in LOG_LEVELS)}",
value=str(logging.INFO)
)
self.DOWNLOAD_LOGGER = LoggerAttribute(
name="download_logger",
description="The logger for downloading.",
value="download"
)
self.TAGGING_LOGGER = LoggerAttribute(
name="tagging_logger",
description="The logger for tagging id3 containers.",
value="tagging"
)
self.CODEX_LOGGER = LoggerAttribute(
name="codex_logger",
description="The logger for streaming the audio into an uniform codex.",
value="codex"
)
self.OBJECT_LOGGER = LoggerAttribute(
name="object_logger",
description="The logger for creating Data-Objects.",
value="object"
)
self.DATABASE_LOGGER = LoggerAttribute(
name="database_logger",
description="The logger for Database operations.",
value="database"
)
self.MUSIFY_LOGGER = LoggerAttribute(
name="musify_logger",
description="The logger for the musify scraper.",
value="musify"
)
self.YOUTUBE_LOGGER = LoggerAttribute(
name="youtube_logger",
description="The logger for the youtube scraper.",
value="youtube"
)
self.YOUTUBE_MUSIC_LOGGER = LoggerAttribute(
name="youtube_music_logger",
description="The logger for the youtube music scraper.\n(The scraper is seperate to the youtube scraper)",
value="youtube_music"
)
self.ENCYCLOPAEDIA_METALLUM_LOGGER = LoggerAttribute(
name="metal_archives_logger",
description="The logger for the metal archives scraper.",
value="metal_archives"
)
self.GENIUS_LOGGER = LoggerAttribute(
name="genius_logger",
description="The logger for the genius scraper",
value="genius"
)
self.attribute_list = [
Description("Logging settings for the actual logging:"),
self.FORMAT,
self.LOG_LEVEL,
EmptyLine(),
Description("Just the names for different logger, for different parts of the programm:"),
self.DOWNLOAD_LOGGER,
self.TAGGING_LOGGER,
self.CODEX_LOGGER,
self.OBJECT_LOGGER,
self.DATABASE_LOGGER,
self.MUSIFY_LOGGER,
self.YOUTUBE_LOGGER,
self.YOUTUBE_MUSIC_LOGGER,
self.ENCYCLOPAEDIA_METALLUM_LOGGER,
self.GENIUS_LOGGER
]
super().__init__()
LOGGING_SECTION = LoggingSection()

72
src/music_kraken/utils/config/sections/misc.py
View File

@ -1,72 +0,0 @@
from ..base_classes import Section, IntAttribute, ListAttribute, BoolAttribute
class MiscSection(Section):
def __init__(self):
self.HASNT_YET_STARTED = BoolAttribute(
name="hasnt_yet_started",
description="If you did already run, and configured everything, this is false.",
value="true"
)
self.ENABLE_RESULT_HISTORY = BoolAttribute(
name="result_history",
description="If enabled, you can go back to the previous results.\n"
"The consequence is a higher meory consumption, because every result is saved.",
value="false"
)
self.HISTORY_LENGTH = IntAttribute(
name="history_length",
description="You can choose how far back you can go in the result history.\n"
"The further you choose to be able to go back, the higher the memory usage.\n"
"'-1' removes the Limit entirely.",
value="8"
)
self.HAPPY_MESSAGES = ListAttribute(
name="happy_messages",
description="Just some nice and wholesome messages.\n"
"If your mindset has traits of a [file corruption], you might not agree.\n"
"But anyways... Freedom of thought, so go ahead and change the messages.",
value=[
"Support the artist.",
"Star Me: https://github.com/HeIIow2/music-downloader",
"🏳️‍⚧️🏳️‍⚧️ Trans rights are human rights. 🏳️‍⚧️🏳️‍⚧️",
"🏳️‍⚧️🏳️‍⚧️ Trans women are women, trans men are men, and enbies are enbies. 🏳️‍⚧️🏳️‍⚧️",
"🏴‍☠️🏴‍☠️ Unite under one flag, fck borders. 🏴‍☠️🏴‍☠️",
"Join my Matrix Space: https://matrix.to/#/#music-kraken:matrix.org",
"Gotta love the BPJM ;-;",
"🏳️‍⚧️🏳️‍⚧️ Protect trans youth. 🏳️‍⚧️🏳️‍⚧️",
]
)
self.MODIFY_GC = BoolAttribute(
name="modify_gc",
description="If set to true, it will modify the gc for the sake of performance.\n"
"This should not drive up ram usage, but if it is, then turn it of.\n"
"Here a blog post about that matter:\n"
"https://mkennedy.codes/posts/python-gc-settings-change-this-and-make-your-app-go-20pc-faster/\n"
"https://web.archive.org/web/20221124122222/https://mkennedy.codes/posts/python-gc-settings-change-this-and-make-your-app-go-20pc-faster/",
value="true"
)
self.ID_BITS = IntAttribute(
name="id_bits",
description="I really dunno why I even made this a setting.. Modifying this is a REALLY dumb idea.",
value="64"
)
self.attribute_list = [
self.HASNT_YET_STARTED,
self.ENABLE_RESULT_HISTORY,
self.HISTORY_LENGTH,
self.HAPPY_MESSAGES,
self.MODIFY_GC,
self.ID_BITS
]
super().__init__()
MISC_SECTION = MiscSection()

59
src/music_kraken/utils/config/sections/paths.py
View File

@ -1,59 +0,0 @@
from pathlib import Path
from ...path_manager import LOCATIONS
from ..base_classes import Section, StringAttribute, ListAttribute
class PathAttribute(StringAttribute):
@property
def object_from_value(self) -> Path:
return Path(self.value.strip())
class PathsSection(Section):
def __init__(self):
self.MUSIC_DIRECTORY = PathAttribute(
name="music_directory",
description="The directory, all the music will be downloaded to.",
value=str(LOCATIONS.MUSIC_DIRECTORY)
)
self.TEMP_DIRECTORY = PathAttribute(
name="temp_directory",
description="All temporary stuff is gonna be dumped in this directory.",
value=str(LOCATIONS.TEMP_DIRECTORY)
)
self.LOG_PATH = PathAttribute(
name="log_file",
description="The path to the logging file",
value=str(LOCATIONS.get_log_file("download_logs.log"))
)
self.NOT_A_GENRE_REGEX = ListAttribute(
name="not_a_genre_regex",
description="These regular expressions tell music-kraken, which sub-folders of the music-directory\n"
"it should ignore, and not count to genres",
value=[
r'^\.' # is hidden/starts with a "."
]
)
self.FFMPEG_BINARY = PathAttribute(
name="ffmpeg_binary",
description="Set the path to the ffmpeg binary.",
value=str(LOCATIONS.FFMPEG_BIN)
)
self.attribute_list = [
self.MUSIC_DIRECTORY,
self.TEMP_DIRECTORY,
self.LOG_PATH,
self.NOT_A_GENRE_REGEX,
self.FFMPEG_BINARY
]
super().__init__()
PATHS_SECTION = PathsSection()

4
src/music_kraken/utils/shared.py
View File

@ -2,9 +2,9 @@ import random
from .config import main_settings
DEBUG = False
DEBUG = True
DEBUG_YOUTUBE_INITIALIZING = DEBUG and False
DEBUG_PAGES = DEBUG and False
DEBUG_PAGES = DEBUG and True
if DEBUG:
print("DEBUG ACTIVE")