music-kraken/music-kraken-core
3
2
Fork 0
Code Issues 11 Pull Requests 1 Packages Projects 2 Releases Wiki Activity

Merge branch 'youtube' into experimental

Browse Source
This commit is contained in:
Hellow2
2023-06-21 08:23:16 +02:00
parent fa367a3087 adc7716dff
commit 1d64aab778
123 changed files with 5679 additions and 4748 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/music_kraken/utils/__init__.py
View File

@@ -1,4 +1 @@
from .config import config, read, write
# tells what exists
__all__ = ["shared", "object_handeling", "phonetic_compares", "functions"]
from .config import config, read_config, write_config

19
src/music_kraken/utils/config/__init__.py
View File

@@ -4,7 +4,22 @@ from .connection import CONNECTION_SECTION
from .misc import MISC_SECTION
from .paths import PATHS_SECTION
from .config import read, write, config
from .paths import LOCATIONS
from .config import Config
read()
config = Config()
def read_config():
if not LOCATIONS.CONFIG_FILE.is_file():
write_config()
config.read_from_config_file(LOCATIONS.CONFIG_FILE)
def write_config():
config.write_to_config_file(LOCATIONS.CONFIG_FILE)
set_name_to_value = config.set_name_to_value
read_config()

42
src/music_kraken/utils/config/audio.py
View File

@@ -106,7 +106,7 @@ ID3.1: {', '.join(_sorted_id3_1_formats)}
self.DOWNLOAD_PATH = StringAttribute(
name="download_path",
value="{genre}/{artist}/{album_type}/{album}",
value="{genre}/{artist}/{album}",
description="The folder music kraken should put the songs into."
)
@@ -116,42 +116,7 @@ ID3.1: {', '.join(_sorted_id3_1_formats)}
description="The filename of the audio file."
)
self.DEFAULT_GENRE = StringAttribute(
name="default_genre",
value="Various Genre",
description="The default value for the genre field."
)
self.DEFAULT_LABEL = StringAttribute(
name="default_label",
value="Various Labels",
description="The Label refers to a lable that signs artists."
)
self.DEFAULT_ARTIST = StringAttribute(
name="default_artist",
value="Various Artists",
description="You know Various Artist."
)
self.DEFAULT_ALBUM = StringAttribute(
name="default_album",
value="Various Album",
description="This value will hopefully not be used."
)
self.DEFAULT_SONG = StringAttribute(
name="default_song",
value="Various Song",
description="If it has to fall back to this value, something did go really wrong."
)
self.DEFAULT_ALBUM_TYPE = StringAttribute(
name="default_album_type",
value="Other",
description="Weirdly enough I barely see this used in file systems."
)
self.ALBUM_TYPE_BLACKLIST = AlbumTypeListAttribute(
name="album_type_blacklist",
description="Music Kraken ignores all albums of those types.\n"
@@ -181,11 +146,6 @@ There are multiple fields, you can use for the path and file name:
""".strip()),
self.DOWNLOAD_PATH,
self.DOWNLOAD_FILE,
self.DEFAULT_ALBUM_TYPE,
self.DEFAULT_ARTIST,
self.DEFAULT_GENRE,
self.DEFAULT_LABEL,
self.DEFAULT_SONG,
self.ALBUM_TYPE_BLACKLIST,
]
super().__init__()

3
src/music_kraken/utils/config/base_classes.py
View File

@@ -144,6 +144,9 @@ class ListAttribute(Attribute):
self.value = []
self.has_default_values = False
if value in self.value:
return
self.value.append(value)
def __str__(self):

19
src/music_kraken/utils/config/config.py
View File

@@ -58,7 +58,7 @@ class Config:
self._name_section_map[name] = element
self._length += 1
def set_name_to_value(self, name: str, value: str):
def set_name_to_value(self, name: str, value: str, silent: bool = True):
"""
:raises SettingValueError, SettingNotFound:
:param name:
@@ -66,6 +66,9 @@ class Config:
:return:
"""
if name not in self._name_section_map:
if silent:
LOGGER.warning(f"The setting \"{name}\" is either deprecated, or doesn't exist.")
return
raise SettingNotFound(setting_name=name)
LOGGER.debug(f"setting: {name} value: {value}")
@@ -122,17 +125,3 @@ class Config:
for section in self._section_list:
for name, attribute in section.name_attribute_map.items():
yield attribute
config = Config()
def read():
if not LOCATIONS.CONFIG_FILE.is_file():
LOGGER.debug("Creating default config file.")
write()
config.read_from_config_file(LOCATIONS.CONFIG_FILE)
def write():
config.write_to_config_file(LOCATIONS.CONFIG_FILE)

84
src/music_kraken/utils/config/connection.py
View File

@@ -1,4 +1,9 @@
from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute
from urllib.parse import urlparse, ParseResult
import re
from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute, StringAttribute
from ..regex import URL_PATTERN
from ..exception.config import SettingValueError
class ProxAttribute(ListAttribute):
@@ -10,6 +15,38 @@ class ProxAttribute(ListAttribute):
}
class UrlStringAttribute(StringAttribute):
def validate(self, value: str):
v = value.strip()
url = re.match(URL_PATTERN, v)
if url is None:
raise SettingValueError(
setting_name=self.name,
setting_value=v,
rule="has to be a valid url"
)
@property
def object_from_value(self) -> ParseResult:
return urlparse(self.value)
class UrlListAttribute(ListAttribute):
def validate(self, value: str):
v = value.strip()
url = re.match(URL_PATTERN, v)
if url is None:
raise SettingValueError(
setting_name=self.name,
setting_value=v,
rule="has to be a valid url"
)
def single_object_from_element(self, value: str):
return urlparse(value)
class ConnectionSection(Section):
def __init__(self):
self.PROXIES = ProxAttribute(
@@ -43,11 +80,54 @@ class ConnectionSection(Section):
value="0.3"
)
# INVIDIOUS INSTANCES LIST
self.INVIDIOUS_INSTANCE = UrlStringAttribute(
name="invidious_instance",
description="This is an attribute, where you can define the invidious instances,\n"
"the youtube downloader should use.\n"
"Here is a list of active ones: https://docs.invidious.io/instances/\n"
"Instances that use cloudflare or have source code changes could cause issues.\n"
"Hidden instances (.onion) will only work, when setting 'tor=true'.",
value="https://yt.artemislena.eu/"
)
self.PIPED_INSTANCE = UrlStringAttribute(
name="piped_instance",
description="This is an attribute, where you can define the pioed instances,\n"
"the youtube downloader should use.\n"
"Here is a list of active ones: https://github.com/TeamPiped/Piped/wiki/Instances\n"
"Instances that use cloudflare or have source code changes could cause issues.\n"
"Hidden instances (.onion) will only work, when setting 'tor=true'.",
value="https://pipedapi.kavin.rocks"
)
self.ALL_YOUTUBE_URLS = UrlListAttribute(
name="youtube_url",
description="This is used to detect, if an url is from youtube, or any alternativ frontend.\n"
"If any instance seems to be missing, run music kraken with the -f flag.",
value=[
"https://www.youtube.com/",
"https://www.youtu.be/",
"https://redirect.invidious.io/",
"https://piped.kavin.rocks/"
]
)
self.SPONSOR_BLOCK = BoolAttribute(
name="use_sponsor_block",
value="true",
description="Use sponsor block to remove adds or simmilar from the youtube videos."
)
self.attribute_list = [
self.USE_TOR,
self.TOR_PORT,
self.CHUNK_SIZE,
self.SHOW_DOWNLOAD_ERRORS_THRESHOLD
self.SHOW_DOWNLOAD_ERRORS_THRESHOLD,
self.INVIDIOUS_INSTANCE,
self.PIPED_INSTANCE,
self.ALL_YOUTUBE_URLS,
self.SPONSOR_BLOCK
]
super().__init__()

25
src/music_kraken/utils/config/misc.py
View File

@@ -3,6 +3,21 @@ from .base_classes import Section, IntAttribute, ListAttribute, BoolAttribute
class MiscSection(Section):
def __init__(self):
self.ENABLE_RESULT_HISTORY = BoolAttribute(
name="result_history",
description="If enabled, you can go back to the previous results.\n"
"The consequence is a higher meory consumption, because every result is saved.",
value="false"
)
self.HISTORY_LENGTH = IntAttribute(
name="history_length",
description="You can choose how far back you can go in the result history.\n"
"The further you choose to be able to go back, the higher the memory usage.\n"
"'-1' removes the Limit entirely.",
value="8"
)
self.HAPPY_MESSAGES = ListAttribute(
name="happy_messages",
description="Just some nice and wholesome messages.\n"
@@ -12,11 +27,11 @@ class MiscSection(Section):
"Support the artist.",
"Star Me: https://github.com/HeIIow2/music-downloader",
"🏳️‍⚧️🏳️‍⚧️ Trans rights are human rights. 🏳️‍⚧️🏳️‍⚧️",
"🏳️‍⚧️🏳️‍⚧️ Trans women are women, trans men are men. 🏳️‍⚧️🏳️‍⚧️",
"🏴‍☠️🏴‍☠️ Unite under one flag, fuck borders. 🏴‍☠️🏴‍☠️",
"🏳️‍⚧️🏳️‍⚧️ Trans women are women, trans men are men, and enbies are enbies. 🏳️‍⚧️🏳️‍⚧️",
"🏴‍☠️🏴‍☠️ Unite under one flag, fck borders. 🏴‍☠️🏴‍☠️",
"Join my Matrix Space: https://matrix.to/#/#music-kraken:matrix.org",
"Gotta love the BPJM!! >:(",
"🏳️‍⚧️🏳️‍⚧️ Protect trans youth. 🏳️‍⚧️🏳️‍⚧️"
"Gotta love the BPJM ;-;",
"🏳️‍⚧️🏳️‍⚧️ Protect trans youth. 🏳️‍⚧️🏳️‍⚧️",
]
)
@@ -37,6 +52,8 @@ class MiscSection(Section):
)
self.attribute_list = [
self.ENABLE_RESULT_HISTORY,
self.HISTORY_LENGTH,
self.HAPPY_MESSAGES,
self.MODIFY_GC,
self.ID_BITS

4
src/music_kraken/utils/enums/source.py
View File

@@ -25,6 +25,10 @@ class SourcePages(Enum):
TWITTER = "twitter" # I will use nitter though lol
MYSPACE = "myspace" # Yes somehow this ancient site is linked EVERYWHERE
MANUAL = "manual"
PRESET = "preset"
@classmethod
def get_homepage(cls, attribute) -> str:
homepage_map = {

11
src/music_kraken/utils/exception/download.py Normal file
View File

@@ -0,0 +1,11 @@
class DownloadException(Exception):
pass
class UrlNotFoundException(DownloadException):
def __init__(self, url: str, *args: object) -> None:
self.url = url
super().__init__(*args)
def __str__(self) -> str:
return f"Couldn't find the page of {self.url}"

2
src/music_kraken/utils/regex.py Normal file
View File

@@ -0,0 +1,2 @@
URL_PATTERN = 'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+'

38
src/music_kraken/utils/shared.py
View File

@@ -1,7 +1,8 @@
import logging
import random
from pathlib import Path
from typing import List, Tuple, Set
from typing import List, Tuple, Set, Dict
from urllib.parse import ParseResult
from .path_manager import LOCATIONS
from .config import LOGGING_SECTION, AUDIO_SECTION, CONNECTION_SECTION, MISC_SECTION, PATHS_SECTION
@@ -66,17 +67,9 @@ AUDIO_FORMAT = AUDIO_SECTION.AUDIO_FORMAT.object_from_value
DOWNLOAD_PATH = AUDIO_SECTION.DOWNLOAD_PATH.object_from_value
DOWNLOAD_FILE = AUDIO_SECTION.DOWNLOAD_FILE.object_from_value
DEFAULT_VALUES = {
"genre": AUDIO_SECTION.DEFAULT_GENRE.object_from_value,
"label": AUDIO_SECTION.DEFAULT_LABEL.object_from_value,
"artist": AUDIO_SECTION.DEFAULT_ARTIST.object_from_value,
"album": AUDIO_SECTION.DEFAULT_ALBUM.object_from_value,
"song": AUDIO_SECTION.DEFAULT_SONG.object_from_value,
"album_type": AUDIO_SECTION.DEFAULT_ALBUM_TYPE.object_from_value,
"audio_format": AUDIO_FORMAT
}
TOR: bool = CONNECTION_SECTION.USE_TOR.object_from_value
PROXIES_LIST: List[Dict[str, str]] = CONNECTION_SECTION.PROXIES.object_from_value
proxies = {}
if len(CONNECTION_SECTION.PROXIES) > 0:
"""
@@ -89,6 +82,11 @@ if TOR:
'http': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}',
'https': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}'
}
INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.object_from_value
PIPED_INSTANCE: ParseResult = CONNECTION_SECTION.PIPED_INSTANCE.object_from_value
ALL_YOUTUBE_URLS: List[ParseResult] = CONNECTION_SECTION.ALL_YOUTUBE_URLS.object_from_value
ENABLE_SPONSOR_BLOCK: bool = CONNECTION_SECTION.SPONSOR_BLOCK.object_from_value
# size of the chunks that are streamed
CHUNK_SIZE = CONNECTION_SECTION.CHUNK_SIZE.object_from_value
@@ -102,3 +100,23 @@ SORT_BY_DATE = AUDIO_SECTION.SORT_BY_DATE.object_from_value
SORT_BY_ALBUM_TYPE = AUDIO_SECTION.SORT_BY_ALBUM_TYPE.object_from_value
ALBUM_TYPE_BLACKLIST: Set[AlbumType] = set(AUDIO_SECTION.ALBUM_TYPE_BLACKLIST.object_from_value)
THREADED = False
ENABLE_RESULT_HISTORY: bool = MISC_SECTION.ENABLE_RESULT_HISTORY.object_from_value
HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value
HELP_MESSAGE = """
to search:
> s: {query or url}
> s: https://musify.club/release/some-random-release-183028492
> s: #a {artist} #r {release} #t {track}
to download:
> d: {option ids or direct url}
> d: 0, 3, 4
> d: 1
> d: https://musify.club/release/some-random-release-183028492
have fun :3
""".strip()

3
src/music_kraken/utils/support_classes/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from .download_result import DownloadResult
from .query import Query
from .thread_classes import EndThread, FinishedSearch

95
src/music_kraken/utils/support_classes/download_result.py Normal file
View File

@@ -0,0 +1,95 @@
from dataclasses import dataclass, field
from typing import List, Tuple
from ...utils.shared import SHOW_DOWNLOAD_ERRORS_THRESHOLD, DOWNLOAD_LOGGER as LOGGER
from ...objects import Target
UNIT_PREFIXES: List[str] = ["", "k", "m", "g", "t"]
UNIT_DIVISOR = 1024
@dataclass
class DownloadResult:
total: int = 0
fail: int = 0
sponsor_segments: int = 0
error_message: str = None
total_size = 0
found_on_disk: int = 0
_error_message_list: List[str] = field(default_factory=list)
@property
def success(self) -> int:
return self.total - self.fail
@property
def success_percentage(self) -> float:
if self.total == 0:
return 0
return self.success / self.total
@property
def failure_percentage(self) -> float:
if self.total == 0:
return 1
return self.fail / self.total
@property
def is_fatal_error(self) -> bool:
return self.error_message is not None
@property
def is_mild_failure(self) -> bool:
if self.is_fatal_error:
return True
return self.failure_percentage > SHOW_DOWNLOAD_ERRORS_THRESHOLD
def _size_val_unit_pref_ind(self, val: float, ind: int) -> Tuple[float, int]:
if val < UNIT_DIVISOR:
return val, ind
if ind >= len(UNIT_PREFIXES):
return val, ind
return self._size_val_unit_pref_ind(val=val / UNIT_DIVISOR, ind=ind + 1)
@property
def formated_size(self) -> str:
total_size, prefix_index = self._size_val_unit_pref_ind(self.total_size, 0)
return f"{total_size:.{2}f} {UNIT_PREFIXES[prefix_index]}B"
def add_target(self, target: Target):
self.total_size += target.size
def merge(self, other: "DownloadResult"):
if other.is_fatal_error:
LOGGER.debug(other.error_message)
self._error_message_list.append(other.error_message)
self.total += 1
self.fail += 1
else:
self.total += other.total
self.fail += other.fail
self._error_message_list.extend(other._error_message_list)
self.sponsor_segments += other.sponsor_segments
self.total_size += other.total_size
self.found_on_disk += other.found_on_disk
def __str__(self):
if self.is_fatal_error:
return self.error_message
head = f"{self.fail} from {self.total} downloads failed:\n" \
f"successrate:\t{int(self.success_percentage * 100)}%\n" \
f"failrate:\t{int(self.failure_percentage * 100)}%\n" \
f"total size:\t{self.formated_size}\n" \
f"skipped segments:\t{self.sponsor_segments}\n" \
f"found on disc:\t{self.found_on_disk}"
if not self.is_mild_failure:
return head
_lines = [head]
_lines.extend(self._error_message_list)
return "\n".join(_lines)

32
src/music_kraken/utils/support_classes/query.py Normal file
View File

@@ -0,0 +1,32 @@
from typing import Optional, List
from ...objects import DatabaseObject, Artist, Album, Song
class Query:
def __init__(
self,
raw_query: str = "",
music_object: DatabaseObject = None
) -> None:
self.raw_query: str = raw_query
self.music_object: Optional[DatabaseObject] = music_object
@property
def is_raw(self) -> bool:
return self.music_object is None
@property
def default_search(self) -> List[str]:
if self.music_object is None:
return [self.raw_query]
if isinstance(self.music_object, Artist):
return [self.music_object.name]
if isinstance(self.music_object, Song):
return [f"{artist.name} - {self.music_object}" for artist in self.music_object.main_artist_collection]
if isinstance(self.music_object, Album):
return [f"{artist.name} - {self.music_object}" for artist in self.music_object.artist_collection]
return [self.raw_query]

12
src/music_kraken/utils/support_classes/thread_classes.py Normal file
View File

@@ -0,0 +1,12 @@
class EndThread:
_has_ended: bool = False
def __bool__(self):
return self._has_ended
def exit(self):
self._has_ended
class FinishedSearch:
pass