From 804bf34a160e20d8262e51ba552e28b2d5fbcf6c Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 20 Apr 2023 15:34:03 +0200 Subject: [PATCH 01/81] s --- src/music_kraken/utils/config/connection.py | 39 +++++++++++++++++++++ src/music_kraken/utils/regex.py | 2 ++ 2 files changed, 41 insertions(+) create mode 100644 src/music_kraken/utils/regex.py diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index 086d927..2bd5824 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -1,4 +1,9 @@ +from urllib.parse import urlparse +import re + from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute +from ..regex import URL_PATTERN +from ..exception.config import SettingValueError class ProxAttribute(ListAttribute): @@ -9,6 +14,21 @@ class ProxAttribute(ListAttribute): 'ftp': value } +class UrlListAttribute(ListAttribute): + def validate(self, value: str): + v = value.strip() + url = re.match(URL_PATTERN, v) + if v != url: + raise SettingValueError( + setting_name=self.name, + setting_value=v, + rule="has to be a valid url" + ) + + def single_object_from_element(self, value: str): + return urlparse(value) + + class ConnectionSection(Section): def __init__(self): @@ -42,6 +62,25 @@ class ConnectionSection(Section): "all the error messages are shown.", value="0.3" ) + + # INVIDIOUS INSTANCES LIST + self.INVIDIOUS_INSTANCE_LIST = UrlListAttribute( + name="invidious_instances", + description="This is a List, where you can define the invidious instances,\n" + "the youtube downloader should use.\n" + "Here is a list of active ones: https://docs.invidious.io/instances/\n" + "Instances that use cloudflare or have source code changes could cause issues.\n" + "Hidden instances (.onion) will only work, when setting 'tor=true'.", + value=[ + "https://yt.artemislena.eu/", + "https://watch.thekitty.zone/", + "https://y.com.sb/" + ] + ) + # INVIDIOUS PROXY + self.INVIDIOUS_PROXY_VIDEOS = BoolAttribute( + name="invidious_proxy_video", + ) self.attribute_list = [ self.USE_TOR, diff --git a/src/music_kraken/utils/regex.py b/src/music_kraken/utils/regex.py new file mode 100644 index 0000000..1290de5 --- /dev/null +++ b/src/music_kraken/utils/regex.py @@ -0,0 +1,2 @@ +URL_PATTERN = 'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+' + From 3892f60c7f03e21c91d355f15dfc5fb27d04fe37 Mon Sep 17 00:00:00 2001 From: Hellow Date: Thu, 20 Apr 2023 15:36:12 +0200 Subject: [PATCH 02/81] completed most of album parsing --- src/music_kraken/pages/musify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 450e41a..50295dd 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -965,7 +965,7 @@ class Musify(Page): /html/musify/album_overview.html - [x] tracklist - - [ ] attributes + - [x] attributes - [ ] ratings :param stop_at_level: From d6d20aaf1bfd34ec4ecf1144d5191c2b08d46473 Mon Sep 17 00:00:00 2001 From: Hellow Date: Thu, 20 Apr 2023 18:47:47 +0200 Subject: [PATCH 03/81] improved function to connect to the internet --- src/music_kraken/connection/__init__.py | 0 src/music_kraken/connection/connection.py | 153 ++++++++++++++++++++ src/music_kraken/connection/rotating.py | 56 +++++++ src/music_kraken/utils/config/connection.py | 8 +- src/music_kraken/utils/shared.py | 3 +- 5 files changed, 216 insertions(+), 4 deletions(-) create mode 100644 src/music_kraken/connection/__init__.py create mode 100644 src/music_kraken/connection/connection.py create mode 100644 src/music_kraken/connection/rotating.py diff --git a/src/music_kraken/connection/__init__.py b/src/music_kraken/connection/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py new file mode 100644 index 0000000..c84779c --- /dev/null +++ b/src/music_kraken/connection/connection.py @@ -0,0 +1,153 @@ +from typing import List, Dict, Callable, Optional, Set +from urllib.parse import urlparse, urlunsplit +import logging + +import requests + +from .rotating import RotatingProxy +from ..utils.shared import PROXIES_LIST + +LOGGER = logging.getLogger("connection") + + +class Connection: + def __init__( + self, + host: str, + proxies: List[dict] = None, + tries: int = (len(PROXIES_LIST) + 1) * 2, + timeout: int = 7, + header_values: Dict[str, str] = None, + session: requests.Session = None, + accepted_response_codes: Set[int] = None, + semantic_not_found: bool = True + ): + if proxies is None: + proxies = PROXIES_LIST + if header_values is None: + header_values = dict() + + self.LOGGER = LOGGER + self.HOST = urlparse(host) + self.TRIES = tries + self.TIMEOUT = timeout + self.rotating_proxy = RotatingProxy(proxy_list=proxies) + + self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200} + self.SEMANTIC_NOT_FOUND = semantic_not_found + + self.session: requests.Session = session + if self.session is None: + self.new_session(**header_values) + else: + self.rotating_proxy.register_session(session) + self.set_header() + + @property + def base_url(self): + return urlunsplit((self.HOST.scheme, self.HOST.netloc, "", "", "")) + + def new_session(self, **header_values): + session = requests.Session() + session.headers = self.get_header(**header_values) + self.rotating_proxy.register_session(session) + self.session = session + + def get_header(self, **header_values) -> Dict[str, str]: + return { + "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", + "Connection": "keep-alive", + "Host": self.HOST.netloc, + "Referer": self.base_url, + **header_values + } + + def set_header(self, **header_values): + self.session.headers = self.get_header(**header_values) + + def rotate(self): + self.rotating_proxy.rotate() + + def _request( + self, + request: Callable, + try_count: int, + accepted_response_code: set, + url: str, + **kwargs + ) -> Optional[requests.Response]: + if try_count >= self.TRIES: + return + + retry = False + try: + r = request(url=url, **kwargs) + except requests.exceptions.Timeout: + self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})") + retry = True + except requests.exceptions.ConnectionError: + self.LOGGER.warning(f"Couldn't connect to \"{url}\": ({try_count}-{self.TRIES})") + retry = True + + if not retry: + if self.SEMANTIC_NOT_FOUND and r.status_code == 404: + self.LOGGER.warning(f"Couldn't find url (404): {url}") + return + if r.status_code in accepted_response_code: + return r + + if not retry: + self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " + f"at {url}. ({try_count}-{self.TRIES})") + self.LOGGER.debug(r.content) + + self.rotate() + + return self._request( + request=request, + try_count=try_count, + accepted_response_code=accepted_response_code, + url=url, + **kwargs + ) + + def get( + self, + url: str, + stream: bool = False, + accepted_response_codes: set = None, + **kwargs + ) -> Optional[requests.Response]: + r = self._request( + request=self.session.get, + try_count=0, + accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + url=url, + stream=stream, + **kwargs + ) + if r is None: + self.LOGGER.warning(f"Max attempts ({self.TRIES}) exceeded for: GET:{url}") + return r + + def post( + self, + url: str, + json: dict, + stream: bool = False, + accepted_response_codes: set = None, + **kwargs + ) -> Optional[requests.Response]: + r = self._request( + request=self.session.post, + try_count=0, + accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + url=url, + json=json, + stream=stream, + **kwargs + ) + if r is None: + self.LOGGER.warning(f"Max attempts ({self.TRIES}) exceeded for: GET:{url}") + self.LOGGER.warning(f"payload: {json}") + return r diff --git a/src/music_kraken/connection/rotating.py b/src/music_kraken/connection/rotating.py new file mode 100644 index 0000000..adb4010 --- /dev/null +++ b/src/music_kraken/connection/rotating.py @@ -0,0 +1,56 @@ +from typing import Dict, List + +import requests + + +class RotatingObject: + """ + This will be used for RotatingProxies and invidious instances. + """ + def __init__(self, object_list: list): + self._object_list: list = object_list + + if len(self._object_list) <= 0: + raise ValueError("There needs to be at least one item in a Rotating structure.") + + self._current_index = 0 + + @property + def object(self): + return self._object_list[self._current_index] + + def __len__(self): + return len(self._object_list) + + @property + def next(self): + self._current_index = (self._current_index + 1) % len(self._object_list) + + return self._object_list[self._current_index] + + +class RotatingProxy(RotatingObject): + def __init__(self, proxy_list: List[Dict[str, str]], session_list: List[requests.Session] = None): + self._session_list: List[requests.Session] = session_list + if self._session_list is None: + self._session_list = [] + + super().__init__(proxy_list if len(proxy_list) > 0 else [{}]) + + def register_session(self, session: requests.Session): + self._session_list.append(session) + session.proxies = self.current_proxy + + def rotate(self): + new_proxy = self.next + + for session in self._session_list: + session.proxies = new_proxy + + @property + def current_proxy(self) -> Dict[str, str]: + return super().object + + @property + def next(self) -> Dict[str, str]: + return super().object diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index 2bd5824..d6bae7a 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -14,6 +14,7 @@ class ProxAttribute(ListAttribute): 'ftp': value } + class UrlListAttribute(ListAttribute): def validate(self, value: str): v = value.strip() @@ -24,10 +25,9 @@ class UrlListAttribute(ListAttribute): setting_value=v, rule="has to be a valid url" ) - + def single_object_from_element(self, value: str): return urlparse(value) - class ConnectionSection(Section): @@ -62,7 +62,7 @@ class ConnectionSection(Section): "all the error messages are shown.", value="0.3" ) - + # INVIDIOUS INSTANCES LIST self.INVIDIOUS_INSTANCE_LIST = UrlListAttribute( name="invidious_instances", @@ -80,6 +80,8 @@ class ConnectionSection(Section): # INVIDIOUS PROXY self.INVIDIOUS_PROXY_VIDEOS = BoolAttribute( name="invidious_proxy_video", + value="false", + description="Downloads the videos using the given instances." ) self.attribute_list = [ diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 782a1e8..ef396e2 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -1,7 +1,7 @@ import logging import random from pathlib import Path -from typing import List, Tuple, Set +from typing import List, Tuple, Set, Dict from .path_manager import LOCATIONS from .config import LOGGING_SECTION, AUDIO_SECTION, CONNECTION_SECTION, MISC_SECTION, PATHS_SECTION @@ -77,6 +77,7 @@ DEFAULT_VALUES = { } TOR: bool = CONNECTION_SECTION.USE_TOR.object_from_value +PROXIES_LIST: List[Dict[str, str]] = CONNECTION_SECTION.PROXIES.object_from_value proxies = {} if len(CONNECTION_SECTION.PROXIES) > 0: """ From e3b1a866a1aa0160a1d539bea1c2dba6ade6c2e2 Mon Sep 17 00:00:00 2001 From: Hellow Date: Thu, 20 Apr 2023 19:37:41 +0200 Subject: [PATCH 04/81] theoretically implemented the request partially --- src/music_kraken/connection/__init__.py | 1 + src/music_kraken/connection/connection.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/music_kraken/connection/__init__.py b/src/music_kraken/connection/__init__.py index e69de29..f70650d 100644 --- a/src/music_kraken/connection/__init__.py +++ b/src/music_kraken/connection/__init__.py @@ -0,0 +1 @@ +from .connection import Connection diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index c84779c..3756475 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -7,8 +7,6 @@ import requests from .rotating import RotatingProxy from ..utils.shared import PROXIES_LIST -LOGGER = logging.getLogger("connection") - class Connection: def __init__( @@ -17,6 +15,7 @@ class Connection: proxies: List[dict] = None, tries: int = (len(PROXIES_LIST) + 1) * 2, timeout: int = 7, + logger: logging.Logger = logging.getLogger("connection"), header_values: Dict[str, str] = None, session: requests.Session = None, accepted_response_codes: Set[int] = None, @@ -27,7 +26,7 @@ class Connection: if header_values is None: header_values = dict() - self.LOGGER = LOGGER + self.LOGGER = logger self.HOST = urlparse(host) self.TRIES = tries self.TIMEOUT = timeout @@ -74,14 +73,18 @@ class Connection: try_count: int, accepted_response_code: set, url: str, + timeout: float, **kwargs ) -> Optional[requests.Response]: if try_count >= self.TRIES: return + if timeout is None: + timeout = self.TIMEOUT + retry = False try: - r = request(url=url, **kwargs) + r = request(url=url, timeout=timeout, **kwargs) except requests.exceptions.Timeout: self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})") retry = True @@ -116,6 +119,7 @@ class Connection: url: str, stream: bool = False, accepted_response_codes: set = None, + timeout: float = None, **kwargs ) -> Optional[requests.Response]: r = self._request( @@ -124,6 +128,7 @@ class Connection: accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, stream=stream, + timeout=timeout, **kwargs ) if r is None: @@ -136,6 +141,7 @@ class Connection: json: dict, stream: bool = False, accepted_response_codes: set = None, + timeout: float = None, **kwargs ) -> Optional[requests.Response]: r = self._request( @@ -143,6 +149,7 @@ class Connection: try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, + timeout=timeout, json=json, stream=stream, **kwargs From 9445e95ef90b2f10ca366b42770723278641b9e5 Mon Sep 17 00:00:00 2001 From: Hellow Date: Thu, 20 Apr 2023 19:45:29 +0200 Subject: [PATCH 05/81] sucessfully migrated metal archives --- src/music_kraken/pages/abstract.py | 3 ++ .../pages/encyclopaedia_metallum.py | 41 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 659c246..b43d73a 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -6,6 +6,7 @@ from typing import Optional, Union, Type, Dict, Set import requests from bs4 import BeautifulSoup +from ..connection import Connection from .support_classes.default_target import DefaultTarget from .support_classes.download_result import DownloadResult from ..objects import ( @@ -30,6 +31,8 @@ class Page: This is an abstract class, laying out the functionality for every other class fetching something """ + CONNECTION: Connection + API_SESSION: requests.Session = requests.Session() API_SESSION.proxies = shared.proxies TIMEOUT = 5 diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 70d2ffb..1592b91 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -5,6 +5,7 @@ from bs4 import BeautifulSoup import pycountry from urllib.parse import urlparse +from ..connection import Connection from ..utils.shared import ENCYCLOPAEDIA_METALLUM_LOGGER, proxies from ..utils import string_processing from .abstract import Page @@ -24,12 +25,10 @@ from ..objects import ( class EncyclopaediaMetallum(Page): - API_SESSION: requests.Session = requests.Session() - API_SESSION.proxies = proxies - API_SESSION.headers = { - "Host": "www.metal-archives.com", - "Connection": "keep-alive" - } + CONNECTION: Connection = Connection( + host="https://www.metal-archives.com/", + logger=ENCYCLOPAEDIA_METALLUM_LOGGER + ) SOURCE_TYPE = SourcePages.ENCYCLOPAEDIA_METALLUM @@ -70,10 +69,10 @@ class EncyclopaediaMetallum(Page): "&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&mDataProp_4=4&_" \ "=1674550595663" - r = cls.get_request(endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)) - if r.status_code != 200: - cls.LOGGER.warning( - f"code {r.status_code} at {endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)}") + r = cls.CONNECTION.get( + endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str) + ) + if r is None: return [] return [cls.get_song_from_json( @@ -92,10 +91,8 @@ class EncyclopaediaMetallum(Page): "=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=1&iColumns=3&sColumns" \ "=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&_=1674563943747" - r = cls.get_request(endpoint.format(artist=query.artist_str, album=query.album_str)) - if r.status_code != 200: - cls.LOGGER.warning( - f"code {r.status_code} at {endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str)}") + r = cls.CONNECTION.get(endpoint.format(artist=query.artist_str, album=query.album_str)) + if r is None: return [] return [cls.get_album_from_json( @@ -111,7 +108,7 @@ class EncyclopaediaMetallum(Page): "=&bandLabelName=&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0" \ "&mDataProp_1=1&mDataProp_2=2&_=1674565459976" - r = cls.get_request(endpoint.format(artist=query.artist)) + r = cls.CONNECTION.get(endpoint.format(artist=query.artist)) if r is None: return [] @@ -134,7 +131,7 @@ class EncyclopaediaMetallum(Page): """ endpoint = "https://www.metal-archives.com/search/ajax-band-search/?field=name&query={query}&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2" - r = cls.get_request(endpoint.format(query=query)) + r = cls.CONNECTION.get(endpoint.format(query=query)) if r is None: return [] @@ -220,7 +217,7 @@ class EncyclopaediaMetallum(Page): discography_url = "https://www.metal-archives.com/band/discography/id/{}/tab/all" # make the request - r = cls.get_request(discography_url.format(ma_artist_id)) + r = cls.CONNECTION.get(discography_url.format(ma_artist_id)) if r is None: return [] soup = cls.get_soup_from_response(r) @@ -257,7 +254,7 @@ class EncyclopaediaMetallum(Page): @classmethod def _fetch_artist_sources(cls, ma_artist_id: str) -> List[Source]: sources_url = "https://www.metal-archives.com/link/ajax-list/type/band/id/{}" - r = cls.get_request(sources_url.format(ma_artist_id)) + r = cls.CONNECTION.get(sources_url.format(ma_artist_id)) if r is None: return [] @@ -405,7 +402,7 @@ class EncyclopaediaMetallum(Page): @classmethod def _fetch_artist_attributes(cls, url: str) -> Artist: - r = cls.get_request(url) + r = cls.CONNECTION.get(url) if r is None: return Artist() soup: BeautifulSoup = cls.get_soup_from_response(r) @@ -417,7 +414,7 @@ class EncyclopaediaMetallum(Page): endpoint = "https://www.metal-archives.com/band/read-more/id/{}" # make the request - r = cls.get_request(endpoint.format(ma_artist_id)) + r = cls.CONNECTION.get(endpoint.format(ma_artist_id)) if r is None: return FormattedText() @@ -570,7 +567,7 @@ class EncyclopaediaMetallum(Page): # Date: Thu, 20 Apr 2023 22:30:45 +0200 Subject: [PATCH 06/81] Update musify.py --- src/music_kraken/pages/musify.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 50295dd..ba7f513 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -8,6 +8,7 @@ import pycountry import requests from bs4 import BeautifulSoup +from ..connection import Connection from .abstract import Page from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType, AlbumStatus @@ -22,6 +23,7 @@ from ..objects import ( Options, Target ) +from ..utils.shared import MUSIFY_LOGGER from ..utils import string_processing, shared from .support_classes.download_result import DownloadResult @@ -76,6 +78,11 @@ class Musify(Page): TRIES = 5 HOST = "https://musify.club" + CONNECTION = Connection( + host="https://musify.club/", + logger=MUSIFY_LOGGER + ) + SOURCE_TYPE = SourcePages.MUSIFY LOGGER = shared.MUSIFY_LOGGER @@ -355,7 +362,7 @@ class Musify(Page): def plaintext_search(cls, query: str) -> Options: search_results = [] - r = cls.get_request(f"https://musify.club/search?searchText={query}") + r = cls.CONNECTION.get(f"https://musify.club/search?searchText={query}") if r is None: return Options() search_soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser") @@ -580,7 +587,7 @@ class Musify(Page): :return: """ - r = cls.get_request(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent") + r = cls.CONNECTION.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent") if r is None: return Artist(_id=url.musify_id) @@ -976,7 +983,7 @@ class Musify(Page): url = cls.parse_url(source.url) endpoint = cls.HOST + "/release/" + url.name_with_id - r = cls.get_request(endpoint) + r = cls.CONNECTION.get(endpoint) if r is None: return Album() @@ -1032,7 +1039,7 @@ class Musify(Page): cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.get_request(endpoint, stream=True) + r = cls.CONNECTION.get(endpoint, stream=True) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") From 12ef9eb3dd1a5df7a369d942ddc1cde790de5e3e Mon Sep 17 00:00:00 2001 From: Hellow Date: Sat, 22 Apr 2023 14:20:19 +0200 Subject: [PATCH 07/81] sda --- documentation/connections.md | 7 +++ src/music_kraken/connection/connection.py | 73 ++++++++++++++++------- src/music_kraken/pages/musify.py | 2 +- 3 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 documentation/connections.md diff --git a/documentation/connections.md b/documentation/connections.md new file mode 100644 index 0000000..9e58e8f --- /dev/null +++ b/documentation/connections.md @@ -0,0 +1,7 @@ +# Connections + +## Functions + +A class, that gives me the options, to make web request + + diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 3756475..b9b1363 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -1,5 +1,5 @@ from typing import List, Dict, Callable, Optional, Set -from urllib.parse import urlparse, urlunsplit +from urllib.parse import urlparse, urlunsplit, ParseResult import logging import requests @@ -17,7 +17,6 @@ class Connection: timeout: int = 7, logger: logging.Logger = logging.getLogger("connection"), header_values: Dict[str, str] = None, - session: requests.Session = None, accepted_response_codes: Set[int] = None, semantic_not_found: bool = True ): @@ -26,6 +25,8 @@ class Connection: if header_values is None: header_values = dict() + self.HEADER_VALUES = header_values + self.LOGGER = logger self.HOST = urlparse(host) self.TRIES = tries @@ -35,38 +36,63 @@ class Connection: self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200} self.SEMANTIC_NOT_FOUND = semantic_not_found - self.session: requests.Session = session - if self.session is None: - self.new_session(**header_values) - else: - self.rotating_proxy.register_session(session) - self.set_header() + self._session_map: Dict[str] = { + self.HOST.netloc: self.new_session() + } - @property - def base_url(self): - return urlunsplit((self.HOST.scheme, self.HOST.netloc, "", "", "")) + def base_url(self, url: ParseResult = None): + if url is None: + url = self.HOST - def new_session(self, **header_values): - session = requests.Session() + return urlunsplit((url.scheme, url.netloc, "", "", "")) + + def _register_session(self, session: requests.Session, **header_values): session.headers = self.get_header(**header_values) self.rotating_proxy.register_session(session) - self.session = session + + def new_session( + self, + url: ParseResult = None, + refer_from_origin: bool = True + ) -> requests.Session: + + header_values = self.HEADER_VALUES.copy() + if url is not None: + header_values["Host"] = url.netloc + + if not refer_from_origin: + header_values["Referer"] = self.base_url(url=url) + + session = requests.Session() + self._register_session(session=session, **header_values) + + return session def get_header(self, **header_values) -> Dict[str, str]: return { "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", "Connection": "keep-alive", "Host": self.HOST.netloc, - "Referer": self.base_url, + "Referer": self.base_url(), **header_values } - def set_header(self, **header_values): - self.session.headers = self.get_header(**header_values) - def rotate(self): self.rotating_proxy.rotate() + def get_session_from_url(self, url: str, refer_from_origin: bool = True) -> requests.Session: + parsed_url = urlparse(url) + + if parsed_url.netloc in self._session_map: + print("saved session") + return self._session_map[parsed_url.netloc] + + self._session_map[parsed_url.netloc] = self.new_session( + url=parsed_url, + refer_from_origin=refer_from_origin + ) + return self._session_map[parsed_url.netloc] + def _request( self, request: Callable, @@ -99,6 +125,9 @@ class Connection: if r.status_code in accepted_response_code: return r + print(r.content) + print(r.headers) + if not retry: self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " f"at {url}. ({try_count}-{self.TRIES})") @@ -111,19 +140,22 @@ class Connection: try_count=try_count, accepted_response_code=accepted_response_code, url=url, + timeout=timeout, **kwargs ) def get( self, url: str, + refer_from_origin: bool = True, stream: bool = False, accepted_response_codes: set = None, timeout: float = None, **kwargs ) -> Optional[requests.Response]: + s = self.get_session_from_url(url, refer_from_origin) r = self._request( - request=self.session.get, + request=s.get, try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, @@ -139,13 +171,14 @@ class Connection: self, url: str, json: dict, + refer_from_origin: bool = True, stream: bool = False, accepted_response_codes: set = None, timeout: float = None, **kwargs ) -> Optional[requests.Response]: r = self._request( - request=self.session.post, + request=self.get_session_from_url(url, refer_from_origin).post, try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index ba7f513..049785e 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -1039,7 +1039,7 @@ class Musify(Page): cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.CONNECTION.get(endpoint, stream=True) + r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Host": "40s.musify.club", "Referer": endpoint}) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") From bd3e042ae228d5affa9b8dbc01cf089bf81b51d1 Mon Sep 17 00:00:00 2001 From: Hellow Date: Sun, 23 Apr 2023 12:08:39 +0200 Subject: [PATCH 08/81] failed attempts --- src/actual_donwload.py | 4 +- src/music_kraken/connection/connection.py | 85 +++++++++---------- src/music_kraken/connection/rotating.py | 25 ++---- .../pages/download_center/search.py | 3 +- src/music_kraken/pages/musify.py | 2 +- src/music_kraken/utils/enums/source.py | 2 + 6 files changed, 54 insertions(+), 67 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 35ab53f..5341779 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -53,7 +53,7 @@ def real_download(): if __name__ == "__main__": music_kraken.cli(genre="test", command_list=[ - "#a Molchat Doma", - "0", + # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", + "https://musify.club/release/ghost-bath-self-loather-2021-1554266", "ok" ]) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index b9b1363..8c1a1e2 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -36,9 +36,9 @@ class Connection: self.ACCEPTED_RESPONSE_CODES = accepted_response_codes or {200} self.SEMANTIC_NOT_FOUND = semantic_not_found - self._session_map: Dict[str] = { - self.HOST.netloc: self.new_session() - } + self.session = requests.Session() + self.session.headers = self.get_header(**self.HEADER_VALUES) + self.session.proxies = self.rotating_proxy.current_proxy def base_url(self, url: ParseResult = None): if url is None: @@ -46,31 +46,9 @@ class Connection: return urlunsplit((url.scheme, url.netloc, "", "", "")) - def _register_session(self, session: requests.Session, **header_values): - session.headers = self.get_header(**header_values) - self.rotating_proxy.register_session(session) - - def new_session( - self, - url: ParseResult = None, - refer_from_origin: bool = True - ) -> requests.Session: - - header_values = self.HEADER_VALUES.copy() - if url is not None: - header_values["Host"] = url.netloc - - if not refer_from_origin: - header_values["Referer"] = self.base_url(url=url) - - session = requests.Session() - self._register_session(session=session, **header_values) - - return session - def get_header(self, **header_values) -> Dict[str, str]: return { - "user-agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", + "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Connection": "keep-alive", "Host": self.HOST.netloc, "Referer": self.base_url(), @@ -78,20 +56,21 @@ class Connection: } def rotate(self): - self.rotating_proxy.rotate() + self.session.proxies = self.rotating_proxy.rotate() - def get_session_from_url(self, url: str, refer_from_origin: bool = True) -> requests.Session: - parsed_url = urlparse(url) + def _update_headers( + self, + headers: Optional[dict], + refer_from_origin: bool, + url: ParseResult + ) -> Dict[str, str]: + if headers is None: + headers = dict() - if parsed_url.netloc in self._session_map: - print("saved session") - return self._session_map[parsed_url.netloc] + if not refer_from_origin: + headers["Referer"] = self.base_url(url=url) - self._session_map[parsed_url.netloc] = self.new_session( - url=parsed_url, - refer_from_origin=refer_from_origin - ) - return self._session_map[parsed_url.netloc] + return headers def _request( self, @@ -100,6 +79,8 @@ class Connection: accepted_response_code: set, url: str, timeout: float, + headers: dict, + refer_from_origin: bool = True, **kwargs ) -> Optional[requests.Response]: if try_count >= self.TRIES: @@ -108,9 +89,20 @@ class Connection: if timeout is None: timeout = self.TIMEOUT + parsed_url = urlparse(url) + + print(url) + print(parsed_url) + + headers = self._update_headers( + headers=headers, + refer_from_origin=refer_from_origin, + url=parsed_url + ) + retry = False try: - r = request(url=url, timeout=timeout, **kwargs) + r: requests.Response = request(url=parsed_url.geturl(), timeout=timeout, headers=headers, **kwargs) except requests.exceptions.Timeout: self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})") retry = True @@ -121,19 +113,19 @@ class Connection: if not retry: if self.SEMANTIC_NOT_FOUND and r.status_code == 404: self.LOGGER.warning(f"Couldn't find url (404): {url}") + print(r.headers) + print(r.request.headers) return if r.status_code in accepted_response_code: return r - print(r.content) - print(r.headers) - if not retry: self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " f"at {url}. ({try_count}-{self.TRIES})") self.LOGGER.debug(r.content) self.rotate() + print(r.headers) return self._request( request=request, @@ -151,16 +143,18 @@ class Connection: stream: bool = False, accepted_response_codes: set = None, timeout: float = None, + headers: dict = None, **kwargs ) -> Optional[requests.Response]: - s = self.get_session_from_url(url, refer_from_origin) r = self._request( - request=s.get, + request=self.session.get, try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, stream=stream, timeout=timeout, + headers=headers, + refer_from_origin=refer_from_origin, **kwargs ) if r is None: @@ -175,14 +169,17 @@ class Connection: stream: bool = False, accepted_response_codes: set = None, timeout: float = None, + headers: dict = None, **kwargs ) -> Optional[requests.Response]: r = self._request( - request=self.get_session_from_url(url, refer_from_origin).post, + request=self.session.post, try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, timeout=timeout, + headers=headers, + refer_from_origin=refer_from_origin, json=json, stream=stream, **kwargs diff --git a/src/music_kraken/connection/rotating.py b/src/music_kraken/connection/rotating.py index adb4010..3b9c6bf 100644 --- a/src/music_kraken/connection/rotating.py +++ b/src/music_kraken/connection/rotating.py @@ -30,27 +30,14 @@ class RotatingObject: class RotatingProxy(RotatingObject): - def __init__(self, proxy_list: List[Dict[str, str]], session_list: List[requests.Session] = None): - self._session_list: List[requests.Session] = session_list - if self._session_list is None: - self._session_list = [] + def __init__(self, proxy_list: List[Dict[str, str]]): + super().__init__( + proxy_list if len(proxy_list) > 0 else [None] + ) - super().__init__(proxy_list if len(proxy_list) > 0 else [{}]) - - def register_session(self, session: requests.Session): - self._session_list.append(session) - session.proxies = self.current_proxy - - def rotate(self): - new_proxy = self.next - - for session in self._session_list: - session.proxies = new_proxy + def rotate(self) -> Dict[str, str]: + return self.next @property def current_proxy(self) -> Dict[str, str]: return super().object - - @property - def next(self) -> Dict[str, str]: - return super().object diff --git a/src/music_kraken/pages/download_center/search.py b/src/music_kraken/pages/download_center/search.py index 915fa21..f1c685c 100644 --- a/src/music_kraken/pages/download_center/search.py +++ b/src/music_kraken/pages/download_center/search.py @@ -6,6 +6,7 @@ from .multiple_options import MultiPageOptions from ..abstract import Page from ..support_classes.download_result import DownloadResult from ...objects import DatabaseObject, Source +from ...utils.enums.source import SourcePages class Search(Download): @@ -116,7 +117,7 @@ class Search(Download): can download directly after """ - source = Source.match_url(url=url) + source = Source.match_url(url=url, referer_page=SourcePages.MANUAL) if source is None: return False diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 049785e..48de5f5 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -1039,7 +1039,7 @@ class Musify(Page): cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Host": "40s.musify.club", "Referer": endpoint}) + r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Connection": "https://musify.club/"}) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") diff --git a/src/music_kraken/utils/enums/source.py b/src/music_kraken/utils/enums/source.py index b536e5c..b324f8d 100644 --- a/src/music_kraken/utils/enums/source.py +++ b/src/music_kraken/utils/enums/source.py @@ -25,6 +25,8 @@ class SourcePages(Enum): TWITTER = "twitter" # I will use nitter though lol MYSPACE = "myspace" # Yes somehow this ancient site is linked EVERYWHERE + MANUAL = "manual" + @classmethod def get_homepage(cls, attribute) -> str: homepage_map = { From c4e1095dfc299203381d40a26d78762a5eea8c31 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 3 May 2023 14:43:08 +0200 Subject: [PATCH 09/81] Update abstract.py --- src/music_kraken/pages/abstract.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index b43d73a..afe6e24 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -1,7 +1,7 @@ import logging import random from copy import copy -from typing import Optional, Union, Type, Dict, Set +from typing import Optional, Union, Type, Dict, Set, List import requests from bs4 import BeautifulSoup @@ -154,6 +154,10 @@ class Page: album_str = property(fget=lambda self: self.get_str(self.album)) song_str = property(fget=lambda self: self.get_str(self.song)) + @classmethod + def search_by_object(cls, data_object: DatabaseObject, filter_none: bool = True) -> List[DatabaseObject]: + return [] + @classmethod def search_by_query(cls, query: str) -> Options: """ From fa7efbe2c84ff8fa429d15aee057cd6776249c38 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 3 May 2023 15:16:01 +0200 Subject: [PATCH 10/81] dsf --- src/music_kraken/connection/connection.py | 15 +++++++++++---- src/music_kraken/pages/musify.py | 2 +- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 8c1a1e2..d5e1a28 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -81,6 +81,7 @@ class Connection: timeout: float, headers: dict, refer_from_origin: bool = True, + raw_url: bool = False, **kwargs ) -> Optional[requests.Response]: if try_count >= self.TRIES: @@ -100,19 +101,21 @@ class Connection: url=parsed_url ) + request_url = parsed_url.geturl() if not raw_url else url + retry = False try: - r: requests.Response = request(url=parsed_url.geturl(), timeout=timeout, headers=headers, **kwargs) + r: requests.Response = request(url=request_url, timeout=timeout, headers=headers, **kwargs) except requests.exceptions.Timeout: - self.LOGGER.warning(f"Request timed out at \"{url}\": ({try_count}-{self.TRIES})") + self.LOGGER.warning(f"Request timed out at \"{request_url}\": ({try_count}-{self.TRIES})") retry = True except requests.exceptions.ConnectionError: - self.LOGGER.warning(f"Couldn't connect to \"{url}\": ({try_count}-{self.TRIES})") + self.LOGGER.warning(f"Couldn't connect to \"{request_url}\": ({try_count}-{self.TRIES})") retry = True if not retry: if self.SEMANTIC_NOT_FOUND and r.status_code == 404: - self.LOGGER.warning(f"Couldn't find url (404): {url}") + self.LOGGER.warning(f"Couldn't find url (404): {request_url}") print(r.headers) print(r.request.headers) return @@ -144,6 +147,7 @@ class Connection: accepted_response_codes: set = None, timeout: float = None, headers: dict = None, + raw_url: bool = False, **kwargs ) -> Optional[requests.Response]: r = self._request( @@ -155,6 +159,7 @@ class Connection: timeout=timeout, headers=headers, refer_from_origin=refer_from_origin, + raw_url=raw_url, **kwargs ) if r is None: @@ -170,6 +175,7 @@ class Connection: accepted_response_codes: set = None, timeout: float = None, headers: dict = None, + raw_url: bool = False, **kwargs ) -> Optional[requests.Response]: r = self._request( @@ -182,6 +188,7 @@ class Connection: refer_from_origin=refer_from_origin, json=json, stream=stream, + raw_url=raw_url, **kwargs ) if r is None: diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 48de5f5..d73e3b9 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -1039,7 +1039,7 @@ class Musify(Page): cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Connection": "https://musify.club/"}) + r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Connection": "https://musify.club/"}, raw_url=True) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") From 82ddc4f5d6bccea26b3606b2968b5c9172639d93 Mon Sep 17 00:00:00 2001 From: Hellow Date: Wed, 3 May 2023 16:14:03 +0200 Subject: [PATCH 11/81] fixed nasty bug, where musify just returned 404 al lthe time --- src/actual_donwload.py | 4 +-- src/music_kraken/connection/connection.py | 41 ++++++++++------------- src/music_kraken/pages/musify.py | 2 +- 3 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 5341779..5c3cb71 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -53,7 +53,7 @@ def real_download(): if __name__ == "__main__": music_kraken.cli(genre="test", command_list=[ - # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", - "https://musify.club/release/ghost-bath-self-loather-2021-1554266", + "https://musify.club/release/molchat-doma-etazhi-2018-1092949", + # "https://musify.club/release/ghost-bath-self-loather-2021-1554266", "ok" ]) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index d5e1a28..70c77ea 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -50,7 +50,7 @@ class Connection: return { "user-agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36", "Connection": "keep-alive", - "Host": self.HOST.netloc, + # "Host": self.HOST.netloc, "Referer": self.base_url(), **header_values } @@ -92,9 +92,6 @@ class Connection: parsed_url = urlparse(url) - print(url) - print(parsed_url) - headers = self._update_headers( headers=headers, refer_from_origin=refer_from_origin, @@ -103,32 +100,30 @@ class Connection: request_url = parsed_url.geturl() if not raw_url else url - retry = False + connection_failed = False try: - r: requests.Response = request(url=request_url, timeout=timeout, headers=headers, **kwargs) - except requests.exceptions.Timeout: - self.LOGGER.warning(f"Request timed out at \"{request_url}\": ({try_count}-{self.TRIES})") - retry = True - except requests.exceptions.ConnectionError: - self.LOGGER.warning(f"Couldn't connect to \"{request_url}\": ({try_count}-{self.TRIES})") - retry = True + r: requests.Response = request(request_url, timeout=timeout, headers=headers, **kwargs) - if not retry: - if self.SEMANTIC_NOT_FOUND and r.status_code == 404: - self.LOGGER.warning(f"Couldn't find url (404): {request_url}") - print(r.headers) - print(r.request.headers) - return if r.status_code in accepted_response_code: return r - if not retry: + if self.SEMANTIC_NOT_FOUND and r.status_code == 404: + self.LOGGER.warning(f"Couldn't find url (404): {request_url}") + return None + + except requests.exceptions.Timeout: + self.LOGGER.warning(f"Request timed out at \"{request_url}\": ({try_count}-{self.TRIES})") + connection_failed = True + except requests.exceptions.ConnectionError: + self.LOGGER.warning(f"Couldn't connect to \"{request_url}\": ({try_count}-{self.TRIES})") + connection_failed = True + + if not connection_failed: self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " f"at {url}. ({try_count}-{self.TRIES})") self.LOGGER.debug(r.content) self.rotate() - print(r.headers) return self._request( request=request, @@ -155,11 +150,11 @@ class Connection: try_count=0, accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, - stream=stream, timeout=timeout, headers=headers, - refer_from_origin=refer_from_origin, raw_url=raw_url, + refer_from_origin=refer_from_origin, + stream=stream, **kwargs ) if r is None: @@ -186,9 +181,9 @@ class Connection: timeout=timeout, headers=headers, refer_from_origin=refer_from_origin, + raw_url=raw_url, json=json, stream=stream, - raw_url=raw_url, **kwargs ) if r is None: diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index d73e3b9..4c1ce04 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -1039,7 +1039,7 @@ class Musify(Page): cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.CONNECTION.get(endpoint, stream=True, allow_redirects=True, headers={"Connection": "https://musify.club/"}, raw_url=True) + r = cls.CONNECTION.get(endpoint, stream=True, raw_url=True) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") From 855ab269f2fdfde63c370f1e84c473a3f593dfe4 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 4 May 2023 15:11:00 +0200 Subject: [PATCH 12/81] Create youtube.md --- documentation/html/youtube/youtube.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 documentation/html/youtube/youtube.md diff --git a/documentation/html/youtube/youtube.md b/documentation/html/youtube/youtube.md new file mode 100644 index 0000000..07e1504 --- /dev/null +++ b/documentation/html/youtube/youtube.md @@ -0,0 +1,19 @@ +# YouTube + +I can get data from youtube with either calling: + +- invidious +- youtube + +I will only get the structured data from youtube music => `{artist} - Topic`. Maybe other stuff will be implemented + +## Functionality + +Every search results, where the channel name doesnt end with ` - Topic` will be ignored. + +### ISRC search + +### YT music + +If searchin for artist, the query should be: +`{artist} - Topic` From 07e2ef3543738d697b0d97ecaf5709af5f4ffe85 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 09:51:14 +0200 Subject: [PATCH 13/81] addet program structure --- documentation/program_structure.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 documentation/program_structure.md diff --git a/documentation/program_structure.md b/documentation/program_structure.md new file mode 100644 index 0000000..42fdc4b --- /dev/null +++ b/documentation/program_structure.md @@ -0,0 +1,22 @@ +# Downloading + +## Query + +- parsing query into music objects + +# Pages + +## from music objects to query + +``` +song: artist1, artist2 + +1. artist1 - song +2. artist2 - song +``` + +``` +artist: song1, song2 + +1. artist +``` From 08339bab68d07ebbd86126ac1863844e31c1d590 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 10:25:12 +0200 Subject: [PATCH 14/81] query function --- src/music_kraken/pages/abstract.py | 4 +-- .../support_classes/__init__.py | 0 .../support_classes/default_target.py | 0 .../support_classes/download_result.py | 0 .../utils/support_classes/query.py | 28 +++++++++++++++++++ 5 files changed, 30 insertions(+), 2 deletions(-) rename src/music_kraken/{pages => utils}/support_classes/__init__.py (100%) rename src/music_kraken/{pages => utils}/support_classes/default_target.py (100%) rename src/music_kraken/{pages => utils}/support_classes/download_result.py (100%) create mode 100644 src/music_kraken/utils/support_classes/query.py diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index afe6e24..0d69604 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -7,8 +7,8 @@ import requests from bs4 import BeautifulSoup from ..connection import Connection -from .support_classes.default_target import DefaultTarget -from .support_classes.download_result import DownloadResult +from ..utils.support_classes.default_target import DefaultTarget +from ..utils.support_classes.download_result import DownloadResult from ..objects import ( Song, Source, diff --git a/src/music_kraken/pages/support_classes/__init__.py b/src/music_kraken/utils/support_classes/__init__.py similarity index 100% rename from src/music_kraken/pages/support_classes/__init__.py rename to src/music_kraken/utils/support_classes/__init__.py diff --git a/src/music_kraken/pages/support_classes/default_target.py b/src/music_kraken/utils/support_classes/default_target.py similarity index 100% rename from src/music_kraken/pages/support_classes/default_target.py rename to src/music_kraken/utils/support_classes/default_target.py diff --git a/src/music_kraken/pages/support_classes/download_result.py b/src/music_kraken/utils/support_classes/download_result.py similarity index 100% rename from src/music_kraken/pages/support_classes/download_result.py rename to src/music_kraken/utils/support_classes/download_result.py diff --git a/src/music_kraken/utils/support_classes/query.py b/src/music_kraken/utils/support_classes/query.py new file mode 100644 index 0000000..dd492d6 --- /dev/null +++ b/src/music_kraken/utils/support_classes/query.py @@ -0,0 +1,28 @@ +from typing import Optional, List + +from ...objects import DatabaseObject, Artist, Album, Song + +class Query: + def __init__( + self, + raw_query: str = "", + music_object: DatabaseObject = None + ) -> None: + self.raw_query: str = raw_query + self.music_object: Optional[DatabaseObject] = None + + @property + def default_search(self) -> List[str]: + if self.music_object is None: + return [self.raw_query] + + if isinstance(self.music_object, Artist): + return [self.music_object.name] + + if isinstance(self.music_object, Song): + return [f"{artist.name} - {self.music_object}" for artist in self.music_object.main_artist_collection] + + if isinstance(self.music_object, Album): + return [f"{artist.name} - {self.music_object}" for artist in self.music_object.artist_collection] + + return [self.raw_query] From 2a00735f720c2e69c2c14b4e48b4d1438ee6b734 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 10:49:52 +0200 Subject: [PATCH 15/81] refactor --- src/metal_archives.py | 2 +- src/music_kraken/pages/abstract.py | 128 ++---------------- .../pages/download_center/search.py | 2 +- .../pages/encyclopaedia_metallum.py | 6 +- src/music_kraken/pages/musify.py | 4 +- src/music_kraken/pages/preset.py | 2 +- .../utils/support_classes/__init__.py | 3 + src/musify_search.py | 2 +- 8 files changed, 23 insertions(+), 126 deletions(-) diff --git a/src/metal_archives.py b/src/metal_archives.py index 33b6b50..e603849 100644 --- a/src/metal_archives.py +++ b/src/metal_archives.py @@ -3,7 +3,7 @@ from music_kraken.pages import EncyclopaediaMetallum def search(): - results = EncyclopaediaMetallum.search_by_query("#a Ghost Bath") + results = EncyclopaediaMetallum._raw_search("#a Ghost Bath") print(results) print(results[0].source_collection) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 0d69604..d5502e3 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -7,8 +7,6 @@ import requests from bs4 import BeautifulSoup from ..connection import Connection -from ..utils.support_classes.default_target import DefaultTarget -from ..utils.support_classes.download_result import DownloadResult from ..objects import ( Song, Source, @@ -24,6 +22,7 @@ from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType from ..audio import write_metadata_to_target, correct_codec from ..utils import shared +from ..utils.support_classes import Query, DownloadResult, DefaultTarget class Page: @@ -41,125 +40,22 @@ class Page: LOGGER = logging.getLogger("this shouldn't be used") SOURCE_TYPE: SourcePages - - @classmethod - def get_request( - cls, - url: str, - stream: bool = False, - accepted_response_codes: set = {200}, - trie: int = 0 - ) -> Optional[requests.Response]: - - retry = False - try: - r = cls.API_SESSION.get(url, timeout=cls.TIMEOUT, stream=stream) - except requests.exceptions.Timeout: - cls.LOGGER.warning(f"request timed out at \"{url}\": ({trie}-{cls.TRIES})") - retry = True - except requests.exceptions.ConnectionError: - cls.LOGGER.warning(f"couldn't connect to \"{url}\": ({trie}-{cls.TRIES})") - retry = True - - if not retry and r.status_code in accepted_response_codes: - return r - - if not retry: - cls.LOGGER.warning(f"{cls.__name__} responded wit {r.status_code} at GET:{url}. ({trie}-{cls.TRIES})") - cls.LOGGER.debug(r.content) - - if trie >= cls.TRIES: - cls.LOGGER.warning("to many tries. Aborting.") - return None - - return cls.get_request(url=url, stream=stream, accepted_response_codes=accepted_response_codes, trie=trie + 1) - - @classmethod - def post_request(cls, url: str, json: dict, accepted_response_codes: set = {200}, trie: int = 0) -> Optional[ - requests.Response]: - retry = False - try: - r = cls.API_SESSION.post(url, json=json, timeout=cls.POST_TIMEOUT) - except requests.exceptions.Timeout: - cls.LOGGER.warning(f"request timed out at \"{url}\": ({trie}-{cls.TRIES})") - retry = True - except requests.exceptions.ConnectionError: - cls.LOGGER.warning(f"couldn't connect to \"{url}\": ({trie}-{cls.TRIES})") - retry = True - - if not retry and r.status_code in accepted_response_codes: - return r - - if not retry: - cls.LOGGER.warning(f"{cls.__name__} responded wit {r.status_code} at POST:{url}. ({trie}-{cls.TRIES})") - cls.LOGGER.debug(r.content) - - if trie >= cls.TRIES: - cls.LOGGER.warning("to many tries. Aborting.") - return None - - cls.LOGGER.warning(f"payload: {json}") - return cls.post_request(url=url, json=json, accepted_response_codes=accepted_response_codes, trie=trie + 1) - + @classmethod def get_soup_from_response(cls, r: requests.Response) -> BeautifulSoup: return BeautifulSoup(r.content, "html.parser") - class Query: - def __init__(self, query: str): - self.query = query - self.is_raw = False - - self.artist = None - self.album = None - self.song = None - - self.parse_query(query=query) - - def __str__(self): - if self.is_raw: - return self.query - return f"{self.artist}; {self.album}; {self.song}" - - def parse_query(self, query: str): - if not '#' in query: - self.is_raw = True - return - - query = query.strip() - parameters = query.split('#') - parameters.remove('') - - for parameter in parameters: - splitted = parameter.split(" ") - type_ = splitted[0] - input_ = " ".join(splitted[1:]).strip() - - if type_ == "a": - self.artist = input_ - continue - if type_ == "r": - self.album = input_ - continue - if type_ == "t": - self.song = input_ - continue - - def get_str(self, string): - if string is None: - return "" - return string - - artist_str = property(fget=lambda self: self.get_str(self.artist)) - album_str = property(fget=lambda self: self.get_str(self.album)) - song_str = property(fget=lambda self: self.get_str(self.song)) + @classmethod + def search(cls, query: Query) -> Options: + results = [] + + for default_query in query.default_search: + results.extend(cls._raw_search(default_query)) + + return Options(results) @classmethod - def search_by_object(cls, data_object: DatabaseObject, filter_none: bool = True) -> List[DatabaseObject]: - return [] - - @classmethod - def search_by_query(cls, query: str) -> Options: + def _raw_search(cls, query: str) -> Options: """ # The Query You can define a new parameter with "#", @@ -174,7 +70,7 @@ class Page: :return possible_music_objects: """ - return Options() + return [] @classmethod def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: diff --git a/src/music_kraken/pages/download_center/search.py b/src/music_kraken/pages/download_center/search.py index f1c685c..10d8433 100644 --- a/src/music_kraken/pages/download_center/search.py +++ b/src/music_kraken/pages/download_center/search.py @@ -67,7 +67,7 @@ class Search(Download): """ for page in self.pages: - self._current_option[page] = page.search_by_query(query=query) + self._current_option[page] = page._raw_search(query=query) def choose_page(self, page: Type[Page]): """ diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 1592b91..54435a5 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -1,13 +1,11 @@ from collections import defaultdict from typing import List, Optional, Dict, Type, Union -import requests from bs4 import BeautifulSoup import pycountry from urllib.parse import urlparse from ..connection import Connection -from ..utils.shared import ENCYCLOPAEDIA_METALLUM_LOGGER, proxies -from ..utils import string_processing +from ..utils.shared import ENCYCLOPAEDIA_METALLUM_LOGGER from .abstract import Page from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType @@ -45,7 +43,7 @@ class EncyclopaediaMetallum(Page): LOGGER = ENCYCLOPAEDIA_METALLUM_LOGGER @classmethod - def search_by_query(cls, query: str) -> Options: + def _raw_search(cls, query: str) -> Options: query_obj = cls.Query(query) if query_obj.is_raw: diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 4c1ce04..305d147 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -25,7 +25,7 @@ from ..objects import ( ) from ..utils.shared import MUSIFY_LOGGER from ..utils import string_processing, shared -from .support_classes.download_result import DownloadResult +from ..utils.support_classes import DownloadResult """ https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent @@ -112,7 +112,7 @@ class Musify(Page): ) @classmethod - def search_by_query(cls, query: str) -> Options: + def _raw_search(cls, query: str) -> Options: query_obj = cls.Query(query) if query_obj.is_raw: diff --git a/src/music_kraken/pages/preset.py b/src/music_kraken/pages/preset.py index 826e3f2..333b265 100644 --- a/src/music_kraken/pages/preset.py +++ b/src/music_kraken/pages/preset.py @@ -46,7 +46,7 @@ class YouTube(Page): @classmethod - def search_by_query(cls, query: str) -> Options: + def _raw_search(cls, query: str) -> Options: return Options() @classmethod diff --git a/src/music_kraken/utils/support_classes/__init__.py b/src/music_kraken/utils/support_classes/__init__.py index e69de29..625ed41 100644 --- a/src/music_kraken/utils/support_classes/__init__.py +++ b/src/music_kraken/utils/support_classes/__init__.py @@ -0,0 +1,3 @@ +from .default_target import DefaultTarget +from .download_result import DownloadResult +from .query import Query diff --git a/src/musify_search.py b/src/musify_search.py index ad79908..09fdf5f 100644 --- a/src/musify_search.py +++ b/src/musify_search.py @@ -3,7 +3,7 @@ from music_kraken.pages import Musify def search(): - results = Musify.search_by_query("#a Ghost Bath") + results = Musify._raw_search("#a Ghost Bath") print(results) From da0fa2c8dcc88b5b7d48dec8a1dc426e228dae3d Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 10:55:44 +0200 Subject: [PATCH 16/81] refactor --- src/music_kraken/__init__.py | 6 +++--- src/music_kraken/download/__init__.py | 2 ++ .../{pages/download_center => download}/download.py | 0 .../{pages/download_center => download}/multiple_options.py | 0 .../{pages/download_center => download}/page_attributes.py | 0 .../{pages/download_center => download}/search.py | 0 src/music_kraken/pages/__init__.py | 5 ----- src/music_kraken/pages/download_center/__init__.py | 5 ----- 8 files changed, 5 insertions(+), 13 deletions(-) create mode 100644 src/music_kraken/download/__init__.py rename src/music_kraken/{pages/download_center => download}/download.py (100%) rename src/music_kraken/{pages/download_center => download}/multiple_options.py (100%) rename src/music_kraken/{pages/download_center => download}/page_attributes.py (100%) rename src/music_kraken/{pages/download_center => download}/search.py (100%) delete mode 100644 src/music_kraken/pages/download_center/__init__.py diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index b549f8c..f4debe2 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -6,7 +6,7 @@ from typing import List import gc import musicbrainzngs -from . import objects, pages +from . import objects, pages, download from .utils import exception, shared, path_manager from .utils.config import config, read, write, PATHS_SECTION from .utils.shared import MUSIC_DIR, MODIFY_GC, NOT_A_GENRE_REGEX, get_random_message @@ -176,7 +176,7 @@ def cli( if verification in agree_inputs: return new_genre - def next_search(_search: pages.Search, query: str) -> bool: + def next_search(_search: download.Search, query: str) -> bool: """ :param _search: :param query: @@ -234,7 +234,7 @@ def cli( print(f"Downloading to: \"{genre}\"") print() - search = pages.Search() + search = download.Search() # directly download url if direct_download_url is not None: diff --git a/src/music_kraken/download/__init__.py b/src/music_kraken/download/__init__.py new file mode 100644 index 0000000..e670c65 --- /dev/null +++ b/src/music_kraken/download/__init__.py @@ -0,0 +1,2 @@ +from .search import Search +from .download import Download diff --git a/src/music_kraken/pages/download_center/download.py b/src/music_kraken/download/download.py similarity index 100% rename from src/music_kraken/pages/download_center/download.py rename to src/music_kraken/download/download.py diff --git a/src/music_kraken/pages/download_center/multiple_options.py b/src/music_kraken/download/multiple_options.py similarity index 100% rename from src/music_kraken/pages/download_center/multiple_options.py rename to src/music_kraken/download/multiple_options.py diff --git a/src/music_kraken/pages/download_center/page_attributes.py b/src/music_kraken/download/page_attributes.py similarity index 100% rename from src/music_kraken/pages/download_center/page_attributes.py rename to src/music_kraken/download/page_attributes.py diff --git a/src/music_kraken/pages/download_center/search.py b/src/music_kraken/download/search.py similarity index 100% rename from src/music_kraken/pages/download_center/search.py rename to src/music_kraken/download/search.py diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index 3794526..0caea8f 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -3,8 +3,3 @@ from .musify import Musify EncyclopaediaMetallum = EncyclopaediaMetallum Musify = Musify - -from . import download_center - -Search = download_center.Search - diff --git a/src/music_kraken/pages/download_center/__init__.py b/src/music_kraken/pages/download_center/__init__.py deleted file mode 100644 index 5adfb95..0000000 --- a/src/music_kraken/pages/download_center/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from . import search -from . import download - -Download = download.Download -Search = search.Search From ab7f4141255662a5d4f5d14ce35b6c01ca962494 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 13:59:24 +0200 Subject: [PATCH 17/81] hs --- src/music_kraken/download/download.py | 4 +- src/music_kraken/download/multiple_options.py | 4 +- src/music_kraken/download/page_attributes.py | 7 +- src/music_kraken/download/search.py | 75 +++++++++++++++++-- src/music_kraken/objects/__init__.py | 2 +- src/music_kraken/objects/option.py | 9 ++- src/music_kraken/pages/__init__.py | 4 +- .../pages/encyclopaedia_metallum.py | 11 +-- src/music_kraken/pages/musify.py | 4 +- 9 files changed, 94 insertions(+), 26 deletions(-) diff --git a/src/music_kraken/download/download.py b/src/music_kraken/download/download.py index d5e3902..febdbf6 100644 --- a/src/music_kraken/download/download.py +++ b/src/music_kraken/download/download.py @@ -1,8 +1,8 @@ from typing import Optional, Tuple, Type, Set, Union, List from . import page_attributes -from ..abstract import Page -from ...objects import Song, Album, Artist, Label, Source +from ..pages import Page +from ..objects import Song, Album, Artist, Label, Source MusicObject = Union[Song, Album, Artist, Label] diff --git a/src/music_kraken/download/multiple_options.py b/src/music_kraken/download/multiple_options.py index 8326a66..d2d99c0 100644 --- a/src/music_kraken/download/multiple_options.py +++ b/src/music_kraken/download/multiple_options.py @@ -2,8 +2,8 @@ from collections import defaultdict from typing import Tuple, List, Dict, Type from . import page_attributes -from ..abstract import Page -from ...objects import Options, DatabaseObject, Source +from ..pages import Page +from ..objects import Options, DatabaseObject, Source class MultiPageOptions: diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 7eb040a..56e18a4 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -1,9 +1,8 @@ from typing import Tuple, Type, Dict -from ...utils.enums.source import SourcePages -from ..abstract import Page -from ..encyclopaedia_metallum import EncyclopaediaMetallum -from ..musify import Musify +from ..utils.enums.source import SourcePages +from ..pages import Page, EncyclopaediaMetallum, Musify + NAME_PAGE_MAP: Dict[str, Type[Page]] = dict() PAGE_NAME_MAP: Dict[Type[Page], str] = dict() diff --git a/src/music_kraken/download/search.py b/src/music_kraken/download/search.py index 10d8433..f7a0ddf 100644 --- a/src/music_kraken/download/search.py +++ b/src/music_kraken/download/search.py @@ -1,12 +1,12 @@ -from typing import Tuple, List, Set, Type, Optional +from typing import Tuple, List, Set, Type, Optional, Dict from . import page_attributes from .download import Download from .multiple_options import MultiPageOptions -from ..abstract import Page -from ..support_classes.download_result import DownloadResult -from ...objects import DatabaseObject, Source -from ...utils.enums.source import SourcePages +from ..pages.abstract import Page +from ..utils.support_classes import DownloadResult, Query +from ..objects import DatabaseObject, Source, Artist, Song, Album +from ..utils.enums.source import SourcePages class Search(Download): @@ -52,6 +52,24 @@ class Search(Download): self._current_option = self._option_history[-1] return self._option_history[-1] + + def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: + song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) + album = None if not "r" in key_text else Album(title=key_text["r"], dynamic=True) + artist = None if not "a" in key_text else Artist(name=key_text["a"], dynamic=True) + + if song is not None: + song.album_collection.append(album) + song.main_artist_collection.append(artist) + return Query(raw_query=query, music_object=song) + + if album is not None: + album.artist_collection.append(artist) + return Query(raw_query=query, music_object=album) + + if artist is not None: + return Query(raw_query=query, music_object=artist) + def search(self, query: str): """ @@ -65,9 +83,54 @@ class Search(Download): doesn't set derived_from thus, can't download right after """ + + special_characters = "#\\" + query = query + " " + + key_text = {} + + skip_next = False + escape_next = False + new_text = "" + latest_key: str = None + for i in range(len(query) - 1): + currenct_char = query[i] + next_char = query[i+1] + + if skip_next: + skip_next = False + continue + + if escape_next: + new_text += currenct_char + escape_next = False + + # escaping + if currenct_char == "\\": + if next_char in special_characters: + escape_next = True + continue + + if currenct_char == "#": + if latest_key is not None: + key_text[latest_key] + + latest_key = next_char + skip_next = True + continue + + new_text += currenct_char + + if latest_key is not None: + key_text[latest_key] = new_text + + + parsed_query: Query = self._process_parsed(key_text, query) + for page in self.pages: - self._current_option[page] = page._raw_search(query=query) + for search in parsed_query.default_search: + self._current_option[page].extend(page._raw_search(query=search)) def choose_page(self, page: Type[Page]): """ diff --git a/src/music_kraken/objects/__init__.py b/src/music_kraken/objects/__init__.py index 374d871..2018550 100644 --- a/src/music_kraken/objects/__init__.py +++ b/src/music_kraken/objects/__init__.py @@ -1,4 +1,5 @@ from ..utils.enums import album +from .option import Options from . import ( song, metadata, @@ -28,5 +29,4 @@ Album = song.Album FormattedText = formatted_text.FormattedText -Options = option.Options Collection = collection.Collection diff --git a/src/music_kraken/objects/option.py b/src/music_kraken/objects/option.py index 977c08d..3af24cf 100644 --- a/src/music_kraken/objects/option.py +++ b/src/music_kraken/objects/option.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, List, Iterable if TYPE_CHECKING: from .parents import DatabaseObject @@ -14,6 +14,13 @@ class Options: def __iter__(self): for database_object in self._data: yield database_object + + def append(self, element: 'DatabaseObject'): + self._data.append(element) + + def extend(self, iterable: Iterable['DatabaseObject']): + for element in iterable: + self.append(element) def get_next_options(self, index: int) -> 'Options': if index >= len(self._data): diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index 0caea8f..0423f94 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -1,5 +1,3 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum from .musify import Musify - -EncyclopaediaMetallum = EncyclopaediaMetallum -Musify = Musify +from .abstract import Page diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 54435a5..553b0b3 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -9,6 +9,7 @@ from ..utils.shared import ENCYCLOPAEDIA_METALLUM_LOGGER from .abstract import Page from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType +from ..utils.support_classes import Query from ..objects import ( Lyrics, Artist, @@ -51,7 +52,7 @@ class EncyclopaediaMetallum(Page): return cls.advanced_search(query_obj) @classmethod - def advanced_search(cls, query: Page.Query) -> Options: + def advanced_search(cls, query: Query) -> Options: if query.song is not None: return Options(cls.search_for_song(query=query)) if query.album is not None: @@ -61,7 +62,7 @@ class EncyclopaediaMetallum(Page): return Options @classmethod - def search_for_song(cls, query: Page.Query) -> List[Song]: + def search_for_song(cls, query: Query) -> List[Song]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/songs/?songTitle={song}&bandName={" \ "artist}&releaseTitle={album}&lyrics=&genre=&sEcho=1&iColumns=5&sColumns=&iDisplayStart=0" \ "&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&mDataProp_4=4&_" \ @@ -82,7 +83,7 @@ class EncyclopaediaMetallum(Page): ) for raw_song in r.json()['aaData']] @classmethod - def search_for_album(cls, query: Page.Query) -> List[Album]: + def search_for_album(cls, query: Query) -> List[Album]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/albums/?bandName={" \ "artist}&releaseTitle={album}&releaseYearFrom=&releaseMonthFrom=&releaseYearTo=&releaseMonthTo" \ "=&country=&location=&releaseLabelName=&releaseCatalogNumber=&releaseIdentifiers" \ @@ -100,7 +101,7 @@ class EncyclopaediaMetallum(Page): ) for raw_album in r.json()['aaData']] @classmethod - def search_for_artist(cls, query: Page.Query) -> List[Artist]: + def search_for_artist(cls, query: Query) -> List[Artist]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/bands/?bandName={" \ "artist}&genre=&country=&yearCreationFrom=&yearCreationTo=&bandNotes=&status=&themes=&location" \ "=&bandLabelName=&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0" \ @@ -122,7 +123,7 @@ class EncyclopaediaMetallum(Page): ] @classmethod - def simple_search(cls, query: Page.Query) -> List[Artist]: + def simple_search(cls, query: Query) -> List[Artist]: """ Searches the default endpoint from metal archives, which intern searches only for bands, but it is the default, thus I am rolling with it diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 305d147..90a6976 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -25,7 +25,7 @@ from ..objects import ( ) from ..utils.shared import MUSIFY_LOGGER from ..utils import string_processing, shared -from ..utils.support_classes import DownloadResult +from ..utils.support_classes import DownloadResult, Query """ https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent @@ -120,7 +120,7 @@ class Musify(Page): return cls.plaintext_search(cls.get_plaintext_query(query_obj)) @classmethod - def get_plaintext_query(cls, query: Page.Query) -> str: + def get_plaintext_query(cls, query: Query) -> str: if query.album is None: return f"{query.artist or '*'} - {query.song or '*'}" return f"{query.artist or '*'} - {query.album or '*'} - {query.song or '*'}" From 67b4a3bc650f48180f22d27b6f2541cf0e80667c Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 16:21:12 +0200 Subject: [PATCH 18/81] cleaned up search --- src/music_kraken/pages/abstract.py | 77 +++++++++++-------- .../pages/encyclopaedia_metallum.py | 53 +++++-------- .../utils/support_classes/query.py | 6 +- 3 files changed, 72 insertions(+), 64 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index d5502e3..4e5495f 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -2,6 +2,7 @@ import logging import random from copy import copy from typing import Optional, Union, Type, Dict, Set, List +import threading import requests from bs4 import BeautifulSoup @@ -25,52 +26,66 @@ from ..utils import shared from ..utils.support_classes import Query, DownloadResult, DefaultTarget -class Page: +class Page(threading.Thread): """ This is an abstract class, laying out the functionality for every other class fetching something """ - CONNECTION: Connection - - API_SESSION: requests.Session = requests.Session() - API_SESSION.proxies = shared.proxies - TIMEOUT = 5 - POST_TIMEOUT = TIMEOUT - TRIES = 5 - LOGGER = logging.getLogger("this shouldn't be used") SOURCE_TYPE: SourcePages + LOGGER = logging.getLogger("this shouldn't be used") + def __init__(self): + threading.Thread.__init__(self) + @classmethod def get_soup_from_response(cls, r: requests.Response) -> BeautifulSoup: return BeautifulSoup(r.content, "html.parser") - @classmethod - def search(cls, query: Query) -> Options: + # to search stuff + def search(self, query: Query) -> List[DatabaseObject]: results = [] - for default_query in query.default_search: - results.extend(cls._raw_search(default_query)) + if query.is_raw: + for search_query in query.default_search: + results.extend(self.general_search(search_query)) + return results - return Options(results) - - @classmethod - def _raw_search(cls, query: str) -> Options: - """ - # The Query - You can define a new parameter with "#", - the letter behind it defines the *type* of parameter, followed by a space - "#a Psychonaut 4 #r Tired, Numb and #t Drop by Drop" - if no # is in the query it gets treated as "unspecified query" - - # Functionality - Returns the best matches from this page for the query, passed in. - - :param query: - :return possible_music_objects: - """ - + music_object = query.music_object + + search_functions = { + Song: self.song_search, + Album: self.album_search, + Artist: self.artist_search, + Label: self.label_search + } + + if type(music_object) in search_functions: + r = search_functions[type(music_object)](music_object) + if len(r) > 0: + return r + + r = [] + for default_query in query.default_search: + results.extend(self.general_search(default_query)) + + return results + + def general_search(self, search_query: str) -> List[DatabaseObject]: return [] + + def label_search(self, label: Label) -> List[Label]: + return [] + + def artist_search(self, artist: Artist) -> List[Artist]: + return [] + + def album_search(self, album: Album) -> List[Album]: + return [] + + def song_search(self, song: Song) -> List[Song]: + return [] + @classmethod def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 553b0b3..364ff85 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -23,6 +23,17 @@ from ..objects import ( ) +ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, { + "Full-length": AlbumType.STUDIO_ALBUM, + "Single": AlbumType.SINGLE, + "EP": AlbumType.EP, + "Demo": AlbumType.DEMO, + "Video": AlbumType.OTHER, + "Live album": AlbumType.LIVE_ALBUM, + "Compilation": AlbumType.COMPILATION_ALBUM +}) + + class EncyclopaediaMetallum(Page): CONNECTION: Connection = Connection( host="https://www.metal-archives.com/", @@ -30,36 +41,14 @@ class EncyclopaediaMetallum(Page): ) SOURCE_TYPE = SourcePages.ENCYCLOPAEDIA_METALLUM - - ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, { - "Full-length": AlbumType.STUDIO_ALBUM, - "Single": AlbumType.SINGLE, - "EP": AlbumType.EP, - "Demo": AlbumType.DEMO, - "Video": AlbumType.OTHER, - "Live album": AlbumType.LIVE_ALBUM, - "Compilation": AlbumType.COMPILATION_ALBUM - }) LOGGER = ENCYCLOPAEDIA_METALLUM_LOGGER - - @classmethod - def _raw_search(cls, query: str) -> Options: - query_obj = cls.Query(query) - - if query_obj.is_raw: - return cls.simple_search(query_obj) - return cls.advanced_search(query_obj) - - @classmethod - def advanced_search(cls, query: Query) -> Options: - if query.song is not None: - return Options(cls.search_for_song(query=query)) - if query.album is not None: - return Options(cls.search_for_album(query=query)) - if query.artist is not None: - return Options(cls.search_for_artist(query=query)) - return Options + + def __init__(self): + self.connection: Connection = Connection( + host="https://www.metal-archives.com/", + logger=ENCYCLOPAEDIA_METALLUM_LOGGER + ) @classmethod def search_for_song(cls, query: Query) -> List[Song]: @@ -123,7 +112,7 @@ class EncyclopaediaMetallum(Page): ] @classmethod - def simple_search(cls, query: Query) -> List[Artist]: + def _raw_search(cls, query: str) -> Options: """ Searches the default endpoint from metal archives, which intern searches only for bands, but it is the default, thus I am rolling with it @@ -132,12 +121,12 @@ class EncyclopaediaMetallum(Page): r = cls.CONNECTION.get(endpoint.format(query=query)) if r is None: - return [] + return Options() - return [ + return Options([ cls.get_artist_from_json(artist_html=raw_artist[0], genre=raw_artist[1], country=raw_artist[2]) for raw_artist in r.json()['aaData'] - ] + ]) @classmethod def get_artist_from_json(cls, artist_html=None, genre=None, country=None) -> Artist: diff --git a/src/music_kraken/utils/support_classes/query.py b/src/music_kraken/utils/support_classes/query.py index dd492d6..239096b 100644 --- a/src/music_kraken/utils/support_classes/query.py +++ b/src/music_kraken/utils/support_classes/query.py @@ -9,7 +9,11 @@ class Query: music_object: DatabaseObject = None ) -> None: self.raw_query: str = raw_query - self.music_object: Optional[DatabaseObject] = None + self.music_object: Optional[DatabaseObject] = music_object + + @property + def is_raw(self) -> bool: + return self.music_object is None @property def default_search(self) -> List[str]: From 4cc36087b677c3e5d12c7b5c9ec255fd911191d2 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 23 May 2023 16:30:35 +0200 Subject: [PATCH 19/81] cleaned up search --- .../pages/encyclopaedia_metallum.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 364ff85..3675102 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -49,16 +49,25 @@ class EncyclopaediaMetallum(Page): host="https://www.metal-archives.com/", logger=ENCYCLOPAEDIA_METALLUM_LOGGER ) + + super().__init__() - @classmethod - def search_for_song(cls, query: Query) -> List[Song]: + def song_search(self, song: Song) -> List[Song]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/songs/?songTitle={song}&bandName={" \ "artist}&releaseTitle={album}&lyrics=&genre=&sEcho=1&iColumns=5&sColumns=&iDisplayStart=0" \ "&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&mDataProp_4=4&_" \ "=1674550595663" - r = cls.CONNECTION.get( - endpoint.format(song=query.song_str, artist=query.artist_str, album=query.album_str) + """ + The difficult question I am facing is, that if I try every artist, with every song, with every album, + I end up with a quadratic runtime complecety O(n^2), where every step means one web request. + + This. + Is not good. + """ + + r = self.connection.get( + endpoint.format(song=song.title, artist=query.artist_str, album=query.album_str) ) if r is None: return [] From 70c7d831c9f1bd1ef68e7b0eb76ce64973a2668b Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 16:50:54 +0200 Subject: [PATCH 20/81] rewritten specific search for me --- src/music_kraken/pages/abstract.py | 11 +- .../pages/encyclopaedia_metallum.py | 215 +++++++++--------- 2 files changed, 115 insertions(+), 111 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 4e5495f..8398caa 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -44,13 +44,6 @@ class Page(threading.Thread): # to search stuff def search(self, query: Query) -> List[DatabaseObject]: - results = [] - - if query.is_raw: - for search_query in query.default_search: - results.extend(self.general_search(search_query)) - return results - music_object = query.music_object search_functions = { @@ -67,9 +60,9 @@ class Page(threading.Thread): r = [] for default_query in query.default_search: - results.extend(self.general_search(default_query)) + r.extend(self.general_search(default_query)) - return results + return r def general_search(self, search_query: str) -> List[DatabaseObject]: return [] diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 3675102..a7e79ed 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -34,14 +34,80 @@ ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, { }) -class EncyclopaediaMetallum(Page): - CONNECTION: Connection = Connection( - host="https://www.metal-archives.com/", - logger=ENCYCLOPAEDIA_METALLUM_LOGGER +def _song_from_json(artist_html=None, album_html=None, release_type=None, title=None, + lyrics_html=None) -> Song: + song_id = None + if lyrics_html is not None: + soup = BeautifulSoup(lyrics_html, 'html.parser') + anchor = soup.find('a') + raw_song_id = anchor.get('id') + song_id = raw_song_id.replace("lyricsLink_", "") + + return Song( + title=title, + main_artist_list=[ + _artist_from_json(artist_html=artist_html) + ], + album_list=[ + _album_from_json(album_html=album_html, release_type=release_type, artist_html=artist_html) + ], + source_list=[ + Source(SourcePages.ENCYCLOPAEDIA_METALLUM, song_id) + ] ) + +def _artist_from_json(artist_html=None, genre=None, country=None) -> Artist: + """ + TODO parse the country to a standart + """ + # parse the html + # parse the html for the band name and link on metal-archives + soup = BeautifulSoup(artist_html, 'html.parser') + anchor = soup.find('a') + artist_name = anchor.text + artist_url = anchor.get('href') + artist_id = artist_url.split("/")[-1] + + anchor.decompose() + strong = soup.find('strong') + if strong is not None: + strong.decompose() + akronyms_ = soup.text[2:-2].split(', ') + + return Artist( + name=artist_name, + source_list=[ + Source(SourcePages.ENCYCLOPAEDIA_METALLUM, artist_url) + ] + ) + + +def _album_from_json(album_html=None, release_type=None, artist_html=None) -> Album: + # parse the html + # Self Loather' + soup = BeautifulSoup(album_html, 'html.parser') + anchor = soup.find('a') + album_name = anchor.text + album_url = anchor.get('href') + album_id = album_url.split("/")[-1] + + album_type = ALBUM_TYPE_MAP[release_type.strip()] + + return Album( + title=album_name, + album_type=album_type, + source_list=[ + Source(SourcePages.ENCYCLOPAEDIA_METALLUM, album_url) + ], + artist_list=[ + _artist_from_json(artist_html=artist_html) + ] + ) + + +class EncyclopaediaMetallum(Page): SOURCE_TYPE = SourcePages.ENCYCLOPAEDIA_METALLUM - LOGGER = ENCYCLOPAEDIA_METALLUM_LOGGER def __init__(self): @@ -66,46 +132,62 @@ class EncyclopaediaMetallum(Page): Is not good. """ - r = self.connection.get( - endpoint.format(song=song.title, artist=query.artist_str, album=query.album_str) - ) - if r is None: - return [] + song_title = song.title + album_titles = ["*"] if song.album_collection.empty else [album.title for album in song.album_collection] + artist_titles = ["*"] if song.main_artist_collection.empty else [artist.name for artist in song.main_artist_collection] - return [cls.get_song_from_json( - artist_html=raw_song[0], - album_html=raw_song[1], - release_type=raw_song[2], - title=raw_song[3], - lyrics_html=raw_song[4] - ) for raw_song in r.json()['aaData']] + search_results = [] - @classmethod - def search_for_album(cls, query: Query) -> List[Album]: + for artist in artist_titles: + for album in album_titles: + r = self.connection.get( + endpoint.format(song=song_title, artist=artist, album=album) + ) + + if r is None: + return [] + + search_results.extend(_song_from_json( + artist_html=raw_song[0], + album_html=raw_song[1], + release_type=raw_song[2], + title=raw_song[3], + lyrics_html=raw_song[4] + ) for raw_song in r.json()['aaData']) + + return search_results + + def album_search(self, album: Album) -> List[Album]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/albums/?bandName={" \ "artist}&releaseTitle={album}&releaseYearFrom=&releaseMonthFrom=&releaseYearTo=&releaseMonthTo" \ "=&country=&location=&releaseLabelName=&releaseCatalogNumber=&releaseIdentifiers" \ "=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=1&iColumns=3&sColumns" \ "=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&_=1674563943747" - r = cls.CONNECTION.get(endpoint.format(artist=query.artist_str, album=query.album_str)) - if r is None: - return [] - return [cls.get_album_from_json( - artist_html=raw_album[0], - album_html=raw_album[1], - release_type=raw_album[2] - ) for raw_album in r.json()['aaData']] + album_title = album.title + artist_titles = ["*"] if album.artist_collection.empty else [artist.name for artist in album.artist_collection] - @classmethod - def search_for_artist(cls, query: Query) -> List[Artist]: + search_results = [] + + for artist in artist_titles: + r = self.connection.get(endpoint.format(artist=artist, album=album_title)) + if r is None: + return [] + + search_results.extend(_album_from_json( + artist_html=raw_album[0], + album_html=raw_album[1], + release_type=raw_album[2] + ) for raw_album in r.json()['aaData']) + + def artist_search(self, artist: Artist) -> List[Artist]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/bands/?bandName={" \ "artist}&genre=&country=&yearCreationFrom=&yearCreationTo=&bandNotes=&status=&themes=&location" \ "=&bandLabelName=&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0" \ "&mDataProp_1=1&mDataProp_2=2&_=1674565459976" - r = cls.CONNECTION.get(endpoint.format(artist=query.artist)) + r = self.connection.get(endpoint.format(artist=artist.name)) if r is None: return [] @@ -116,7 +198,7 @@ class EncyclopaediaMetallum(Page): return [] return [ - cls.get_artist_from_json(artist_html=raw_artist[0], genre=raw_artist[1], country=raw_artist[2]) + _artist_from_json(artist_html=raw_artist[0], genre=raw_artist[1], country=raw_artist[2]) for raw_artist in r.json()['aaData'] ] @@ -137,77 +219,6 @@ class EncyclopaediaMetallum(Page): for raw_artist in r.json()['aaData'] ]) - @classmethod - def get_artist_from_json(cls, artist_html=None, genre=None, country=None) -> Artist: - """ - TODO parse the country to a standart - """ - # parse the html - # parse the html for the band name and link on metal-archives - soup = BeautifulSoup(artist_html, 'html.parser') - anchor = soup.find('a') - artist_name = anchor.text - artist_url = anchor.get('href') - artist_id = artist_url.split("/")[-1] - - anchor.decompose() - strong = soup.find('strong') - if strong is not None: - strong.decompose() - akronyms_ = soup.text[2:-2].split(', ') - - return Artist( - name=artist_name, - source_list=[ - Source(SourcePages.ENCYCLOPAEDIA_METALLUM, artist_url) - ] - ) - - @classmethod - def get_album_from_json(cls, album_html=None, release_type=None, artist_html=None) -> Album: - # parse the html - # Self Loather' - soup = BeautifulSoup(album_html, 'html.parser') - anchor = soup.find('a') - album_name = anchor.text - album_url = anchor.get('href') - album_id = album_url.split("/")[-1] - - album_type = cls.ALBUM_TYPE_MAP[release_type.strip()] - - return Album( - title=album_name, - album_type=album_type, - source_list=[ - Source(SourcePages.ENCYCLOPAEDIA_METALLUM, album_url) - ], - artist_list=[ - cls.get_artist_from_json(artist_html=artist_html) - ] - ) - - @classmethod - def get_song_from_json(cls, artist_html=None, album_html=None, release_type=None, title=None, - lyrics_html=None) -> Song: - song_id = None - if lyrics_html is not None: - soup = BeautifulSoup(lyrics_html, 'html.parser') - anchor = soup.find('a') - raw_song_id = anchor.get('id') - song_id = raw_song_id.replace("lyricsLink_", "") - - return Song( - title=title, - main_artist_list=[ - cls.get_artist_from_json(artist_html=artist_html) - ], - album_list=[ - cls.get_album_from_json(album_html=album_html, release_type=release_type, artist_html=artist_html) - ], - source_list=[ - Source(SourcePages.ENCYCLOPAEDIA_METALLUM, song_id) - ] - ) @classmethod def _fetch_artist_discography(cls, ma_artist_id: str) -> List[Album]: From 70e0a16b201f5176e963966f8282173914f7dcc3 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 16:53:07 +0200 Subject: [PATCH 21/81] rewritten general search for me --- src/music_kraken/pages/encyclopaedia_metallum.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index a7e79ed..012351a 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -20,6 +20,7 @@ from ..objects import ( FormattedText, Label, Options, + DatabaseObject ) @@ -202,22 +203,21 @@ class EncyclopaediaMetallum(Page): for raw_artist in r.json()['aaData'] ] - @classmethod - def _raw_search(cls, query: str) -> Options: + def general_search(self, query: str) -> List[DatabaseObject]: """ Searches the default endpoint from metal archives, which intern searches only for bands, but it is the default, thus I am rolling with it """ endpoint = "https://www.metal-archives.com/search/ajax-band-search/?field=name&query={query}&sEcho=1&iColumns=3&sColumns=&iDisplayStart=0&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2" - r = cls.CONNECTION.get(endpoint.format(query=query)) + r = self.connection.get(endpoint.format(query=query)) if r is None: - return Options() + return [] - return Options([ - cls.get_artist_from_json(artist_html=raw_artist[0], genre=raw_artist[1], country=raw_artist[2]) + return [ + _artist_from_json(artist_html=raw_artist[0], genre=raw_artist[1], country=raw_artist[2]) for raw_artist in r.json()['aaData'] - ]) + ] @classmethod From 8500b714a2b6f0960a3957462d6eac8e0cc3da91 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 17:53:03 +0200 Subject: [PATCH 22/81] rewritten general search for me --- src/music_kraken/download/download.py | 15 +++++++----- src/music_kraken/download/multiple_options.py | 4 ++-- src/music_kraken/download/page_attributes.py | 23 +++++++++---------- src/music_kraken/download/search.py | 16 ++++++------- .../pages/encyclopaedia_metallum.py | 5 ++-- 5 files changed, 32 insertions(+), 31 deletions(-) diff --git a/src/music_kraken/download/download.py b/src/music_kraken/download/download.py index febdbf6..195b04c 100644 --- a/src/music_kraken/download/download.py +++ b/src/music_kraken/download/download.py @@ -10,12 +10,15 @@ MusicObject = Union[Song, Album, Artist, Label] class Download: def __init__( self, - pages: Tuple[Type[Page]] = page_attributes.ALL_PAGES, - exclude_pages: Set[Type[Page]] = set(), + pages: Tuple[Page] = page_attributes.ALL_PAGES, + exclude_pages=None, exclude_shady: bool = False, ) -> None: - _page_list: List[Type[Page]] = [] - _audio_page_list: List[Type[Page]] = [] + if exclude_pages is None: + exclude_pages = set() + + _page_list: List[Page] = [] + _audio_page_list: List[Page] = [] for page in pages: if exclude_shady and page in page_attributes.SHADY_PAGES: @@ -28,8 +31,8 @@ class Download: if page in page_attributes.AUDIO_PAGES: _audio_page_list.append(page) - self.pages: Tuple[Type[Page]] = tuple(_page_list) - self.audio_pages: Tuple[Type[Page]] = tuple(_audio_page_list) + self.pages: Tuple[Page] = tuple(_page_list) + self.audio_pages: Tuple[Page] = tuple(_audio_page_list) def fetch_details(self, music_object: MusicObject) -> MusicObject: for page in self.pages: diff --git a/src/music_kraken/download/multiple_options.py b/src/music_kraken/download/multiple_options.py index d2d99c0..4585952 100644 --- a/src/music_kraken/download/multiple_options.py +++ b/src/music_kraken/download/multiple_options.py @@ -34,11 +34,11 @@ class MultiPageOptions: def __len__(self) -> int: return self._length - def get_page_str(self, page: Type[Page]) -> str: + def get_page_str(self, page: Page) -> str: page_name_fill = "-" max_page_len = 21 - return f"({page_attributes.PAGE_NAME_MAP[page]}) ------------------------{page.__name__:{page_name_fill}<{max_page_len}}------------" + return f"({page_attributes.PAGE_NAME_MAP[page]}) ------------------------{type(page).__name__:{page_name_fill}<{max_page_len}}------------" def string_from_all_pages(self) -> str: if self._length == 1: diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 56e18a4..e51afec 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -4,27 +4,26 @@ from ..utils.enums.source import SourcePages from ..pages import Page, EncyclopaediaMetallum, Musify -NAME_PAGE_MAP: Dict[str, Type[Page]] = dict() -PAGE_NAME_MAP: Dict[Type[Page], str] = dict() -SOURCE_PAGE_MAP: Dict[SourcePages, Type[Page]] = dict() +NAME_PAGE_MAP: Dict[str, Page] = dict() +PAGE_NAME_MAP: Dict[Page, str] = dict() +SOURCE_PAGE_MAP: Dict[SourcePages, Page] = dict() -ALL_PAGES: Tuple[Type[Page]] = ( - EncyclopaediaMetallum, - Musify +ALL_PAGES: Tuple[Page, ...] = ( + EncyclopaediaMetallum(), ) -AUDIO_PAGES: Tuple[Type[Page]] = ( - Musify, +AUDIO_PAGES: Tuple[Page, ...] = ( + Musify(), ) -SHADY_PAGES: Tuple[Type[Page]] = ( - Musify, +SHADY_PAGES: Tuple[Page, ...] = ( + Musify(), ) -# this needs to be case insensitive +# this needs to be case-insensitive SHORTHANDS = ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z') for i, page in enumerate(ALL_PAGES): - NAME_PAGE_MAP[page.__name__.lower()] = page + NAME_PAGE_MAP[type(page).__name__.lower()] = page NAME_PAGE_MAP[SHORTHANDS[i].lower()] = page PAGE_NAME_MAP[page] = SHORTHANDS[i] diff --git a/src/music_kraken/download/search.py b/src/music_kraken/download/search.py index f7a0ddf..ba053c5 100644 --- a/src/music_kraken/download/search.py +++ b/src/music_kraken/download/search.py @@ -94,7 +94,7 @@ class Search(Download): new_text = "" latest_key: str = None for i in range(len(query) - 1): - currenct_char = query[i] + current_char = query[i] next_char = query[i+1] if skip_next: @@ -102,24 +102,25 @@ class Search(Download): continue if escape_next: - new_text += currenct_char + new_text += current_char escape_next = False # escaping - if currenct_char == "\\": + if current_char == "\\": if next_char in special_characters: escape_next = True continue - if currenct_char == "#": + if current_char == "#": if latest_key is not None: - key_text[latest_key] + key_text[latest_key] = new_text + new_text = "" latest_key = next_char skip_next = True continue - new_text += currenct_char + new_text += current_char if latest_key is not None: key_text[latest_key] = new_text @@ -129,8 +130,7 @@ class Search(Download): for page in self.pages: - for search in parsed_query.default_search: - self._current_option[page].extend(page._raw_search(query=search)) + self._current_option[page].extend(page.search(parsed_query)) def choose_page(self, page: Type[Page]): """ diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 012351a..8a57fae 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -35,8 +35,7 @@ ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, { }) -def _song_from_json(artist_html=None, album_html=None, release_type=None, title=None, - lyrics_html=None) -> Song: +def _song_from_json(artist_html=None, album_html=None, release_type=None, title=None, lyrics_html=None) -> Song: song_id = None if lyrics_html is not None: soup = BeautifulSoup(lyrics_html, 'html.parser') @@ -60,7 +59,7 @@ def _song_from_json(artist_html=None, album_html=None, release_type=None, title= def _artist_from_json(artist_html=None, genre=None, country=None) -> Artist: """ - TODO parse the country to a standart + TODO parse the country to a standard """ # parse the html # parse the html for the band name and link on metal-archives From 14c754f81221496c9cc7077fa4a67292adce1429 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 18:09:53 +0200 Subject: [PATCH 23/81] refactored cleaning of objects --- src/music_kraken/pages/abstract.py | 109 +++++++++++++++-------------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 8398caa..fefa2ac 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -26,6 +26,55 @@ from ..utils import shared from ..utils.support_classes import Query, DownloadResult, DefaultTarget +INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] +INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] + +def _clean_music_object(music_object: INDEPENDENT_DB_OBJECTS, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): + if type(music_object) == Label: + return _clean_label(label=music_object, collections=collections) + if type(music_object) == Artist: + return _clean_artist(artist=music_object, collections=collections) + if type(music_object) == Album: + return _clean_album(album=music_object, collections=collections) + if type(music_object) == Song: + return _clean_song(song=music_object, collections=collections) + + +def _clean_collection(collection: Collection, collection_dict: Dict[INDEPENDENT_DB_TYPES, Collection]): + if collection.element_type not in collection_dict: + return + + for i, element in enumerate(collection): + r = collection_dict[collection.element_type].append(element, merge_into_existing=True) + collection[i] = r.current_element + + if not r.was_the_same: + _clean_music_object(r.current_element, collection_dict) + + +def _clean_label(label: Label, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): + _clean_collection(label.current_artist_collection, collections) + _clean_collection(label.album_collection, collections) + + +def _clean_artist(artist: Artist, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): + _clean_collection(artist.main_album_collection, collections) + _clean_collection(artist.feature_song_collection, collections) + _clean_collection(artist.label_collection, collections) + + +def _clean_album(album: Album, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): + _clean_collection(album.label_collection, collections) + _clean_collection(album.song_collection, collections) + _clean_collection(album.artist_collection, collections) + + +def _clean_song(song: Song, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): + _clean_collection(song.album_collection, collections) + _clean_collection(song.feature_artist_collection, collections) + _clean_collection(song.main_artist_collection, collections) + + class Page(threading.Thread): """ This is an abstract class, laying out the @@ -37,6 +86,9 @@ class Page(threading.Thread): def __init__(self): threading.Thread.__init__(self) + + def run(self) -> None: + pass @classmethod def get_soup_from_response(cls, r: requests.Response) -> BeautifulSoup: @@ -119,10 +171,12 @@ class Page(threading.Thread): Song: Collection(element_type=Song) } - cls._clean_music_object(new_music_object, collections) + if not isinstance(new_music_object, INDEPENDENT_DB_OBJECTS): + raise TypeError(f"Can't clean the object, because it isn't a valid type: {type(new_music_object)} | {type(INDEPENDENT_DB_OBJECTS)}") + + _clean_music_object(new_music_object, collections) music_object.merge(new_music_object) - music_object.compile(merge_into=True) return music_object @@ -163,57 +217,6 @@ class Page(threading.Thread): if obj_type == Label: return cls._fetch_label_from_source(source=source, stop_at_level=stop_at_level) - @classmethod - def _clean_music_object(cls, music_object: Union[Label, Album, Artist, Song], - collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - if type(music_object) == Label: - return cls._clean_label(label=music_object, collections=collections) - if type(music_object) == Artist: - return cls._clean_artist(artist=music_object, collections=collections) - if type(music_object) == Album: - return cls._clean_album(album=music_object, collections=collections) - if type(music_object) == Song: - return cls._clean_song(song=music_object, collections=collections) - - @classmethod - def _clean_collection(cls, collection: Collection, - collection_dict: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - if collection.element_type not in collection_dict: - return - - for i, element in enumerate(collection): - r = collection_dict[collection.element_type].append(element, merge_into_existing=True) - collection[i] = r.current_element - - if not r.was_the_same: - cls._clean_music_object(r.current_element, collection_dict) - - @classmethod - def _clean_label(cls, label: Label, - collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - cls._clean_collection(label.current_artist_collection, collections) - cls._clean_collection(label.album_collection, collections) - - @classmethod - def _clean_artist(cls, artist: Artist, - collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - cls._clean_collection(artist.main_album_collection, collections) - cls._clean_collection(artist.feature_song_collection, collections) - cls._clean_collection(artist.label_collection, collections) - - @classmethod - def _clean_album(cls, album: Album, - collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - cls._clean_collection(album.label_collection, collections) - cls._clean_collection(album.song_collection, collections) - cls._clean_collection(album.artist_collection, collections) - - @classmethod - def _clean_song(cls, song: Song, - collections: Dict[Union[Type[Song], Type[Album], Type[Artist], Type[Label]], Collection]): - cls._clean_collection(song.album_collection, collections) - cls._clean_collection(song.feature_artist_collection, collections) - cls._clean_collection(song.main_artist_collection, collections) @classmethod def download( From ff01bae12dc5ff60fa838a150716acbdea148a97 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 18:11:56 +0200 Subject: [PATCH 24/81] refactored cleaning of objects --- src/music_kraken/pages/abstract.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index fefa2ac..6c00e64 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -132,12 +132,11 @@ class Page(threading.Thread): return [] - @classmethod - def fetch_details(cls, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: + def fetch_details(self, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: """ when a music object with laccing data is passed in, it returns the SAME object **(no copy)** with more detailed data. - If you for example put in an album, it fetches the tracklist + If you for example put in, an album, it fetches the tracklist :param music_object: :param stop_at_level: @@ -155,7 +154,7 @@ class Page(threading.Thread): had_sources = False source: Source - for source in music_object.source_collection.get_sources_from_page(cls.SOURCE_TYPE): + for source in music_object.source_collection.get_sources_from_page(self.SOURCE_TYPE): new_music_object.merge( cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) had_sources = True From 51770ec0a9db8dea6ff78d36508247fef8bfedae Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 23 May 2023 21:06:01 +0200 Subject: [PATCH 25/81] much progress --- src/music_kraken/pages/abstract.py | 65 +++++++++++++----------------- 1 file changed, 29 insertions(+), 36 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 6c00e64..4a943b3 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -75,6 +75,21 @@ def _clean_song(song: Song, collections: Dict[INDEPENDENT_DB_TYPES, Collection]) _clean_collection(song.main_artist_collection, collections) +def post_process_object(database_object: DatabaseObject, clean_up: bool = True) -> DatabaseObject: + if isinstance(database_object, INDEPENDENT_DB_OBJECTS) and clean_up: + collections = { + Label: Collection(element_type=Label), + Artist: Collection(element_type=Artist), + Album: Collection(element_type=Album), + Song: Collection(element_type=Song) + } + + _clean_music_object(database_object, collections) + + database_object.compile(merge_into=True) + return database_object + + class Page(threading.Thread): """ This is an abstract class, laying out the @@ -132,9 +147,9 @@ class Page(threading.Thread): return [] - def fetch_details(self, music_object: Union[Song, Album, Artist, Label], stop_at_level: int = 1) -> DatabaseObject: + def fetch_details(self, music_object: DatabaseObject, stop_at_level: int = 1) -> DatabaseObject: """ - when a music object with laccing data is passed in, it returns + when a music object with lacking data is passed in, it returns the SAME object **(no copy)** with more detailed data. If you for example put in, an album, it fetches the tracklist @@ -149,55 +164,33 @@ class Page(threading.Thread): :return detailed_music_object: IT MODIFIES THE INPUT OBJ """ + # creating a new object, of the same type new_music_object: DatabaseObject = type(music_object)() - had_sources = False + # only certain database objects, have a source list + if isinstance(music_object, INDEPENDENT_DB_TYPES): + source: Source + for source in music_object.source_collection.get_sources_from_page(self.SOURCE_TYPE): + new_music_object.merge( + self.fetch_object_from_source(source=source, enforce_type=type(music_object), stop_at_level=stop_at_level, post_process=False)) - source: Source - for source in music_object.source_collection.get_sources_from_page(self.SOURCE_TYPE): - new_music_object.merge( - cls._fetch_object_from_source(source=source, obj_type=type(music_object), stop_at_level=stop_at_level)) - had_sources = True - - if not had_sources: - music_object.compile(merge_into=True) - return music_object - - collections = { - Label: Collection(element_type=Label), - Artist: Collection(element_type=Artist), - Album: Collection(element_type=Album), - Song: Collection(element_type=Song) - } - - if not isinstance(new_music_object, INDEPENDENT_DB_OBJECTS): - raise TypeError(f"Can't clean the object, because it isn't a valid type: {type(new_music_object)} | {type(INDEPENDENT_DB_OBJECTS)}") - - _clean_music_object(new_music_object, collections) + new_music_object = post_process_object(new_music_object) music_object.merge(new_music_object) music_object.compile(merge_into=True) return music_object - @classmethod - def fetch_object_from_source(cls, source: Source, stop_at_level: int = 2): - obj_type = cls._get_type_of_url(source.url) + def fetch_object_from_source(self, source: Source, stop_at_level: int = 2, enforce_type: Type[DatabaseObject] = None, post_process: bool = True) -> DatabaseObject: + obj_type = self._get_type_of_url(source.url) if obj_type is None: return None music_object = cls._fetch_object_from_source(source=source, obj_type=obj_type, stop_at_level=stop_at_level) - collections = { - Label: Collection(element_type=Label), - Artist: Collection(element_type=Artist), - Album: Collection(element_type=Album), - Song: Collection(element_type=Song) - } + if post_process: + return post_process_object(music_object) - cls._clean_music_object(music_object, collections) - - music_object.compile(merge_into=True) return music_object @classmethod From 0fd743e81fc34004852abd2ce77b04224d5574bb Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 24 May 2023 08:50:56 +0200 Subject: [PATCH 26/81] much progress --- src/music_kraken/pages/abstract.py | 76 ++++++------ .../pages/encyclopaedia_metallum.py | 113 ++++++++---------- 2 files changed, 87 insertions(+), 102 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 4a943b3..cd29db3 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -29,6 +29,7 @@ from ..utils.support_classes import Query, DownloadResult, DefaultTarget INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] + def _clean_music_object(music_object: INDEPENDENT_DB_OBJECTS, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): if type(music_object) == Label: return _clean_label(label=music_object, collections=collections) @@ -104,9 +105,11 @@ class Page(threading.Thread): def run(self) -> None: pass + + def get_source_type(self, source: Source) -> Optional[INDEPENDENT_DB_TYPES]: + return None - @classmethod - def get_soup_from_response(cls, r: requests.Response) -> BeautifulSoup: + def get_soup_from_response(self, r: requests.Response) -> BeautifulSoup: return BeautifulSoup(r.content, "html.parser") # to search stuff @@ -168,7 +171,7 @@ class Page(threading.Thread): new_music_object: DatabaseObject = type(music_object)() # only certain database objects, have a source list - if isinstance(music_object, INDEPENDENT_DB_TYPES): + if isinstance(music_object, INDEPENDENT_DB_OBJECTS): source: Source for source in music_object.source_collection.get_sources_from_page(self.SOURCE_TYPE): new_music_object.merge( @@ -181,34 +184,43 @@ class Page(threading.Thread): return music_object - def fetch_object_from_source(self, source: Source, stop_at_level: int = 2, enforce_type: Type[DatabaseObject] = None, post_process: bool = True) -> DatabaseObject: - obj_type = self._get_type_of_url(source.url) + def fetch_object_from_source(self, source: Source, stop_at_level: int = 2, enforce_type: Type[DatabaseObject] = None, post_process: bool = True) -> Optional[DatabaseObject]: + obj_type = self.get_source_type(source) + if obj_type is None: return None + if enforce_type != obj_type and enforce_type is not None: + self.LOGGER.warning(f"Object type isn't type to enforce: {enforce_type}, {obj_type}") + return None + + music_object: DatabaseObject = None + + fetch_map = { + Song: self.fetch_song, + Album: self.fetch_album, + Artist: self.fetch_artist, + Label: self.fetch_label + } + + if obj_type in fetch_map: + music_object = fetch_map[obj_type](source, stop_at_level) - music_object = cls._fetch_object_from_source(source=source, obj_type=obj_type, stop_at_level=stop_at_level) - - if post_process: + if post_process and music_object is not None: return post_process_object(music_object) return music_object + + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: + return Song() - @classmethod - def _fetch_object_from_source(cls, source: Source, - obj_type: Union[Type[Song], Type[Album], Type[Artist], Type[Label]], - stop_at_level: int = 1) -> Union[Song, Album, Artist, Label]: - if obj_type == Artist: - return cls._fetch_artist_from_source(source=source, stop_at_level=stop_at_level) + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + return Album() - if obj_type == Song: - return cls._fetch_song_from_source(source=source, stop_at_level=stop_at_level) - - if obj_type == Album: - return cls._fetch_album_from_source(source=source, stop_at_level=stop_at_level) - - if obj_type == Label: - return cls._fetch_label_from_source(source=source, stop_at_level=stop_at_level) + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + return Artist() + def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: + return Label() @classmethod def download( @@ -461,26 +473,6 @@ class Page(threading.Thread): return r - @classmethod - def _fetch_song_from_source(cls, source: Source, stop_at_level: int = 1) -> Song: - return Song() - - @classmethod - def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: - return Album() - - @classmethod - def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: - return Artist() - - @classmethod - def _fetch_label_from_source(cls, source: Source, stop_at_level: int = 1) -> Label: - return Label() - - @classmethod - def _get_type_of_url(cls, url: str) -> Optional[Union[Type[Song], Type[Album], Type[Artist], Type[Label]]]: - return None - @classmethod def _download_song_to_targets(cls, source: Source, target: Target, desc: str = None) -> DownloadResult: return DownloadResult() diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 8a57fae..42e0e5f 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -218,16 +218,14 @@ class EncyclopaediaMetallum(Page): for raw_artist in r.json()['aaData'] ] - - @classmethod - def _fetch_artist_discography(cls, ma_artist_id: str) -> List[Album]: + def _fetch_artist_discography(self, ma_artist_id: str) -> List[Album]: discography_url = "https://www.metal-archives.com/band/discography/id/{}/tab/all" # make the request - r = cls.CONNECTION.get(discography_url.format(ma_artist_id)) + r = self.connection.get(discography_url.format(ma_artist_id)) if r is None: return [] - soup = cls.get_soup_from_response(r) + soup = self.get_soup_from_response(r) discography = [] @@ -251,21 +249,20 @@ class EncyclopaediaMetallum(Page): Album( title=album_name, date=date_obj, - album_type=cls.ALBUM_TYPE_MAP[raw_album_type], - source_list=[Source(SourcePages.ENCYCLOPAEDIA_METALLUM, album_url)] + album_type=ALBUM_TYPE_MAP[raw_album_type], + source_list=[Source(self.SOURCE_TYPE, album_url)] ) ) return discography - @classmethod - def _fetch_artist_sources(cls, ma_artist_id: str) -> List[Source]: + def _fetch_artist_sources(self, ma_artist_id: str) -> List[Source]: sources_url = "https://www.metal-archives.com/link/ajax-list/type/band/id/{}" - r = cls.CONNECTION.get(sources_url.format(ma_artist_id)) + r = self.connection.get(sources_url.format(ma_artist_id)) if r is None: return [] - soup = cls.get_soup_from_response(r) + soup = self.get_soup_from_response(r) if soup.find("span", {"id": "noLinks"}) is not None: return [] @@ -289,12 +286,11 @@ class EncyclopaediaMetallum(Page): if url is None: continue - source_list.append(Source.match_url(url, referer_page=cls.SOURCE_TYPE)) + source_list.append(Source.match_url(url, referer_page=self.SOURCE_TYPE)) return source_list - @classmethod - def _parse_artist_attributes(cls, artist_soup: BeautifulSoup) -> Artist: + def _parse_artist_attributes(self, artist_soup: BeautifulSoup) -> Artist: name: str = None country: pycountry.Countrie = None formed_in_year: int = None @@ -311,7 +307,7 @@ class EncyclopaediaMetallum(Page): if title_text.count(bad_name_substring) == 1: name = title_text.replace(bad_name_substring, "") else: - cls.LOGGER.debug(f"the title of the page is \"{title_text}\"") + self.LOGGER.debug(f"the title of the page is \"{title_text}\"") """ TODO @@ -341,7 +337,7 @@ class EncyclopaediaMetallum(Page): href = anchor.get("href") if href is not None: - source_list.append(Source(cls.SOURCE_TYPE, href)) + source_list.append(Source(self.SOURCE_TYPE, href)) name = anchor.get_text(strip=True) @@ -400,35 +396,32 @@ class EncyclopaediaMetallum(Page): Label( name=label_name, source_list=[ - Source(cls.SOURCE_TYPE, label_url) + Source(self.SOURCE_TYPE, label_url) ] ) ], source_list=source_list ) - @classmethod - def _fetch_artist_attributes(cls, url: str) -> Artist: - r = cls.CONNECTION.get(url) + def _fetch_artist_attributes(self, url: str) -> Artist: + r = self.connection.get(url) if r is None: return Artist() - soup: BeautifulSoup = cls.get_soup_from_response(r) + soup: BeautifulSoup = self.get_soup_from_response(r) - return cls._parse_artist_attributes(artist_soup=soup) + return self._parse_artist_attributes(artist_soup=soup) - @classmethod - def _fetch_band_notes(cls, ma_artist_id: str) -> Optional[FormattedText]: + def _fetch_band_notes(self, ma_artist_id: str) -> Optional[FormattedText]: endpoint = "https://www.metal-archives.com/band/read-more/id/{}" # make the request - r = cls.CONNECTION.get(endpoint.format(ma_artist_id)) + r = self.connection.get(endpoint.format(ma_artist_id)) if r is None: return FormattedText() return FormattedText(html=r.text) - @classmethod - def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: """ What it could fetch, and what is implemented: @@ -440,28 +433,27 @@ class EncyclopaediaMetallum(Page): [x] band notes: https://www.metal-archives.com/band/read-more/id/3540372489 """ - artist = cls._fetch_artist_attributes(source.url) + artist = self._fetch_artist_attributes(source.url) artist_id = source.url.split("/")[-1] - artist_sources = cls._fetch_artist_sources(artist_id) + artist_sources = self._fetch_artist_sources(artist_id) artist.source_collection.extend(artist_sources) - band_notes = cls._fetch_band_notes(artist_id) + band_notes = self._fetch_band_notes(artist_id) if band_notes is not None: artist.notes = band_notes - discography: List[Album] = cls._fetch_artist_discography(artist_id) + discography: List[Album] = self._fetch_artist_discography(artist_id) if stop_at_level > 1: for album in discography: - for source in album.source_collection.get_sources_from_page(cls.SOURCE_TYPE): - album.merge(cls._fetch_album_from_source(source, stop_at_level=stop_at_level-1)) + for source in album.source_collection.get_sources_from_page(self.SOURCE_TYPE): + album.merge(self._fetch_album_from_source(source, stop_at_level=stop_at_level-1)) artist.main_album_collection.extend(discography) return artist - @classmethod - def _parse_album_track_row(cls, track_row: BeautifulSoup) -> Song: + def _parse_album_track_row(self, track_row: BeautifulSoup) -> Song: """ # id and tracksort @@ -482,7 +474,7 @@ class EncyclopaediaMetallum(Page): track_id = track_sort_soup.find("a").get("name").strip() if track_row.find("a", {"href": f"#{track_id}"}) is not None: - source_list.append(Source(cls.SOURCE_TYPE, track_id)) + source_list.append(Source(self.SOURCE_TYPE, track_id)) title = row_list[1].text.strip() @@ -500,9 +492,7 @@ class EncyclopaediaMetallum(Page): source_list=source_list ) - - @classmethod - def _parse_album_attributes(cls, album_soup: BeautifulSoup, stop_at_level: int = 1) -> Album: + def _parse_album_attributes(self, album_soup: BeautifulSoup, stop_at_level: int = 1) -> Album: tracklist: List[Song] = [] artist_list = [] album_name: str = None @@ -522,12 +512,12 @@ class EncyclopaediaMetallum(Page): href = anchor.get("href") if href is not None: - source_list.append(Source(cls.SOURCE_TYPE, href.strip())) + source_list.append(Source(self.SOURCE_TYPE, href.strip())) album_name = anchor.get_text(strip=True) elif len(album_soup_list) > 1: - cls.LOGGER.debug("there are more than 1 album soups") + self.LOGGER.debug("there are more than 1 album soups") artist_soup_list = album_info_soup.find_all("h2", {"class": "band_name"}) @@ -537,7 +527,7 @@ class EncyclopaediaMetallum(Page): href = anchor.get("href") if href is not None: - artist_sources.append(Source(cls.SOURCE_TYPE, href.strip())) + artist_sources.append(Source(self.SOURCE_TYPE, href.strip())) artist_name = anchor.get_text(strip=True) @@ -547,13 +537,13 @@ class EncyclopaediaMetallum(Page): )) elif len(artist_soup_list) > 1: - cls.LOGGER.debug("there are more than 1 artist soups") + self.LOGGER.debug("there are more than 1 artist soups") _parse_album_info(album_info_soup=album_soup.find(id="album_info")) tracklist_soup = album_soup.find("table", {"class": "table_lyrics"}).find("tbody") for track_soup in tracklist_soup.find_all("tr", {"class": ["even", "odd"]}): - tracklist.append(cls._parse_album_track_row(track_row=track_soup)) + tracklist.append(self._parse_album_track_row(track_row=track_soup)) return Album( title=album_name, @@ -562,8 +552,7 @@ class EncyclopaediaMetallum(Page): song_list=tracklist ) - @classmethod - def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: """ I am preeeety sure I can get way more data than... nothing from there @@ -574,23 +563,22 @@ class EncyclopaediaMetallum(Page): #
1.
Song: + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: song_id = source.url return Song( lyrics_list=[ - cls._fetch_lyrics(song_id=song_id) + self._fetch_lyrics(song_id=song_id) ] ) - @classmethod - def _get_type_of_url(cls, url: str) -> Optional[Union[Type[Song], Type[Album], Type[Artist], Type[Label]]]: + def get_source_type(self, source: Source): + if self.SOURCE_TYPE != source.page_enum: + return None + + url = source.url + if url is None: + return None + parsed_url = urlparse(url) path: List[str] = parsed_url.path.split("/") From 7126db96669514df43a6cd8bf220fbcc52ca3c89 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 24 May 2023 09:16:29 +0200 Subject: [PATCH 27/81] fixed recursion depth error --- src/actual_donwload.py | 5 ++++- src/music_kraken/__init__.py | 2 ++ src/music_kraken/pages/abstract.py | 36 ++++++++++++++++++------------ 3 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 5c3cb71..b6fb29b 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -53,7 +53,10 @@ def real_download(): if __name__ == "__main__": music_kraken.cli(genre="test", command_list=[ - "https://musify.club/release/molchat-doma-etazhi-2018-1092949", + # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", # "https://musify.club/release/ghost-bath-self-loather-2021-1554266", + "#a Ghost Bath", + "0", + "4", "ok" ]) diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index f4debe2..3e6d260 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -13,6 +13,8 @@ from .utils.shared import MUSIC_DIR, MODIFY_GC, NOT_A_GENRE_REGEX, get_random_me from .utils.string_processing import fit_to_file_system +import sys; sys.setrecursionlimit(100) + if MODIFY_GC: """ At the start I modify the garbage collector to run a bit fewer times. diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index cd29db3..33140db 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -75,9 +75,8 @@ def _clean_song(song: Song, collections: Dict[INDEPENDENT_DB_TYPES, Collection]) _clean_collection(song.feature_artist_collection, collections) _clean_collection(song.main_artist_collection, collections) - -def post_process_object(database_object: DatabaseObject, clean_up: bool = True) -> DatabaseObject: - if isinstance(database_object, INDEPENDENT_DB_OBJECTS) and clean_up: +def clean_object(dirty_object: DatabaseObject) -> DatabaseObject: + if isinstance(dirty_object, INDEPENDENT_DB_OBJECTS): collections = { Label: Collection(element_type=Label), Artist: Collection(element_type=Artist), @@ -85,10 +84,21 @@ def post_process_object(database_object: DatabaseObject, clean_up: bool = True) Song: Collection(element_type=Song) } - _clean_music_object(database_object, collections) + return _clean_music_object(dirty_object, collections) + +def build_new_object(new_object: DatabaseObject) -> DatabaseObject: + new_object = clean_object(new_object) + new_object.compile(merge_into=False) + + return new_object - database_object.compile(merge_into=True) - return database_object +def merge_together(old_object: DatabaseObject, new_object: DatabaseObject) -> DatabaseObject: + new_object = clean_object(new_object) + + old_object.merge(new_object) + old_object.compile(merge_into=False) + + return old_object class Page(threading.Thread): @@ -177,12 +187,7 @@ class Page(threading.Thread): new_music_object.merge( self.fetch_object_from_source(source=source, enforce_type=type(music_object), stop_at_level=stop_at_level, post_process=False)) - new_music_object = post_process_object(new_music_object) - - music_object.merge(new_music_object) - music_object.compile(merge_into=True) - - return music_object + return merge_together(music_object, new_music_object) def fetch_object_from_source(self, source: Source, stop_at_level: int = 2, enforce_type: Type[DatabaseObject] = None, post_process: bool = True) -> Optional[DatabaseObject]: obj_type = self.get_source_type(source) @@ -204,9 +209,12 @@ class Page(threading.Thread): if obj_type in fetch_map: music_object = fetch_map[obj_type](source, stop_at_level) + else: + self.LOGGER.warning(f"Can't fetch details of type: {obj_type}") + return None - if post_process and music_object is not None: - return post_process_object(music_object) + if post_process and music_object: + return build_new_object(music_object) return music_object From dd0708eef0c7a0bfea46d3f81853a626756a65e7 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 24 May 2023 09:50:27 +0200 Subject: [PATCH 28/81] fixed issue wit the way returning works --- src/music_kraken/pages/abstract.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 33140db..ef7fab0 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -84,7 +84,8 @@ def clean_object(dirty_object: DatabaseObject) -> DatabaseObject: Song: Collection(element_type=Song) } - return _clean_music_object(dirty_object, collections) + _clean_music_object(dirty_object, collections) + return dirty_object def build_new_object(new_object: DatabaseObject) -> DatabaseObject: new_object = clean_object(new_object) From 19af808d16e3a31309acbc11e5d1485229a6a33d Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 24 May 2023 10:12:03 +0200 Subject: [PATCH 29/81] added preset --- src/music_kraken/__init__.py | 2 - src/music_kraken/objects/__init__.py | 44 ++++-------- src/music_kraken/objects/parents.py | 3 + src/music_kraken/pages/abstract.py | 10 ++- src/music_kraken/pages/preset.py | 98 +++++++++++--------------- src/music_kraken/utils/enums/source.py | 2 + 6 files changed, 69 insertions(+), 90 deletions(-) diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index 3e6d260..f4debe2 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -13,8 +13,6 @@ from .utils.shared import MUSIC_DIR, MODIFY_GC, NOT_A_GENRE_REGEX, get_random_me from .utils.string_processing import fit_to_file_system -import sys; sys.setrecursionlimit(100) - if MODIFY_GC: """ At the start I modify the garbage collector to run a bit fewer times. diff --git a/src/music_kraken/objects/__init__.py b/src/music_kraken/objects/__init__.py index 2018550..f458303 100644 --- a/src/music_kraken/objects/__init__.py +++ b/src/music_kraken/objects/__init__.py @@ -1,32 +1,18 @@ -from ..utils.enums import album from .option import Options -from . import ( - song, - metadata, - source, - parents, - formatted_text, - option, - collection +from .parents import DatabaseObject + +from .metadata import Metadata, Mapping as ID3Mapping, ID3Timestamp + +from .source import Source, SourcePages, SourceTypes + +from .song import ( + Song, + Album, + Artist, + Target, + Lyrics, + Label ) -DatabaseObject = parents.DatabaseObject - -Metadata = metadata.Metadata -ID3Mapping = metadata.Mapping -ID3Timestamp = metadata.ID3Timestamp - -Source = source.Source - -Song = song.Song -Artist = song.Artist -Source = source.Source -Target = song.Target -Lyrics = song.Lyrics -Label = song.Label - -Album = song.Album - -FormattedText = formatted_text.FormattedText - -Collection = collection.Collection +from .formatted_text import FormattedText +from .collection import Collection diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index bf0ad54..f692dea 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -65,6 +65,9 @@ class DatabaseObject: return list() def merge(self, other, override: bool = False): + if other is None: + return + if self is other: return diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index ef7fab0..d5cd8fa 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -117,7 +117,7 @@ class Page(threading.Thread): def run(self) -> None: pass - def get_source_type(self, source: Source) -> Optional[INDEPENDENT_DB_TYPES]: + def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: return None def get_soup_from_response(self, r: requests.Response) -> BeautifulSoup: @@ -186,7 +186,13 @@ class Page(threading.Thread): source: Source for source in music_object.source_collection.get_sources_from_page(self.SOURCE_TYPE): new_music_object.merge( - self.fetch_object_from_source(source=source, enforce_type=type(music_object), stop_at_level=stop_at_level, post_process=False)) + self.fetch_object_from_source( + source=source, + enforce_type=type(music_object), + stop_at_level=stop_at_level, + post_process=False + ) + ) return merge_together(music_object, new_music_object) diff --git a/src/music_kraken/pages/preset.py b/src/music_kraken/pages/preset.py index 333b265..b2af7fd 100644 --- a/src/music_kraken/pages/preset.py +++ b/src/music_kraken/pages/preset.py @@ -1,13 +1,9 @@ -from collections import defaultdict -from dataclasses import dataclass -from enum import Enum -from typing import List, Optional, Type, Union +from typing import List, Optional, Type from urllib.parse import urlparse +import logging -import pycountry -import requests -from bs4 import BeautifulSoup +from music_kraken.objects import Source, DatabaseObject from .abstract import Page from ..objects import ( Artist, @@ -15,61 +11,49 @@ from ..objects import ( SourcePages, Song, Album, - ID3Timestamp, - FormattedText, Label, - Options, - AlbumType, - AlbumStatus, - Target ) -from ..utils import string_processing, shared -from .support_classes.download_result import DownloadResult +from ..connection import Connection +class Preset(Page): + # CHANGE + SOURCE_TYPE = SourcePages.PRESET + LOGGER = logging.getLogger("preset") -class YouTube(Page): - API_SESSION: requests.Session = requests.Session() - API_SESSION.headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", - "Connection": "keep-alive", - "Referer": "https://www.youtube.com/" - } - API_SESSION.proxies = shared.proxies - TIMEOUT = 7 - POST_TIMEOUT = 15 - TRIES = 5 - HOST = "https://www.youtube.com" + def __init__(self): + self.connection: Connection = Connection( + host="https://www.preset.cum/", + logger=self.LOGGER + ) + + super().__init__() - SOURCE_TYPE = SourcePages.YOUTUBE + def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: + return super().get_source_type(source) + + def general_search(self, search_query: str) -> List[DatabaseObject]: + return [] + + def label_search(self, label: Label) -> List[Label]: + return [] + + def artist_search(self, artist: Artist) -> List[Artist]: + return [] + + def album_search(self, album: Album) -> List[Album]: + return [] + + def song_search(self, song: Song) -> List[Song]: + return [] + + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: + return Song() - LOGGER = shared.YOUTUBE_LOGGER + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + return Album() + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + return Artist() - @classmethod - def _raw_search(cls, query: str) -> Options: - return Options() - - @classmethod - def plaintext_search(cls, query: str) -> Options: - search_results = [] - - return Options(search_results) - - @classmethod - def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: - artist: Artist = Artist(source_list=[source]) - - return artist - - @classmethod - def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: - album: Album = Album(source_list=[source]) - return album - - @classmethod - def _get_type_of_url(cls, url: str) -> Optional[Union[Type[Song], Type[Album], Type[Artist], Type[Label]]]: - return None - - @classmethod - def _download_song_to_targets(cls, source: Source, target: Target, desc: str = None) -> DownloadResult: - return DownloadResult() + def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: + return Label() diff --git a/src/music_kraken/utils/enums/source.py b/src/music_kraken/utils/enums/source.py index b324f8d..964de00 100644 --- a/src/music_kraken/utils/enums/source.py +++ b/src/music_kraken/utils/enums/source.py @@ -26,6 +26,8 @@ class SourcePages(Enum): MYSPACE = "myspace" # Yes somehow this ancient site is linked EVERYWHERE MANUAL = "manual" + + PRESET = "preset" @classmethod def get_homepage(cls, attribute) -> str: From 486da4b43cf86e0253a442d9edf1bf5bbd34e8a8 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 24 May 2023 17:32:22 +0200 Subject: [PATCH 30/81] musify --- .VSCodeCounter/2023-02-27_12-22-06/details.md | 78 -- .../2023-02-27_12-22-06/diff-details.md | 49 - .VSCodeCounter/2023-02-27_12-22-06/diff.csv | 36 - .VSCodeCounter/2023-02-27_12-22-06/diff.md | 30 - .VSCodeCounter/2023-02-27_12-22-06/diff.txt | 67 -- .../2023-02-27_12-22-06/results.csv | 65 -- .../2023-02-27_12-22-06/results.json | 1 - .VSCodeCounter/2023-02-27_12-22-06/results.md | 40 - .../2023-02-27_12-22-06/results.txt | 106 -- .VSCodeCounter/2023-05-24_11-17-57/details.md | 104 ++ .../2023-05-24_11-17-57/diff-details.md | 76 ++ .VSCodeCounter/2023-05-24_11-17-57/diff.csv | 63 ++ .VSCodeCounter/2023-05-24_11-17-57/diff.md | 40 + .VSCodeCounter/2023-05-24_11-17-57/diff.txt | 104 ++ .../2023-05-24_11-17-57/results.csv | 91 ++ .../2023-05-24_11-17-57/results.json | 1 + .VSCodeCounter/2023-05-24_11-17-57/results.md | 46 + .../2023-05-24_11-17-57/results.txt | 138 +++ src/music_kraken/pages/musify.py | 1000 +++++++++-------- 19 files changed, 1177 insertions(+), 958 deletions(-) delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/details.md delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/diff-details.md delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/diff.csv delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/diff.md delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/diff.txt delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/results.csv delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/results.json delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/results.md delete mode 100644 .VSCodeCounter/2023-02-27_12-22-06/results.txt create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/details.md create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/diff-details.md create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/diff.csv create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/diff.md create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/diff.txt create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/results.csv create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/results.json create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/results.md create mode 100644 .VSCodeCounter/2023-05-24_11-17-57/results.txt diff --git a/.VSCodeCounter/2023-02-27_12-22-06/details.md b/.VSCodeCounter/2023-02-27_12-22-06/details.md deleted file mode 100644 index 323f8aa..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/details.md +++ /dev/null @@ -1,78 +0,0 @@ -# Details - -Date : 2023-02-27 12:22:06 - -Directory /home/lars/Projects/music-downloader/src - -Total : 63 files, 4095 codes, 1060 comments, 1216 blanks, all 6371 lines - -[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) - -## Files -| filename | language | code | comment | blank | total | -| :--- | :--- | ---: | ---: | ---: | ---: | -| [src/__init__.py](/src/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/create_custom_objects.py](/src/create_custom_objects.py) | Python | 80 | 3 | 18 | 101 | -| [src/metal_archives.py](/src/metal_archives.py) | Python | 48 | 4 | 15 | 67 | -| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | 57 | 8 | 25 | 90 | -| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 3 | 2 | 3 | 8 | -| [src/music_kraken/database/__init__.py](/src/music_kraken/database/__init__.py) | Python | 18 | 0 | 5 | 23 | -| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 116 | 23 | 49 | 188 | -| [src/music_kraken/database/database.py](/src/music_kraken/database/database.py) | Python | 84 | 48 | 26 | 158 | -| [src/music_kraken/database/object_cache.py](/src/music_kraken/database/object_cache.py) | Python | 35 | 56 | 16 | 107 | -| [src/music_kraken/database/old_database.py](/src/music_kraken/database/old_database.py) | Python | 432 | 154 | 115 | 701 | -| [src/music_kraken/database/read.py](/src/music_kraken/database/read.py) | Python | 0 | 0 | 1 | 1 | -| [src/music_kraken/database/temp_database.py](/src/music_kraken/database/temp_database.py) | Python | 12 | 0 | 8 | 20 | -| [src/music_kraken/database/write.py](/src/music_kraken/database/write.py) | Python | 168 | 62 | 55 | 285 | -| [src/music_kraken/not_used_anymore/__init__.py](/src/music_kraken/not_used_anymore/__init__.py) | Python | 0 | 0 | 3 | 3 | -| [src/music_kraken/not_used_anymore/fetch_audio.py](/src/music_kraken/not_used_anymore/fetch_audio.py) | Python | 75 | 12 | 20 | 107 | -| [src/music_kraken/not_used_anymore/fetch_source.py](/src/music_kraken/not_used_anymore/fetch_source.py) | Python | 54 | 1 | 16 | 71 | -| [src/music_kraken/not_used_anymore/metadata/__init__.py](/src/music_kraken/not_used_anymore/metadata/__init__.py) | Python | 6 | 0 | 2 | 8 | -| [src/music_kraken/not_used_anymore/metadata/metadata_fetch.py](/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py) | Python | 257 | 24 | 65 | 346 | -| [src/music_kraken/not_used_anymore/metadata/metadata_search.py](/src/music_kraken/not_used_anymore/metadata/metadata_search.py) | Python | 253 | 40 | 72 | 365 | -| [src/music_kraken/not_used_anymore/metadata/sources/__init__.py](/src/music_kraken/not_used_anymore/metadata/sources/__init__.py) | Python | 3 | 0 | 2 | 5 | -| [src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py](/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py) | Python | 42 | 6 | 12 | 60 | -| [src/music_kraken/not_used_anymore/sources/__init__.py](/src/music_kraken/not_used_anymore/sources/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/music_kraken/not_used_anymore/sources/genius.py](/src/music_kraken/not_used_anymore/sources/genius.py) | Python | 115 | 16 | 42 | 173 | -| [src/music_kraken/not_used_anymore/sources/local_files.py](/src/music_kraken/not_used_anymore/sources/local_files.py) | Python | 40 | 0 | 18 | 58 | -| [src/music_kraken/not_used_anymore/sources/musify.py](/src/music_kraken/not_used_anymore/sources/musify.py) | Python | 136 | 9 | 37 | 182 | -| [src/music_kraken/not_used_anymore/sources/source.py](/src/music_kraken/not_used_anymore/sources/source.py) | Python | 11 | 5 | 8 | 24 | -| [src/music_kraken/not_used_anymore/sources/youtube.py](/src/music_kraken/not_used_anymore/sources/youtube.py) | Python | 71 | 4 | 24 | 99 | -| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 23 | 0 | 7 | 30 | -| [src/music_kraken/objects/album.py](/src/music_kraken/objects/album.py) | Python | 15 | 6 | 5 | 26 | -| [src/music_kraken/objects/artist.py](/src/music_kraken/objects/artist.py) | Python | 18 | 0 | 5 | 23 | -| [src/music_kraken/objects/collection.py](/src/music_kraken/objects/collection.py) | Python | 52 | 15 | 24 | 91 | -| [src/music_kraken/objects/formatted_text.py](/src/music_kraken/objects/formatted_text.py) | Python | 53 | 57 | 20 | 130 | -| [src/music_kraken/objects/lyrics.py](/src/music_kraken/objects/lyrics.py) | Python | 21 | 0 | 7 | 28 | -| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 262 | 68 | 63 | 393 | -| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 22 | 15 | 12 | 49 | -| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 332 | 81 | 89 | 502 | -| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 116 | 38 | 41 | 195 | -| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 22 | 7 | 7 | 36 | -| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | 7 | 0 | 5 | 12 | -| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | 73 | 68 | 27 | 168 | -| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | 341 | 66 | 91 | 498 | -| [src/music_kraken/pages/youtube.py](/src/music_kraken/pages/youtube.py) | Python | 25 | 16 | 6 | 47 | -| [src/music_kraken/static_files/new_db.sql](/src/music_kraken/static_files/new_db.sql) | SQLite | 72 | 0 | 10 | 82 | -| [src/music_kraken/static_files/temp_database_structure.sql](/src/music_kraken/static_files/temp_database_structure.sql) | SQLite | 135 | 0 | 10 | 145 | -| [src/music_kraken/tagging/__init__.py](/src/music_kraken/tagging/__init__.py) | Python | 8 | 0 | 2 | 10 | -| [src/music_kraken/tagging/id3.py](/src/music_kraken/tagging/id3.py) | Python | 51 | 4 | 20 | 75 | -| [src/music_kraken/target/__init__.py](/src/music_kraken/target/__init__.py) | Python | 4 | 0 | 2 | 6 | -| [src/music_kraken/target/set_target.py](/src/music_kraken/target/set_target.py) | Python | 37 | 7 | 18 | 62 | -| [src/music_kraken/utils/__init__.py](/src/music_kraken/utils/__init__.py) | Python | 1 | 1 | 1 | 3 | -| [src/music_kraken/utils/functions.py](/src/music_kraken/utils/functions.py) | Python | 3 | 0 | 1 | 4 | -| [src/music_kraken/utils/object_handeling.py](/src/music_kraken/utils/object_handeling.py) | Python | 19 | 0 | 6 | 25 | -| [src/music_kraken/utils/phonetic_compares.py](/src/music_kraken/utils/phonetic_compares.py) | Python | 39 | 2 | 17 | 58 | -| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | 62 | 3 | 10 | 75 | -| [src/music_kraken/utils/string_processing.py](/src/music_kraken/utils/string_processing.py) | Python | 2 | 5 | 2 | 9 | -| [src/music_kraken_cli.py](/src/music_kraken_cli.py) | Python | 3 | 0 | 3 | 6 | -| [src/music_kraken_gtk.py](/src/music_kraken_gtk.py) | Python | 3 | 0 | 2 | 5 | -| [src/test.db](/src/test.db) | Database | 91 | 0 | 1 | 92 | -| [src/tests/__init__.py](/src/tests/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/tests/conftest.py](/src/tests/conftest.py) | Python | 3 | 1 | 2 | 6 | -| [src/tests/example_data_objects.py](/src/tests/example_data_objects.py) | Python | 36 | 5 | 6 | 47 | -| [src/try-programming-interface.py](/src/try-programming-interface.py) | Python | 14 | 98 | 22 | 134 | -| [src/try.py](/src/try.py) | Python | 1 | 0 | 3 | 4 | -| [src/try_python.py](/src/try_python.py) | Python | 13 | 20 | 9 | 42 | - -[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/diff-details.md b/.VSCodeCounter/2023-02-27_12-22-06/diff-details.md deleted file mode 100644 index 94a033f..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/diff-details.md +++ /dev/null @@ -1,49 +0,0 @@ -# Diff Details - -Date : 2023-02-27 12:22:06 - -Directory /home/lars/Projects/music-downloader/src - -Total : 34 files, 520 codes, 285 comments, 188 blanks, all 993 lines - -[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details - -## Files -| filename | language | code | comment | blank | total | -| :--- | :--- | ---: | ---: | ---: | ---: | -| [src/metal_archives.py](/src/metal_archives.py) | Python | 3 | -2 | -2 | -1 | -| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | -6 | -18 | -8 | -32 | -| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 116 | 23 | 49 | 188 | -| [src/music_kraken/database/database.py](/src/music_kraken/database/database.py) | Python | -345 | -64 | -85 | -494 | -| [src/music_kraken/database/object_cache.py](/src/music_kraken/database/object_cache.py) | Python | 35 | 56 | 16 | 107 | -| [src/music_kraken/database/objects/__init__.py](/src/music_kraken/database/objects/__init__.py) | Python | -20 | 0 | -7 | -27 | -| [src/music_kraken/database/objects/artist.py](/src/music_kraken/database/objects/artist.py) | Python | -18 | 0 | -5 | -23 | -| [src/music_kraken/database/objects/formatted_text.py](/src/music_kraken/database/objects/formatted_text.py) | Python | -48 | -57 | -16 | -121 | -| [src/music_kraken/database/objects/metadata.py](/src/music_kraken/database/objects/metadata.py) | Python | -251 | -68 | -61 | -380 | -| [src/music_kraken/database/objects/parents.py](/src/music_kraken/database/objects/parents.py) | Python | -40 | -8 | -19 | -67 | -| [src/music_kraken/database/objects/song.py](/src/music_kraken/database/objects/song.py) | Python | -323 | -64 | -85 | -472 | -| [src/music_kraken/database/objects/source.py](/src/music_kraken/database/objects/source.py) | Python | -116 | -38 | -41 | -195 | -| [src/music_kraken/database/old_database.py](/src/music_kraken/database/old_database.py) | Python | 432 | 154 | 115 | 701 | -| [src/music_kraken/database/read.py](/src/music_kraken/database/read.py) | Python | 0 | 0 | 1 | 1 | -| [src/music_kraken/database/write.py](/src/music_kraken/database/write.py) | Python | 168 | 62 | 55 | 285 | -| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 23 | 0 | 7 | 30 | -| [src/music_kraken/objects/album.py](/src/music_kraken/objects/album.py) | Python | 15 | 6 | 5 | 26 | -| [src/music_kraken/objects/artist.py](/src/music_kraken/objects/artist.py) | Python | 18 | 0 | 5 | 23 | -| [src/music_kraken/objects/collection.py](/src/music_kraken/objects/collection.py) | Python | 52 | 15 | 24 | 91 | -| [src/music_kraken/objects/formatted_text.py](/src/music_kraken/objects/formatted_text.py) | Python | 53 | 57 | 20 | 130 | -| [src/music_kraken/objects/lyrics.py](/src/music_kraken/objects/lyrics.py) | Python | 21 | 0 | 7 | 28 | -| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 262 | 68 | 63 | 393 | -| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 22 | 15 | 12 | 49 | -| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 332 | 81 | 89 | 502 | -| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 116 | 38 | 41 | 195 | -| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 22 | 7 | 7 | 36 | -| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | 3 | 0 | 0 | 3 | -| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | 42 | 6 | 15 | 63 | -| [src/music_kraken_cli.py](/src/music_kraken_cli.py) | Python | -88 | -9 | -29 | -126 | -| [src/tests/__init__.py](/src/tests/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/tests/conftest.py](/src/tests/conftest.py) | Python | 3 | 1 | 2 | 6 | -| [src/tests/example_data_objects.py](/src/tests/example_data_objects.py) | Python | 36 | 5 | 6 | 47 | -| [src/try.py](/src/try.py) | Python | 1 | 0 | 3 | 4 | -| [src/try_python.py](/src/try_python.py) | Python | 0 | 19 | 3 | 22 | - -[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/diff.csv b/.VSCodeCounter/2023-02-27_12-22-06/diff.csv deleted file mode 100644 index f377d2d..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/diff.csv +++ /dev/null @@ -1,36 +0,0 @@ -"filename", "language", "Python", "comment", "blank", "total" -"/home/lars/Projects/music-downloader/src/metal_archives.py", "Python", 3, -2, -2, -1 -"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", -6, -18, -8, -32 -"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 116, 23, 49, 188 -"/home/lars/Projects/music-downloader/src/music_kraken/database/database.py", "Python", -345, -64, -85, -494 -"/home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py", "Python", 35, 56, 16, 107 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/__init__.py", "Python", -20, 0, -7, -27 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/artist.py", "Python", -18, 0, -5, -23 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/formatted_text.py", "Python", -48, -57, -16, -121 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/metadata.py", "Python", -251, -68, -61, -380 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/parents.py", "Python", -40, -8, -19, -67 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/song.py", "Python", -323, -64, -85, -472 -"/home/lars/Projects/music-downloader/src/music_kraken/database/objects/source.py", "Python", -116, -38, -41, -195 -"/home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py", "Python", 432, 154, 115, 701 -"/home/lars/Projects/music-downloader/src/music_kraken/database/read.py", "Python", 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/database/write.py", "Python", 168, 62, 55, 285 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 23, 0, 7, 30 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/album.py", "Python", 15, 6, 5, 26 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py", "Python", 18, 0, 5, 23 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py", "Python", 52, 15, 24, 91 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py", "Python", 53, 57, 20, 130 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py", "Python", 21, 0, 7, 28 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 262, 68, 63, 393 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 22, 15, 12, 49 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 332, 81, 89, 502 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 116, 38, 41, 195 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 22, 7, 7, 36 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", 3, 0, 0, 3 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", 42, 6, 15, 63 -"/home/lars/Projects/music-downloader/src/music_kraken_cli.py", "Python", -88, -9, -29, -126 -"/home/lars/Projects/music-downloader/src/tests/__init__.py", "Python", 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/tests/conftest.py", "Python", 3, 1, 2, 6 -"/home/lars/Projects/music-downloader/src/tests/example_data_objects.py", "Python", 36, 5, 6, 47 -"/home/lars/Projects/music-downloader/src/try.py", "Python", 1, 0, 3, 4 -"/home/lars/Projects/music-downloader/src/try_python.py", "Python", 0, 19, 3, 22 -"Total", "-", 520, 285, 188, 993 \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/diff.md b/.VSCodeCounter/2023-02-27_12-22-06/diff.md deleted file mode 100644 index ab5ef72..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/diff.md +++ /dev/null @@ -1,30 +0,0 @@ -# Diff Summary - -Date : 2023-02-27 12:22:06 - -Directory /home/lars/Projects/music-downloader/src - -Total : 34 files, 520 codes, 285 comments, 188 blanks, all 993 lines - -[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) - -## Languages -| language | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| Python | 34 | 520 | 285 | 188 | 993 | - -## Directories -| path | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| . | 34 | 520 | 285 | 188 | 993 | -| . (Files) | 4 | -84 | 8 | -25 | -101 | -| music_kraken | 27 | 565 | 271 | 204 | 1,040 | -| music_kraken (Files) | 1 | -6 | -18 | -8 | -32 | -| music_kraken/database | 13 | -410 | -4 | -83 | -497 | -| music_kraken/database (Files) | 6 | 406 | 231 | 151 | 788 | -| music_kraken/database/objects | 7 | -816 | -235 | -234 | -1,285 | -| music_kraken/objects | 11 | 936 | 287 | 280 | 1,503 | -| music_kraken/pages | 2 | 45 | 6 | 15 | 66 | -| tests | 3 | 39 | 6 | 9 | 54 | - -[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/diff.txt b/.VSCodeCounter/2023-02-27_12-22-06/diff.txt deleted file mode 100644 index fe120cc..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/diff.txt +++ /dev/null @@ -1,67 +0,0 @@ -Date : 2023-02-27 12:22:06 -Directory : /home/lars/Projects/music-downloader/src -Total : 34 files, 520 codes, 285 comments, 188 blanks, all 993 lines - -Languages -+----------+------------+------------+------------+------------+------------+ -| language | files | code | comment | blank | total | -+----------+------------+------------+------------+------------+------------+ -| Python | 34 | 520 | 285 | 188 | 993 | -+----------+------------+------------+------------+------------+------------+ - -Directories -+------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| path | files | code | comment | blank | total | -+------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| . | 34 | 520 | 285 | 188 | 993 | -| . (Files) | 4 | -84 | 8 | -25 | -101 | -| music_kraken | 27 | 565 | 271 | 204 | 1,040 | -| music_kraken (Files) | 1 | -6 | -18 | -8 | -32 | -| music_kraken/database | 13 | -410 | -4 | -83 | -497 | -| music_kraken/database (Files) | 6 | 406 | 231 | 151 | 788 | -| music_kraken/database/objects | 7 | -816 | -235 | -234 | -1,285 | -| music_kraken/objects | 11 | 936 | 287 | 280 | 1,503 | -| music_kraken/pages | 2 | 45 | 6 | 15 | 66 | -| tests | 3 | 39 | 6 | 9 | 54 | -+------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ - -Files -+------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| filename | language | code | comment | blank | total | -+------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| /home/lars/Projects/music-downloader/src/metal_archives.py | Python | 3 | -2 | -2 | -1 | -| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | -6 | -18 | -8 | -32 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 116 | 23 | 49 | 188 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/database.py | Python | -345 | -64 | -85 | -494 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py | Python | 35 | 56 | 16 | 107 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/__init__.py | Python | -20 | 0 | -7 | -27 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/artist.py | Python | -18 | 0 | -5 | -23 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/formatted_text.py | Python | -48 | -57 | -16 | -121 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/metadata.py | Python | -251 | -68 | -61 | -380 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/parents.py | Python | -40 | -8 | -19 | -67 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/song.py | Python | -323 | -64 | -85 | -472 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/objects/source.py | Python | -116 | -38 | -41 | -195 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py | Python | 432 | 154 | 115 | 701 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/read.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/write.py | Python | 168 | 62 | 55 | 285 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 23 | 0 | 7 | 30 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/album.py | Python | 15 | 6 | 5 | 26 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py | Python | 18 | 0 | 5 | 23 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py | Python | 52 | 15 | 24 | 91 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py | Python | 53 | 57 | 20 | 130 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py | Python | 21 | 0 | 7 | 28 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 262 | 68 | 63 | 393 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 22 | 15 | 12 | 49 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 332 | 81 | 89 | 502 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 116 | 38 | 41 | 195 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 22 | 7 | 7 | 36 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | 3 | 0 | 0 | 3 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | 42 | 6 | 15 | 63 | -| /home/lars/Projects/music-downloader/src/music_kraken_cli.py | Python | -88 | -9 | -29 | -126 | -| /home/lars/Projects/music-downloader/src/tests/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/tests/conftest.py | Python | 3 | 1 | 2 | 6 | -| /home/lars/Projects/music-downloader/src/tests/example_data_objects.py | Python | 36 | 5 | 6 | 47 | -| /home/lars/Projects/music-downloader/src/try.py | Python | 1 | 0 | 3 | 4 | -| /home/lars/Projects/music-downloader/src/try_python.py | Python | 0 | 19 | 3 | 22 | -| Total | | 520 | 285 | 188 | 993 | -+------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/results.csv b/.VSCodeCounter/2023-02-27_12-22-06/results.csv deleted file mode 100644 index b1cd8bb..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/results.csv +++ /dev/null @@ -1,65 +0,0 @@ -"filename", "language", "Python", "SQLite", "Database", "comment", "blank", "total" -"/home/lars/Projects/music-downloader/src/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/create_custom_objects.py", "Python", 80, 0, 0, 3, 18, 101 -"/home/lars/Projects/music-downloader/src/metal_archives.py", "Python", 48, 0, 0, 4, 15, 67 -"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", 57, 0, 0, 8, 25, 90 -"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 3, 0, 0, 2, 3, 8 -"/home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py", "Python", 18, 0, 0, 0, 5, 23 -"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 116, 0, 0, 23, 49, 188 -"/home/lars/Projects/music-downloader/src/music_kraken/database/database.py", "Python", 84, 0, 0, 48, 26, 158 -"/home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py", "Python", 35, 0, 0, 56, 16, 107 -"/home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py", "Python", 432, 0, 0, 154, 115, 701 -"/home/lars/Projects/music-downloader/src/music_kraken/database/read.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py", "Python", 12, 0, 0, 0, 8, 20 -"/home/lars/Projects/music-downloader/src/music_kraken/database/write.py", "Python", 168, 0, 0, 62, 55, 285 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py", "Python", 0, 0, 0, 0, 3, 3 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py", "Python", 75, 0, 0, 12, 20, 107 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py", "Python", 54, 0, 0, 1, 16, 71 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py", "Python", 6, 0, 0, 0, 2, 8 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py", "Python", 257, 0, 0, 24, 65, 346 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py", "Python", 253, 0, 0, 40, 72, 365 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py", "Python", 3, 0, 0, 0, 2, 5 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py", "Python", 42, 0, 0, 6, 12, 60 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py", "Python", 115, 0, 0, 16, 42, 173 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py", "Python", 40, 0, 0, 0, 18, 58 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py", "Python", 136, 0, 0, 9, 37, 182 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py", "Python", 11, 0, 0, 5, 8, 24 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py", "Python", 71, 0, 0, 4, 24, 99 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 23, 0, 0, 0, 7, 30 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/album.py", "Python", 15, 0, 0, 6, 5, 26 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py", "Python", 18, 0, 0, 0, 5, 23 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py", "Python", 52, 0, 0, 15, 24, 91 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py", "Python", 53, 0, 0, 57, 20, 130 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py", "Python", 21, 0, 0, 0, 7, 28 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 262, 0, 0, 68, 63, 393 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 22, 0, 0, 15, 12, 49 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 332, 0, 0, 81, 89, 502 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 116, 0, 0, 38, 41, 195 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 22, 0, 0, 7, 7, 36 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", 7, 0, 0, 0, 5, 12 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", 73, 0, 0, 68, 27, 168 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", 341, 0, 0, 66, 91, 498 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py", "Python", 25, 0, 0, 16, 6, 47 -"/home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql", "SQLite", 0, 72, 0, 0, 10, 82 -"/home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql", "SQLite", 0, 135, 0, 0, 10, 145 -"/home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py", "Python", 8, 0, 0, 0, 2, 10 -"/home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py", "Python", 51, 0, 0, 4, 20, 75 -"/home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py", "Python", 4, 0, 0, 0, 2, 6 -"/home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py", "Python", 37, 0, 0, 7, 18, 62 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py", "Python", 1, 0, 0, 1, 1, 3 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py", "Python", 3, 0, 0, 0, 1, 4 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py", "Python", 19, 0, 0, 0, 6, 25 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py", "Python", 39, 0, 0, 2, 17, 58 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", 62, 0, 0, 3, 10, 75 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py", "Python", 2, 0, 0, 5, 2, 9 -"/home/lars/Projects/music-downloader/src/music_kraken_cli.py", "Python", 3, 0, 0, 0, 3, 6 -"/home/lars/Projects/music-downloader/src/music_kraken_gtk.py", "Python", 3, 0, 0, 0, 2, 5 -"/home/lars/Projects/music-downloader/src/test.db", "Database", 0, 0, 91, 0, 1, 92 -"/home/lars/Projects/music-downloader/src/tests/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/tests/conftest.py", "Python", 3, 0, 0, 1, 2, 6 -"/home/lars/Projects/music-downloader/src/tests/example_data_objects.py", "Python", 36, 0, 0, 5, 6, 47 -"/home/lars/Projects/music-downloader/src/try-programming-interface.py", "Python", 14, 0, 0, 98, 22, 134 -"/home/lars/Projects/music-downloader/src/try.py", "Python", 1, 0, 0, 0, 3, 4 -"/home/lars/Projects/music-downloader/src/try_python.py", "Python", 13, 0, 0, 20, 9, 42 -"Total", "-", 3797, 207, 91, 1060, 1216, 6371 \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/results.json b/.VSCodeCounter/2023-02-27_12-22-06/results.json deleted file mode 100644 index 7e8791c..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/results.json +++ /dev/null @@ -1 +0,0 @@ -{"file:///home/lars/Projects/music-downloader/src/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/try_python.py":{"language":"Python","code":13,"comment":20,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/database.py":{"language":"Python","code":84,"comment":48,"blank":26},"file:///home/lars/Projects/music-downloader/src/music_kraken/__init__.py":{"language":"Python","code":57,"comment":8,"blank":25},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py":{"language":"Python","code":116,"comment":23,"blank":49},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/read.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/write.py":{"language":"Python","code":168,"comment":62,"blank":55},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py":{"language":"Python","code":432,"comment":154,"blank":115},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py":{"language":"Python","code":18,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py":{"language":"Python","code":35,"comment":56,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py":{"language":"Python","code":12,"comment":0,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py":{"language":"Python","code":1,"comment":1,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py":{"language":"Python","code":2,"comment":5,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py":{"language":"Python","code":19,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py":{"language":"Python","code":39,"comment":2,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py":{"language":"Python","code":62,"comment":3,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py":{"language":"Python","code":7,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py":{"language":"Python","code":73,"comment":68,"blank":27},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py":{"language":"Python","code":25,"comment":16,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py":{"language":"Python","code":23,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/song.py":{"language":"Python","code":332,"comment":81,"blank":89},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/album.py":{"language":"Python","code":15,"comment":6,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py":{"language":"Python","code":341,"comment":66,"blank":91},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql":{"language":"SQLite","code":72,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql":{"language":"SQLite","code":135,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/source.py":{"language":"Python","code":116,"comment":38,"blank":41},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py":{"language":"Python","code":262,"comment":68,"blank":63},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/target.py":{"language":"Python","code":22,"comment":7,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py":{"language":"Python","code":22,"comment":15,"blank":12},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py":{"language":"Python","code":52,"comment":15,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py":{"language":"Python","code":21,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py":{"language":"Python","code":53,"comment":57,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py":{"language":"Python","code":18,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py":{"language":"Python","code":8,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/__main__.py":{"language":"Python","code":3,"comment":2,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py":{"language":"Python","code":4,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py":{"language":"Python","code":37,"comment":7,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py":{"language":"Python","code":0,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py":{"language":"Python","code":11,"comment":5,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py":{"language":"Python","code":75,"comment":12,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py":{"language":"Python","code":115,"comment":16,"blank":42},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py":{"language":"Python","code":71,"comment":4,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py":{"language":"Python","code":136,"comment":9,"blank":37},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py":{"language":"Python","code":40,"comment":0,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py":{"language":"Python","code":54,"comment":1,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py":{"language":"Python","code":6,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py":{"language":"Python","code":253,"comment":40,"blank":72},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py":{"language":"Python","code":42,"comment":6,"blank":12},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py":{"language":"Python","code":257,"comment":24,"blank":65},"file:///home/lars/Projects/music-downloader/src/test.db":{"language":"Database","code":91,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/tests/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/tests/conftest.py":{"language":"Python","code":3,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/tests/example_data_objects.py":{"language":"Python","code":36,"comment":5,"blank":6},"file:///home/lars/Projects/music-downloader/src/try.py":{"language":"Python","code":1,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/metal_archives.py":{"language":"Python","code":48,"comment":4,"blank":15},"file:///home/lars/Projects/music-downloader/src/create_custom_objects.py":{"language":"Python","code":80,"comment":3,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken_gtk.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken_cli.py":{"language":"Python","code":3,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/try-programming-interface.py":{"language":"Python","code":14,"comment":98,"blank":22},"file:///home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py":{"language":"Python","code":51,"comment":4,"blank":20}} \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/results.md b/.VSCodeCounter/2023-02-27_12-22-06/results.md deleted file mode 100644 index 5b6ad3b..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/results.md +++ /dev/null @@ -1,40 +0,0 @@ -# Summary - -Date : 2023-02-27 12:22:06 - -Directory /home/lars/Projects/music-downloader/src - -Total : 63 files, 4095 codes, 1060 comments, 1216 blanks, all 6371 lines - -Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) - -## Languages -| language | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| Python | 60 | 3,797 | 1,060 | 1,195 | 6,052 | -| SQLite | 2 | 207 | 0 | 20 | 227 | -| Database | 1 | 91 | 0 | 1 | 92 | - -## Directories -| path | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| . | 63 | 4,095 | 1,060 | 1,216 | 6,371 | -| . (Files) | 9 | 253 | 125 | 74 | 452 | -| music_kraken | 51 | 3,803 | 929 | 1,133 | 5,865 | -| music_kraken (Files) | 2 | 60 | 10 | 28 | 98 | -| music_kraken/database | 8 | 865 | 343 | 275 | 1,483 | -| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | -| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | -| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | -| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | -| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | -| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | -| music_kraken/objects | 11 | 936 | 287 | 280 | 1,503 | -| music_kraken/pages | 4 | 446 | 150 | 129 | 725 | -| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | -| music_kraken/tagging | 2 | 59 | 4 | 22 | 85 | -| music_kraken/target | 2 | 41 | 7 | 20 | 68 | -| music_kraken/utils | 6 | 126 | 11 | 37 | 174 | -| tests | 3 | 39 | 6 | 9 | 54 | - -Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-27_12-22-06/results.txt b/.VSCodeCounter/2023-02-27_12-22-06/results.txt deleted file mode 100644 index 1c4a102..0000000 --- a/.VSCodeCounter/2023-02-27_12-22-06/results.txt +++ /dev/null @@ -1,106 +0,0 @@ -Date : 2023-02-27 12:22:06 -Directory : /home/lars/Projects/music-downloader/src -Total : 63 files, 4095 codes, 1060 comments, 1216 blanks, all 6371 lines - -Languages -+----------+------------+------------+------------+------------+------------+ -| language | files | code | comment | blank | total | -+----------+------------+------------+------------+------------+------------+ -| Python | 60 | 3,797 | 1,060 | 1,195 | 6,052 | -| SQLite | 2 | 207 | 0 | 20 | 227 | -| Database | 1 | 91 | 0 | 1 | 92 | -+----------+------------+------------+------------+------------+------------+ - -Directories -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| path | files | code | comment | blank | total | -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| . | 63 | 4,095 | 1,060 | 1,216 | 6,371 | -| . (Files) | 9 | 253 | 125 | 74 | 452 | -| music_kraken | 51 | 3,803 | 929 | 1,133 | 5,865 | -| music_kraken (Files) | 2 | 60 | 10 | 28 | 98 | -| music_kraken/database | 8 | 865 | 343 | 275 | 1,483 | -| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | -| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | -| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | -| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | -| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | -| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | -| music_kraken/objects | 11 | 936 | 287 | 280 | 1,503 | -| music_kraken/pages | 4 | 446 | 150 | 129 | 725 | -| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | -| music_kraken/tagging | 2 | 59 | 4 | 22 | 85 | -| music_kraken/target | 2 | 41 | 7 | 20 | 68 | -| music_kraken/utils | 6 | 126 | 11 | 37 | 174 | -| tests | 3 | 39 | 6 | 9 | 54 | -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ - -Files -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| filename | language | code | comment | blank | total | -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| /home/lars/Projects/music-downloader/src/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/create_custom_objects.py | Python | 80 | 3 | 18 | 101 | -| /home/lars/Projects/music-downloader/src/metal_archives.py | Python | 48 | 4 | 15 | 67 | -| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | 57 | 8 | 25 | 90 | -| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 3 | 2 | 3 | 8 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py | Python | 18 | 0 | 5 | 23 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 116 | 23 | 49 | 188 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/database.py | Python | 84 | 48 | 26 | 158 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py | Python | 35 | 56 | 16 | 107 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py | Python | 432 | 154 | 115 | 701 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/read.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py | Python | 12 | 0 | 8 | 20 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/write.py | Python | 168 | 62 | 55 | 285 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py | Python | 0 | 0 | 3 | 3 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py | Python | 75 | 12 | 20 | 107 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py | Python | 54 | 1 | 16 | 71 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py | Python | 6 | 0 | 2 | 8 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py | Python | 257 | 24 | 65 | 346 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py | Python | 253 | 40 | 72 | 365 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py | Python | 3 | 0 | 2 | 5 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py | Python | 42 | 6 | 12 | 60 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py | Python | 115 | 16 | 42 | 173 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py | Python | 40 | 0 | 18 | 58 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py | Python | 136 | 9 | 37 | 182 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py | Python | 11 | 5 | 8 | 24 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py | Python | 71 | 4 | 24 | 99 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 23 | 0 | 7 | 30 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/album.py | Python | 15 | 6 | 5 | 26 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py | Python | 18 | 0 | 5 | 23 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py | Python | 52 | 15 | 24 | 91 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py | Python | 53 | 57 | 20 | 130 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py | Python | 21 | 0 | 7 | 28 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 262 | 68 | 63 | 393 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 22 | 15 | 12 | 49 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 332 | 81 | 89 | 502 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 116 | 38 | 41 | 195 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 22 | 7 | 7 | 36 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | 7 | 0 | 5 | 12 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | 73 | 68 | 27 | 168 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | 341 | 66 | 91 | 498 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py | Python | 25 | 16 | 6 | 47 | -| /home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql | SQLite | 72 | 0 | 10 | 82 | -| /home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql | SQLite | 135 | 0 | 10 | 145 | -| /home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py | Python | 8 | 0 | 2 | 10 | -| /home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py | Python | 51 | 4 | 20 | 75 | -| /home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py | Python | 4 | 0 | 2 | 6 | -| /home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py | Python | 37 | 7 | 18 | 62 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py | Python | 1 | 1 | 1 | 3 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py | Python | 3 | 0 | 1 | 4 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py | Python | 19 | 0 | 6 | 25 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py | Python | 39 | 2 | 17 | 58 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | 62 | 3 | 10 | 75 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py | Python | 2 | 5 | 2 | 9 | -| /home/lars/Projects/music-downloader/src/music_kraken_cli.py | Python | 3 | 0 | 3 | 6 | -| /home/lars/Projects/music-downloader/src/music_kraken_gtk.py | Python | 3 | 0 | 2 | 5 | -| /home/lars/Projects/music-downloader/src/test.db | Database | 91 | 0 | 1 | 92 | -| /home/lars/Projects/music-downloader/src/tests/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/tests/conftest.py | Python | 3 | 1 | 2 | 6 | -| /home/lars/Projects/music-downloader/src/tests/example_data_objects.py | Python | 36 | 5 | 6 | 47 | -| /home/lars/Projects/music-downloader/src/try-programming-interface.py | Python | 14 | 98 | 22 | 134 | -| /home/lars/Projects/music-downloader/src/try.py | Python | 1 | 0 | 3 | 4 | -| /home/lars/Projects/music-downloader/src/try_python.py | Python | 13 | 20 | 9 | 42 | -| Total | | 4,095 | 1,060 | 1,216 | 6,371 | -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/details.md b/.VSCodeCounter/2023-05-24_11-17-57/details.md new file mode 100644 index 0000000..3f7aca9 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/details.md @@ -0,0 +1,104 @@ +# Details + +Date : 2023-05-24 11:17:57 + +Directory /home/lars/Projects/music-downloader/src + +Total : 89 files, 6536 codes, 1157 comments, 1980 blanks, all 9673 lines + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | +| [src/__init__.py](/src/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/actual_donwload.py](/src/actual_donwload.py) | Python | 43 | 2 | 18 | 63 | +| [src/create_custom_objects.py](/src/create_custom_objects.py) | Python | 58 | 0 | 6 | 64 | +| [src/metal_archives.py](/src/metal_archives.py) | Python | 30 | 0 | 12 | 42 | +| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | 189 | 21 | 60 | 270 | +| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 88 | 3 | 21 | 112 | +| [src/music_kraken/audio/__init__.py](/src/music_kraken/audio/__init__.py) | Python | 7 | 0 | 3 | 10 | +| [src/music_kraken/audio/codec.py](/src/music_kraken/audio/codec.py) | Python | 25 | 0 | 8 | 33 | +| [src/music_kraken/audio/metadata.py](/src/music_kraken/audio/metadata.py) | Python | 60 | 4 | 24 | 88 | +| [src/music_kraken/connection/__init__.py](/src/music_kraken/connection/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/connection/connection.py](/src/music_kraken/connection/connection.py) | Python | 162 | 1 | 30 | 193 | +| [src/music_kraken/connection/rotating.py](/src/music_kraken/connection/rotating.py) | Python | 27 | 3 | 14 | 44 | +| [src/music_kraken/database/__init__.py](/src/music_kraken/database/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 122 | 24 | 52 | 198 | +| [src/music_kraken/database/database.py](/src/music_kraken/database/database.py) | Python | 104 | 47 | 38 | 189 | +| [src/music_kraken/download/__init__.py](/src/music_kraken/download/__init__.py) | Python | 2 | 0 | 1 | 3 | +| [src/music_kraken/download/download.py](/src/music_kraken/download/download.py) | Python | 35 | 0 | 14 | 49 | +| [src/music_kraken/download/multiple_options.py](/src/music_kraken/download/multiple_options.py) | Python | 69 | 0 | 31 | 100 | +| [src/music_kraken/download/page_attributes.py](/src/music_kraken/download/page_attributes.py) | Python | 21 | 1 | 10 | 32 | +| [src/music_kraken/download/search.py](/src/music_kraken/download/search.py) | Python | 130 | 24 | 56 | 210 | +| [src/music_kraken/not_used_anymore/__init__.py](/src/music_kraken/not_used_anymore/__init__.py) | Python | 0 | 0 | 3 | 3 | +| [src/music_kraken/not_used_anymore/fetch_audio.py](/src/music_kraken/not_used_anymore/fetch_audio.py) | Python | 75 | 12 | 20 | 107 | +| [src/music_kraken/not_used_anymore/fetch_source.py](/src/music_kraken/not_used_anymore/fetch_source.py) | Python | 54 | 1 | 16 | 71 | +| [src/music_kraken/not_used_anymore/metadata/__init__.py](/src/music_kraken/not_used_anymore/metadata/__init__.py) | Python | 6 | 0 | 2 | 8 | +| [src/music_kraken/not_used_anymore/metadata/metadata_fetch.py](/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py) | Python | 257 | 24 | 65 | 346 | +| [src/music_kraken/not_used_anymore/metadata/metadata_search.py](/src/music_kraken/not_used_anymore/metadata/metadata_search.py) | Python | 253 | 40 | 72 | 365 | +| [src/music_kraken/not_used_anymore/metadata/sources/__init__.py](/src/music_kraken/not_used_anymore/metadata/sources/__init__.py) | Python | 3 | 0 | 2 | 5 | +| [src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py](/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py) | Python | 42 | 6 | 12 | 60 | +| [src/music_kraken/not_used_anymore/sources/__init__.py](/src/music_kraken/not_used_anymore/sources/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/not_used_anymore/sources/genius.py](/src/music_kraken/not_used_anymore/sources/genius.py) | Python | 115 | 16 | 42 | 173 | +| [src/music_kraken/not_used_anymore/sources/local_files.py](/src/music_kraken/not_used_anymore/sources/local_files.py) | Python | 40 | 0 | 18 | 58 | +| [src/music_kraken/not_used_anymore/sources/musify.py](/src/music_kraken/not_used_anymore/sources/musify.py) | Python | 136 | 9 | 37 | 182 | +| [src/music_kraken/not_used_anymore/sources/source.py](/src/music_kraken/not_used_anymore/sources/source.py) | Python | 11 | 5 | 8 | 24 | +| [src/music_kraken/not_used_anymore/sources/youtube.py](/src/music_kraken/not_used_anymore/sources/youtube.py) | Python | 71 | 4 | 24 | 99 | +| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 14 | 0 | 5 | 19 | +| [src/music_kraken/objects/cache.py](/src/music_kraken/objects/cache.py) | Python | 37 | 56 | 18 | 111 | +| [src/music_kraken/objects/collection.py](/src/music_kraken/objects/collection.py) | Python | 91 | 31 | 39 | 161 | +| [src/music_kraken/objects/formatted_text.py](/src/music_kraken/objects/formatted_text.py) | Python | 50 | 10 | 19 | 79 | +| [src/music_kraken/objects/lyrics.py](/src/music_kraken/objects/lyrics.py) | Python | 25 | 0 | 7 | 32 | +| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 261 | 62 | 61 | 384 | +| [src/music_kraken/objects/option.py](/src/music_kraken/objects/option.py) | Python | 28 | 0 | 13 | 41 | +| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 65 | 33 | 33 | 131 | +| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 473 | 113 | 115 | 701 | +| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 90 | 16 | 32 | 138 | +| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 63 | 15 | 23 | 101 | +| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | 357 | 34 | 103 | 494 | +| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | 432 | 90 | 127 | 649 | +| [src/music_kraken/pages/musify.py](/src/music_kraken/pages/musify.py) | Python | 629 | 289 | 187 | 1,105 | +| [src/music_kraken/pages/preset.py](/src/music_kraken/pages/preset.py) | Python | 43 | 1 | 16 | 60 | +| [src/music_kraken/pages/youtube.py](/src/music_kraken/pages/youtube.py) | Python | 25 | 16 | 6 | 47 | +| [src/music_kraken/static_files/new_db.sql](/src/music_kraken/static_files/new_db.sql) | SQLite | 72 | 0 | 10 | 82 | +| [src/music_kraken/static_files/temp_database_structure.sql](/src/music_kraken/static_files/temp_database_structure.sql) | SQLite | 135 | 0 | 10 | 145 | +| [src/music_kraken/utils/__init__.py](/src/music_kraken/utils/__init__.py) | Python | 2 | 1 | 2 | 5 | +| [src/music_kraken/utils/config/__init__.py](/src/music_kraken/utils/config/__init__.py) | Python | 7 | 0 | 4 | 11 | +| [src/music_kraken/utils/config/audio.py](/src/music_kraken/utils/config/audio.py) | Python | 152 | 15 | 28 | 195 | +| [src/music_kraken/utils/config/base_classes.py](/src/music_kraken/utils/config/base_classes.py) | Python | 136 | 35 | 61 | 232 | +| [src/music_kraken/utils/config/config.py](/src/music_kraken/utils/config/config.py) | Python | 92 | 16 | 30 | 138 | +| [src/music_kraken/utils/config/connection.py](/src/music_kraken/utils/config/connection.py) | Python | 81 | 2 | 15 | 98 | +| [src/music_kraken/utils/config/logging.py](/src/music_kraken/utils/config/logging.py) | Python | 104 | 4 | 17 | 125 | +| [src/music_kraken/utils/config/misc.py](/src/music_kraken/utils/config/misc.py) | Python | 40 | 0 | 9 | 49 | +| [src/music_kraken/utils/config/paths.py](/src/music_kraken/utils/config/paths.py) | Python | 40 | 0 | 13 | 53 | +| [src/music_kraken/utils/enums/__init__.py](/src/music_kraken/utils/enums/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/utils/enums/album.py](/src/music_kraken/utils/enums/album.py) | Python | 16 | 6 | 5 | 27 | +| [src/music_kraken/utils/enums/source.py](/src/music_kraken/utils/enums/source.py) | Python | 40 | 1 | 8 | 49 | +| [src/music_kraken/utils/exception/__init__.py](/src/music_kraken/utils/exception/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/utils/exception/config.py](/src/music_kraken/utils/exception/config.py) | Python | 14 | 8 | 7 | 29 | +| [src/music_kraken/utils/functions.py](/src/music_kraken/utils/functions.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/utils/object_handeling.py](/src/music_kraken/utils/object_handeling.py) | Python | 19 | 0 | 6 | 25 | +| [src/music_kraken/utils/path_manager/__init__.py](/src/music_kraken/utils/path_manager/__init__.py) | Python | 2 | 0 | 2 | 4 | +| [src/music_kraken/utils/path_manager/config_directory.py](/src/music_kraken/utils/path_manager/config_directory.py) | Python | 4 | 0 | 4 | 8 | +| [src/music_kraken/utils/path_manager/locations.py](/src/music_kraken/utils/path_manager/locations.py) | Python | 16 | 0 | 9 | 25 | +| [src/music_kraken/utils/path_manager/music_directory.py](/src/music_kraken/utils/path_manager/music_directory.py) | Python | 36 | 9 | 14 | 59 | +| [src/music_kraken/utils/phonetic_compares.py](/src/music_kraken/utils/phonetic_compares.py) | Python | 39 | 2 | 17 | 58 | +| [src/music_kraken/utils/regex.py](/src/music_kraken/utils/regex.py) | Python | 1 | 0 | 2 | 3 | +| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | 63 | 22 | 21 | 106 | +| [src/music_kraken/utils/string_processing.py](/src/music_kraken/utils/string_processing.py) | Python | 16 | 5 | 11 | 32 | +| [src/music_kraken/utils/support_classes/__init__.py](/src/music_kraken/utils/support_classes/__init__.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/utils/support_classes/default_target.py](/src/music_kraken/utils/support_classes/default_target.py) | Python | 56 | 0 | 15 | 71 | +| [src/music_kraken/utils/support_classes/download_result.py](/src/music_kraken/utils/support_classes/download_result.py) | Python | 69 | 0 | 21 | 90 | +| [src/music_kraken/utils/support_classes/query.py](/src/music_kraken/utils/support_classes/query.py) | Python | 24 | 0 | 9 | 33 | +| [src/music_kraken_cli.py](/src/music_kraken_cli.py) | Python | 3 | 0 | 3 | 6 | +| [src/music_kraken_gtk.py](/src/music_kraken_gtk.py) | Python | 3 | 0 | 2 | 5 | +| [src/musify_search.py](/src/musify_search.py) | Python | 38 | 0 | 14 | 52 | +| [src/tests/__init__.py](/src/tests/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/tests/conftest.py](/src/tests/conftest.py) | Python | 3 | 1 | 2 | 6 | +| [src/tests/test_building_objects.py](/src/tests/test_building_objects.py) | Python | 81 | 1 | 13 | 95 | +| [src/tests/test_download.py](/src/tests/test_download.py) | Python | 30 | 1 | 12 | 43 | +| [src/tests/test_objects.py](/src/tests/test_objects.py) | Python | 173 | 15 | 51 | 239 | + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/diff-details.md b/.VSCodeCounter/2023-05-24_11-17-57/diff-details.md new file mode 100644 index 0000000..279a11c --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/diff-details.md @@ -0,0 +1,76 @@ +# Diff Details + +Date : 2023-05-24 11:17:57 + +Directory /home/lars/Projects/music-downloader/src + +Total : 61 files, 1280 codes, 157 comments, 437 blanks, all 1874 lines + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | +| [src/actual_donwload.py](/src/actual_donwload.py) | Python | 6 | 2 | 1 | 9 | +| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | 103 | 9 | 32 | 144 | +| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 85 | 3 | 18 | 106 | +| [src/music_kraken/audio/__init__.py](/src/music_kraken/audio/__init__.py) | Python | 7 | 0 | 3 | 10 | +| [src/music_kraken/audio/codec.py](/src/music_kraken/audio/codec.py) | Python | 25 | 0 | 8 | 33 | +| [src/music_kraken/audio/metadata.py](/src/music_kraken/audio/metadata.py) | Python | 60 | 4 | 24 | 88 | +| [src/music_kraken/connection/__init__.py](/src/music_kraken/connection/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/connection/connection.py](/src/music_kraken/connection/connection.py) | Python | 162 | 1 | 30 | 193 | +| [src/music_kraken/connection/rotating.py](/src/music_kraken/connection/rotating.py) | Python | 27 | 3 | 14 | 44 | +| [src/music_kraken/download/__init__.py](/src/music_kraken/download/__init__.py) | Python | 2 | 0 | 1 | 3 | +| [src/music_kraken/download/download.py](/src/music_kraken/download/download.py) | Python | 35 | 0 | 14 | 49 | +| [src/music_kraken/download/multiple_options.py](/src/music_kraken/download/multiple_options.py) | Python | 69 | 0 | 31 | 100 | +| [src/music_kraken/download/page_attributes.py](/src/music_kraken/download/page_attributes.py) | Python | 21 | 1 | 10 | 32 | +| [src/music_kraken/download/search.py](/src/music_kraken/download/search.py) | Python | 130 | 24 | 56 | 210 | +| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | -15 | 0 | -3 | -18 | +| [src/music_kraken/objects/album.py](/src/music_kraken/objects/album.py) | Python | -16 | -6 | -5 | -27 | +| [src/music_kraken/objects/option.py](/src/music_kraken/objects/option.py) | Python | 5 | 0 | 2 | 7 | +| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 3 | 2 | 2 | 7 | +| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 37 | 20 | 16 | 73 | +| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | -30 | -1 | -9 | -40 | +| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 5 | 4 | 2 | 11 | +| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | -3 | 0 | -4 | -7 | +| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | -71 | -10 | 0 | -81 | +| [src/music_kraken/pages/download_center/__init__.py](/src/music_kraken/pages/download_center/__init__.py) | Python | -4 | 0 | -2 | -6 | +| [src/music_kraken/pages/download_center/download.py](/src/music_kraken/pages/download_center/download.py) | Python | -33 | 0 | -13 | -46 | +| [src/music_kraken/pages/download_center/multiple_options.py](/src/music_kraken/pages/download_center/multiple_options.py) | Python | -69 | 0 | -31 | -100 | +| [src/music_kraken/pages/download_center/page_attributes.py](/src/music_kraken/pages/download_center/page_attributes.py) | Python | -24 | -1 | -9 | -34 | +| [src/music_kraken/pages/download_center/search.py](/src/music_kraken/pages/download_center/search.py) | Python | -85 | -23 | -38 | -146 | +| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | -19 | 7 | 10 | -2 | +| [src/music_kraken/pages/musify.py](/src/music_kraken/pages/musify.py) | Python | 65 | 30 | 32 | 127 | +| [src/music_kraken/pages/preset.py](/src/music_kraken/pages/preset.py) | Python | 43 | 1 | 16 | 60 | +| [src/music_kraken/pages/support_classes/__init__.py](/src/music_kraken/pages/support_classes/__init__.py) | Python | 0 | 0 | -1 | -1 | +| [src/music_kraken/pages/support_classes/default_target.py](/src/music_kraken/pages/support_classes/default_target.py) | Python | -55 | 0 | -15 | -70 | +| [src/music_kraken/pages/support_classes/download_result.py](/src/music_kraken/pages/support_classes/download_result.py) | Python | -51 | 0 | -15 | -66 | +| [src/music_kraken/tagging/__init__.py](/src/music_kraken/tagging/__init__.py) | Python | -5 | 0 | -2 | -7 | +| [src/music_kraken/tagging/id3.py](/src/music_kraken/tagging/id3.py) | Python | -60 | -4 | -24 | -88 | +| [src/music_kraken/utils/__init__.py](/src/music_kraken/utils/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/utils/config/__init__.py](/src/music_kraken/utils/config/__init__.py) | Python | 7 | 0 | 4 | 11 | +| [src/music_kraken/utils/config/audio.py](/src/music_kraken/utils/config/audio.py) | Python | 152 | 15 | 28 | 195 | +| [src/music_kraken/utils/config/base_classes.py](/src/music_kraken/utils/config/base_classes.py) | Python | 136 | 35 | 61 | 232 | +| [src/music_kraken/utils/config/config.py](/src/music_kraken/utils/config/config.py) | Python | 92 | 16 | 30 | 138 | +| [src/music_kraken/utils/config/connection.py](/src/music_kraken/utils/config/connection.py) | Python | 81 | 2 | 15 | 98 | +| [src/music_kraken/utils/config/logging.py](/src/music_kraken/utils/config/logging.py) | Python | 104 | 4 | 17 | 125 | +| [src/music_kraken/utils/config/misc.py](/src/music_kraken/utils/config/misc.py) | Python | 40 | 0 | 9 | 49 | +| [src/music_kraken/utils/config/paths.py](/src/music_kraken/utils/config/paths.py) | Python | 40 | 0 | 13 | 53 | +| [src/music_kraken/utils/enums/__init__.py](/src/music_kraken/utils/enums/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/utils/enums/album.py](/src/music_kraken/utils/enums/album.py) | Python | 16 | 6 | 5 | 27 | +| [src/music_kraken/utils/enums/source.py](/src/music_kraken/utils/enums/source.py) | Python | 40 | 1 | 8 | 49 | +| [src/music_kraken/utils/exception/__init__.py](/src/music_kraken/utils/exception/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/utils/exception/config.py](/src/music_kraken/utils/exception/config.py) | Python | 14 | 8 | 7 | 29 | +| [src/music_kraken/utils/path_manager/__init__.py](/src/music_kraken/utils/path_manager/__init__.py) | Python | 2 | 0 | 2 | 4 | +| [src/music_kraken/utils/path_manager/config_directory.py](/src/music_kraken/utils/path_manager/config_directory.py) | Python | 4 | 0 | 4 | 8 | +| [src/music_kraken/utils/path_manager/locations.py](/src/music_kraken/utils/path_manager/locations.py) | Python | 16 | 0 | 9 | 25 | +| [src/music_kraken/utils/path_manager/music_directory.py](/src/music_kraken/utils/path_manager/music_directory.py) | Python | 36 | 9 | 14 | 59 | +| [src/music_kraken/utils/regex.py](/src/music_kraken/utils/regex.py) | Python | 1 | 0 | 2 | 3 | +| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | -12 | -5 | 4 | -13 | +| [src/music_kraken/utils/string_processing.py](/src/music_kraken/utils/string_processing.py) | Python | 6 | 0 | 4 | 10 | +| [src/music_kraken/utils/support_classes/__init__.py](/src/music_kraken/utils/support_classes/__init__.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/utils/support_classes/default_target.py](/src/music_kraken/utils/support_classes/default_target.py) | Python | 56 | 0 | 15 | 71 | +| [src/music_kraken/utils/support_classes/download_result.py](/src/music_kraken/utils/support_classes/download_result.py) | Python | 69 | 0 | 21 | 90 | +| [src/music_kraken/utils/support_classes/query.py](/src/music_kraken/utils/support_classes/query.py) | Python | 24 | 0 | 9 | 33 | + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/diff.csv b/.VSCodeCounter/2023-05-24_11-17-57/diff.csv new file mode 100644 index 0000000..34a7e92 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/diff.csv @@ -0,0 +1,63 @@ +"filename", "language", "Python", "comment", "blank", "total" +"/home/lars/Projects/music-downloader/src/actual_donwload.py", "Python", 6, 2, 1, 9 +"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", 103, 9, 32, 144 +"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 85, 3, 18, 106 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py", "Python", 7, 0, 3, 10 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py", "Python", 25, 0, 8, 33 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py", "Python", 60, 4, 24, 88 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py", "Python", 1, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py", "Python", 162, 1, 30, 193 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py", "Python", 27, 3, 14, 44 +"/home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py", "Python", 2, 0, 1, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/download/download.py", "Python", 35, 0, 14, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py", "Python", 69, 0, 31, 100 +"/home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py", "Python", 21, 1, 10, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/download/search.py", "Python", 130, 24, 56, 210 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", -15, 0, -3, -18 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/album.py", "Python", -16, -6, -5, -27 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/option.py", "Python", 5, 0, 2, 7 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 3, 2, 2, 7 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 37, 20, 16, 73 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", -30, -1, -9, -40 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 5, 4, 2, 11 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", -3, 0, -4, -7 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", -71, -10, 0, -81 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/__init__.py", "Python", -4, 0, -2, -6 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/download.py", "Python", -33, 0, -13, -46 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/multiple_options.py", "Python", -69, 0, -31, -100 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/page_attributes.py", "Python", -24, -1, -9, -34 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/search.py", "Python", -85, -23, -38, -146 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", -19, 7, 10, -2 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py", "Python", 65, 30, 32, 127 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py", "Python", 43, 1, 16, 60 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/__init__.py", "Python", 0, 0, -1, -1 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/default_target.py", "Python", -55, 0, -15, -70 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/download_result.py", "Python", -51, 0, -15, -66 +"/home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py", "Python", -5, 0, -2, -7 +"/home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py", "Python", -60, -4, -24, -88 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py", "Python", 1, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py", "Python", 7, 0, 4, 11 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py", "Python", 152, 15, 28, 195 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py", "Python", 136, 35, 61, 232 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py", "Python", 92, 16, 30, 138 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py", "Python", 81, 2, 15, 98 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py", "Python", 104, 4, 17, 125 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py", "Python", 40, 0, 9, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py", "Python", 40, 0, 13, 53 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py", "Python", 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py", "Python", 16, 6, 5, 27 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py", "Python", 40, 1, 8, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py", "Python", 1, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py", "Python", 14, 8, 7, 29 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py", "Python", 2, 0, 2, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py", "Python", 4, 0, 4, 8 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py", "Python", 16, 0, 9, 25 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py", "Python", 36, 9, 14, 59 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py", "Python", 1, 0, 2, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", -12, -5, 4, -13 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py", "Python", 6, 0, 4, 10 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py", "Python", 3, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py", "Python", 56, 0, 15, 71 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py", "Python", 69, 0, 21, 90 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py", "Python", 24, 0, 9, 33 +"Total", "-", 1280, 157, 437, 1874 \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/diff.md b/.VSCodeCounter/2023-05-24_11-17-57/diff.md new file mode 100644 index 0000000..64d4b4d --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/diff.md @@ -0,0 +1,40 @@ +# Diff Summary + +Date : 2023-05-24 11:17:57 + +Directory /home/lars/Projects/music-downloader/src + +Total : 61 files, 1280 codes, 157 comments, 437 blanks, all 1874 lines + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| Python | 61 | 1,280 | 157 | 437 | 1,874 | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| . | 61 | 1,280 | 157 | 437 | 1,874 | +| . (Files) | 1 | 6 | 2 | 1 | 9 | +| music_kraken | 60 | 1,274 | 155 | 436 | 1,865 | +| music_kraken (Files) | 2 | 188 | 12 | 50 | 250 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/connection | 3 | 190 | 4 | 45 | 239 | +| music_kraken/download | 5 | 257 | 25 | 112 | 394 | +| music_kraken/objects | 7 | -11 | 19 | 5 | 13 | +| music_kraken/pages | 13 | -306 | 4 | -70 | -372 | +| music_kraken/pages (Files) | 5 | 15 | 28 | 54 | 97 | +| music_kraken/pages/download_center | 5 | -215 | -24 | -93 | -332 | +| music_kraken/pages/support_classes | 3 | -106 | 0 | -31 | -137 | +| music_kraken/tagging | 2 | -65 | -4 | -26 | -95 | +| music_kraken/utils | 25 | 929 | 91 | 285 | 1,305 | +| music_kraken/utils (Files) | 4 | -4 | -5 | 11 | 2 | +| music_kraken/utils/config | 8 | 652 | 72 | 177 | 901 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 2 | 15 | 8 | 8 | 31 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 4 | 152 | 0 | 46 | 198 | + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/diff.txt b/.VSCodeCounter/2023-05-24_11-17-57/diff.txt new file mode 100644 index 0000000..cb7308f --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/diff.txt @@ -0,0 +1,104 @@ +Date : 2023-05-24 11:17:57 +Directory : /home/lars/Projects/music-downloader/src +Total : 61 files, 1280 codes, 157 comments, 437 blanks, all 1874 lines + +Languages ++----------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++----------+------------+------------+------------+------------+------------+ +| Python | 61 | 1,280 | 157 | 437 | 1,874 | ++----------+------------+------------+------------+------------+------------+ + +Directories ++-------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++-------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| . | 61 | 1,280 | 157 | 437 | 1,874 | +| . (Files) | 1 | 6 | 2 | 1 | 9 | +| music_kraken | 60 | 1,274 | 155 | 436 | 1,865 | +| music_kraken (Files) | 2 | 188 | 12 | 50 | 250 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/connection | 3 | 190 | 4 | 45 | 239 | +| music_kraken/download | 5 | 257 | 25 | 112 | 394 | +| music_kraken/objects | 7 | -11 | 19 | 5 | 13 | +| music_kraken/pages | 13 | -306 | 4 | -70 | -372 | +| music_kraken/pages (Files) | 5 | 15 | 28 | 54 | 97 | +| music_kraken/pages/download_center | 5 | -215 | -24 | -93 | -332 | +| music_kraken/pages/support_classes | 3 | -106 | 0 | -31 | -137 | +| music_kraken/tagging | 2 | -65 | -4 | -26 | -95 | +| music_kraken/utils | 25 | 929 | 91 | 285 | 1,305 | +| music_kraken/utils (Files) | 4 | -4 | -5 | 11 | 2 | +| music_kraken/utils/config | 8 | 652 | 72 | 177 | 901 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 2 | 15 | 8 | 8 | 31 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 4 | 152 | 0 | 46 | 198 | ++-------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ + +Files ++-------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++-------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| /home/lars/Projects/music-downloader/src/actual_donwload.py | Python | 6 | 2 | 1 | 9 | +| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | 103 | 9 | 32 | 144 | +| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 85 | 3 | 18 | 106 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py | Python | 7 | 0 | 3 | 10 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py | Python | 25 | 0 | 8 | 33 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py | Python | 60 | 4 | 24 | 88 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py | Python | 162 | 1 | 30 | 193 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py | Python | 27 | 3 | 14 | 44 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py | Python | 2 | 0 | 1 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/download.py | Python | 35 | 0 | 14 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py | Python | 69 | 0 | 31 | 100 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py | Python | 21 | 1 | 10 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/search.py | Python | 130 | 24 | 56 | 210 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | -15 | 0 | -3 | -18 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/album.py | Python | -16 | -6 | -5 | -27 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/option.py | Python | 5 | 0 | 2 | 7 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 3 | 2 | 2 | 7 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 37 | 20 | 16 | 73 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | -30 | -1 | -9 | -40 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 5 | 4 | 2 | 11 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | -3 | 0 | -4 | -7 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | -71 | -10 | 0 | -81 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/__init__.py | Python | -4 | 0 | -2 | -6 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/download.py | Python | -33 | 0 | -13 | -46 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/multiple_options.py | Python | -69 | 0 | -31 | -100 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/page_attributes.py | Python | -24 | -1 | -9 | -34 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/download_center/search.py | Python | -85 | -23 | -38 | -146 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | -19 | 7 | 10 | -2 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py | Python | 65 | 30 | 32 | 127 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py | Python | 43 | 1 | 16 | 60 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/__init__.py | Python | 0 | 0 | -1 | -1 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/default_target.py | Python | -55 | 0 | -15 | -70 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/support_classes/download_result.py | Python | -51 | 0 | -15 | -66 | +| /home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py | Python | -5 | 0 | -2 | -7 | +| /home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py | Python | -60 | -4 | -24 | -88 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py | Python | 7 | 0 | 4 | 11 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py | Python | 152 | 15 | 28 | 195 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py | Python | 136 | 35 | 61 | 232 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py | Python | 92 | 16 | 30 | 138 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py | Python | 81 | 2 | 15 | 98 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py | Python | 104 | 4 | 17 | 125 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py | Python | 40 | 0 | 9 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py | Python | 40 | 0 | 13 | 53 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py | Python | 16 | 6 | 5 | 27 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py | Python | 40 | 1 | 8 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py | Python | 14 | 8 | 7 | 29 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py | Python | 2 | 0 | 2 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py | Python | 4 | 0 | 4 | 8 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py | Python | 16 | 0 | 9 | 25 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py | Python | 36 | 9 | 14 | 59 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py | Python | 1 | 0 | 2 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | -12 | -5 | 4 | -13 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py | Python | 6 | 0 | 4 | 10 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py | Python | 56 | 0 | 15 | 71 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py | Python | 69 | 0 | 21 | 90 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py | Python | 24 | 0 | 9 | 33 | +| Total | | 1,280 | 157 | 437 | 1,874 | ++-------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/results.csv b/.VSCodeCounter/2023-05-24_11-17-57/results.csv new file mode 100644 index 0000000..2bbbd07 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/results.csv @@ -0,0 +1,91 @@ +"filename", "language", "Python", "SQLite", "comment", "blank", "total" +"/home/lars/Projects/music-downloader/src/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/actual_donwload.py", "Python", 43, 0, 2, 18, 63 +"/home/lars/Projects/music-downloader/src/create_custom_objects.py", "Python", 58, 0, 0, 6, 64 +"/home/lars/Projects/music-downloader/src/metal_archives.py", "Python", 30, 0, 0, 12, 42 +"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", 189, 0, 21, 60, 270 +"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 88, 0, 3, 21, 112 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py", "Python", 7, 0, 0, 3, 10 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py", "Python", 25, 0, 0, 8, 33 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py", "Python", 60, 0, 4, 24, 88 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py", "Python", 1, 0, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py", "Python", 162, 0, 1, 30, 193 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py", "Python", 27, 0, 3, 14, 44 +"/home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 122, 0, 24, 52, 198 +"/home/lars/Projects/music-downloader/src/music_kraken/database/database.py", "Python", 104, 0, 47, 38, 189 +"/home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py", "Python", 2, 0, 0, 1, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/download/download.py", "Python", 35, 0, 0, 14, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py", "Python", 69, 0, 0, 31, 100 +"/home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py", "Python", 21, 0, 1, 10, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/download/search.py", "Python", 130, 0, 24, 56, 210 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py", "Python", 0, 0, 0, 3, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py", "Python", 75, 0, 12, 20, 107 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py", "Python", 54, 0, 1, 16, 71 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py", "Python", 6, 0, 0, 2, 8 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py", "Python", 257, 0, 24, 65, 346 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py", "Python", 253, 0, 40, 72, 365 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py", "Python", 3, 0, 0, 2, 5 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py", "Python", 42, 0, 6, 12, 60 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py", "Python", 115, 0, 16, 42, 173 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py", "Python", 40, 0, 0, 18, 58 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py", "Python", 136, 0, 9, 37, 182 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py", "Python", 11, 0, 5, 8, 24 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py", "Python", 71, 0, 4, 24, 99 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 14, 0, 0, 5, 19 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py", "Python", 37, 0, 56, 18, 111 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py", "Python", 91, 0, 31, 39, 161 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py", "Python", 50, 0, 10, 19, 79 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py", "Python", 25, 0, 0, 7, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 261, 0, 62, 61, 384 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/option.py", "Python", 28, 0, 0, 13, 41 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 65, 0, 33, 33, 131 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 473, 0, 113, 115, 701 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 90, 0, 16, 32, 138 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 63, 0, 15, 23, 101 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", 3, 0, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", 357, 0, 34, 103, 494 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", 432, 0, 90, 127, 649 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py", "Python", 629, 0, 289, 187, 1105 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py", "Python", 43, 0, 1, 16, 60 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py", "Python", 25, 0, 16, 6, 47 +"/home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql", "SQLite", 0, 72, 0, 10, 82 +"/home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql", "SQLite", 0, 135, 0, 10, 145 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py", "Python", 2, 0, 1, 2, 5 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py", "Python", 7, 0, 0, 4, 11 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py", "Python", 152, 0, 15, 28, 195 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py", "Python", 136, 0, 35, 61, 232 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py", "Python", 92, 0, 16, 30, 138 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py", "Python", 81, 0, 2, 15, 98 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py", "Python", 104, 0, 4, 17, 125 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py", "Python", 40, 0, 0, 9, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py", "Python", 40, 0, 0, 13, 53 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py", "Python", 16, 0, 6, 5, 27 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py", "Python", 40, 0, 1, 8, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py", "Python", 1, 0, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py", "Python", 14, 0, 8, 7, 29 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py", "Python", 3, 0, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py", "Python", 19, 0, 0, 6, 25 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py", "Python", 2, 0, 0, 2, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py", "Python", 4, 0, 0, 4, 8 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py", "Python", 16, 0, 0, 9, 25 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py", "Python", 36, 0, 9, 14, 59 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py", "Python", 39, 0, 2, 17, 58 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py", "Python", 1, 0, 0, 2, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", 63, 0, 22, 21, 106 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py", "Python", 16, 0, 5, 11, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py", "Python", 3, 0, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py", "Python", 56, 0, 0, 15, 71 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py", "Python", 69, 0, 0, 21, 90 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py", "Python", 24, 0, 0, 9, 33 +"/home/lars/Projects/music-downloader/src/music_kraken_cli.py", "Python", 3, 0, 0, 3, 6 +"/home/lars/Projects/music-downloader/src/music_kraken_gtk.py", "Python", 3, 0, 0, 2, 5 +"/home/lars/Projects/music-downloader/src/musify_search.py", "Python", 38, 0, 0, 14, 52 +"/home/lars/Projects/music-downloader/src/tests/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/tests/conftest.py", "Python", 3, 0, 1, 2, 6 +"/home/lars/Projects/music-downloader/src/tests/test_building_objects.py", "Python", 81, 0, 1, 13, 95 +"/home/lars/Projects/music-downloader/src/tests/test_download.py", "Python", 30, 0, 1, 12, 43 +"/home/lars/Projects/music-downloader/src/tests/test_objects.py", "Python", 173, 0, 15, 51, 239 +"Total", "-", 6329, 207, 1157, 1980, 9673 \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/results.json b/.VSCodeCounter/2023-05-24_11-17-57/results.json new file mode 100644 index 0000000..5668823 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/results.json @@ -0,0 +1 @@ +{"file:///home/lars/Projects/music-downloader/src/musify_search.py":{"language":"Python","code":38,"comment":0,"blank":14},"file:///home/lars/Projects/music-downloader/src/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/__init__.py":{"language":"Python","code":189,"comment":21,"blank":60},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py":{"language":"Python","code":7,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py":{"language":"Python","code":25,"comment":0,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py":{"language":"Python","code":60,"comment":4,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/database.py":{"language":"Python","code":104,"comment":47,"blank":38},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py":{"language":"Python","code":122,"comment":24,"blank":52},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py":{"language":"Python","code":69,"comment":0,"blank":31},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/search.py":{"language":"Python","code":130,"comment":24,"blank":56},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/download.py":{"language":"Python","code":35,"comment":0,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py":{"language":"Python","code":2,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py":{"language":"Python","code":2,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py":{"language":"Python","code":21,"comment":1,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py":{"language":"Python","code":1,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py":{"language":"Python","code":14,"comment":8,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py":{"language":"Python","code":16,"comment":5,"blank":11},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py":{"language":"Python","code":1,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py":{"language":"Python","code":19,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py":{"language":"Python","code":16,"comment":6,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py":{"language":"Python","code":40,"comment":1,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py":{"language":"Python","code":39,"comment":2,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py":{"language":"Python","code":24,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py":{"language":"Python","code":69,"comment":0,"blank":21},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py":{"language":"Python","code":92,"comment":16,"blank":30},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py":{"language":"Python","code":56,"comment":0,"blank":15},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py":{"language":"Python","code":40,"comment":0,"blank":13},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py":{"language":"Python","code":7,"comment":0,"blank":4},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py":{"language":"Python","code":104,"comment":4,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py":{"language":"Python","code":81,"comment":2,"blank":15},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py":{"language":"Python","code":63,"comment":22,"blank":21},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py":{"language":"Python","code":40,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py":{"language":"Python","code":136,"comment":35,"blank":61},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py":{"language":"Python","code":152,"comment":15,"blank":28},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py":{"language":"Python","code":4,"comment":0,"blank":4},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py":{"language":"Python","code":2,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py":{"language":"Python","code":36,"comment":9,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py":{"language":"Python","code":357,"comment":34,"blank":103},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py":{"language":"Python","code":16,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py":{"language":"Python","code":43,"comment":1,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py":{"language":"Python","code":629,"comment":289,"blank":187},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py":{"language":"Python","code":25,"comment":16,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py":{"language":"Python","code":432,"comment":90,"blank":127},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py":{"language":"Python","code":27,"comment":3,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py":{"language":"Python","code":1,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py":{"language":"Python","code":14,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py":{"language":"Python","code":162,"comment":1,"blank":30},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql":{"language":"SQLite","code":72,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql":{"language":"SQLite","code":135,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py":{"language":"Python","code":261,"comment":62,"blank":61},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py":{"language":"Python","code":37,"comment":56,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/source.py":{"language":"Python","code":90,"comment":16,"blank":32},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py":{"language":"Python","code":65,"comment":33,"blank":33},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/target.py":{"language":"Python","code":63,"comment":15,"blank":23},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py":{"language":"Python","code":25,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/option.py":{"language":"Python","code":28,"comment":0,"blank":13},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/song.py":{"language":"Python","code":473,"comment":113,"blank":115},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py":{"language":"Python","code":91,"comment":31,"blank":39},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py":{"language":"Python","code":50,"comment":10,"blank":19},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py":{"language":"Python","code":0,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/__main__.py":{"language":"Python","code":88,"comment":3,"blank":21},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py":{"language":"Python","code":75,"comment":12,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py":{"language":"Python","code":11,"comment":5,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py":{"language":"Python","code":71,"comment":4,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py":{"language":"Python","code":115,"comment":16,"blank":42},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py":{"language":"Python","code":136,"comment":9,"blank":37},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py":{"language":"Python","code":6,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py":{"language":"Python","code":253,"comment":40,"blank":72},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py":{"language":"Python","code":54,"comment":1,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py":{"language":"Python","code":40,"comment":0,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py":{"language":"Python","code":257,"comment":24,"blank":65},"file:///home/lars/Projects/music-downloader/src/tests/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/tests/test_objects.py":{"language":"Python","code":173,"comment":15,"blank":51},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py":{"language":"Python","code":42,"comment":6,"blank":12},"file:///home/lars/Projects/music-downloader/src/tests/test_building_objects.py":{"language":"Python","code":81,"comment":1,"blank":13},"file:///home/lars/Projects/music-downloader/src/tests/test_download.py":{"language":"Python","code":30,"comment":1,"blank":12},"file:///home/lars/Projects/music-downloader/src/metal_archives.py":{"language":"Python","code":30,"comment":0,"blank":12},"file:///home/lars/Projects/music-downloader/src/tests/conftest.py":{"language":"Python","code":3,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/create_custom_objects.py":{"language":"Python","code":58,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken_gtk.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken_cli.py":{"language":"Python","code":3,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/actual_donwload.py":{"language":"Python","code":43,"comment":2,"blank":18}} \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/results.md b/.VSCodeCounter/2023-05-24_11-17-57/results.md new file mode 100644 index 0000000..ceeb9f0 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/results.md @@ -0,0 +1,46 @@ +# Summary + +Date : 2023-05-24 11:17:57 + +Directory /home/lars/Projects/music-downloader/src + +Total : 89 files, 6536 codes, 1157 comments, 1980 blanks, all 9673 lines + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| Python | 87 | 6,329 | 1,157 | 1,960 | 9,446 | +| SQLite | 2 | 207 | 0 | 20 | 227 | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| . | 89 | 6,536 | 1,157 | 1,980 | 9,673 | +| . (Files) | 7 | 175 | 2 | 56 | 233 | +| music_kraken | 77 | 6,074 | 1,137 | 1,845 | 9,056 | +| music_kraken (Files) | 2 | 277 | 24 | 81 | 382 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/connection | 3 | 190 | 4 | 45 | 239 | +| music_kraken/database | 3 | 226 | 71 | 91 | 388 | +| music_kraken/download | 5 | 257 | 25 | 112 | 394 | +| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | +| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | +| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | +| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | +| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | +| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | +| music_kraken/objects | 11 | 1,197 | 336 | 365 | 1,898 | +| music_kraken/pages | 6 | 1,489 | 430 | 440 | 2,359 | +| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | +| music_kraken/utils | 28 | 1,076 | 126 | 334 | 1,536 | +| music_kraken/utils (Files) | 7 | 143 | 30 | 60 | 233 | +| music_kraken/utils/config | 8 | 652 | 72 | 177 | 901 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 2 | 15 | 8 | 8 | 31 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 4 | 152 | 0 | 46 | 198 | +| tests | 5 | 287 | 18 | 79 | 384 | + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-05-24_11-17-57/results.txt b/.VSCodeCounter/2023-05-24_11-17-57/results.txt new file mode 100644 index 0000000..3dfdbb8 --- /dev/null +++ b/.VSCodeCounter/2023-05-24_11-17-57/results.txt @@ -0,0 +1,138 @@ +Date : 2023-05-24 11:17:57 +Directory : /home/lars/Projects/music-downloader/src +Total : 89 files, 6536 codes, 1157 comments, 1980 blanks, all 9673 lines + +Languages ++----------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++----------+------------+------------+------------+------------+------------+ +| Python | 87 | 6,329 | 1,157 | 1,960 | 9,446 | +| SQLite | 2 | 207 | 0 | 20 | 227 | ++----------+------------+------------+------------+------------+------------+ + +Directories ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| . | 89 | 6,536 | 1,157 | 1,980 | 9,673 | +| . (Files) | 7 | 175 | 2 | 56 | 233 | +| music_kraken | 77 | 6,074 | 1,137 | 1,845 | 9,056 | +| music_kraken (Files) | 2 | 277 | 24 | 81 | 382 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/connection | 3 | 190 | 4 | 45 | 239 | +| music_kraken/database | 3 | 226 | 71 | 91 | 388 | +| music_kraken/download | 5 | 257 | 25 | 112 | 394 | +| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | +| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | +| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | +| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | +| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | +| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | +| music_kraken/objects | 11 | 1,197 | 336 | 365 | 1,898 | +| music_kraken/pages | 6 | 1,489 | 430 | 440 | 2,359 | +| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | +| music_kraken/utils | 28 | 1,076 | 126 | 334 | 1,536 | +| music_kraken/utils (Files) | 7 | 143 | 30 | 60 | 233 | +| music_kraken/utils/config | 8 | 652 | 72 | 177 | 901 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 2 | 15 | 8 | 8 | 31 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 4 | 152 | 0 | 46 | 198 | +| tests | 5 | 287 | 18 | 79 | 384 | ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ + +Files ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| /home/lars/Projects/music-downloader/src/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/actual_donwload.py | Python | 43 | 2 | 18 | 63 | +| /home/lars/Projects/music-downloader/src/create_custom_objects.py | Python | 58 | 0 | 6 | 64 | +| /home/lars/Projects/music-downloader/src/metal_archives.py | Python | 30 | 0 | 12 | 42 | +| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | 189 | 21 | 60 | 270 | +| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 88 | 3 | 21 | 112 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py | Python | 7 | 0 | 3 | 10 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py | Python | 25 | 0 | 8 | 33 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py | Python | 60 | 4 | 24 | 88 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py | Python | 162 | 1 | 30 | 193 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py | Python | 27 | 3 | 14 | 44 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 122 | 24 | 52 | 198 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/database.py | Python | 104 | 47 | 38 | 189 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py | Python | 2 | 0 | 1 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/download.py | Python | 35 | 0 | 14 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py | Python | 69 | 0 | 31 | 100 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py | Python | 21 | 1 | 10 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/search.py | Python | 130 | 24 | 56 | 210 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py | Python | 0 | 0 | 3 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py | Python | 75 | 12 | 20 | 107 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py | Python | 54 | 1 | 16 | 71 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py | Python | 6 | 0 | 2 | 8 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py | Python | 257 | 24 | 65 | 346 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py | Python | 253 | 40 | 72 | 365 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py | Python | 3 | 0 | 2 | 5 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py | Python | 42 | 6 | 12 | 60 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py | Python | 115 | 16 | 42 | 173 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py | Python | 40 | 0 | 18 | 58 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py | Python | 136 | 9 | 37 | 182 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py | Python | 11 | 5 | 8 | 24 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py | Python | 71 | 4 | 24 | 99 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 14 | 0 | 5 | 19 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py | Python | 37 | 56 | 18 | 111 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py | Python | 91 | 31 | 39 | 161 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py | Python | 50 | 10 | 19 | 79 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py | Python | 25 | 0 | 7 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 261 | 62 | 61 | 384 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/option.py | Python | 28 | 0 | 13 | 41 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 65 | 33 | 33 | 131 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 473 | 113 | 115 | 701 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 90 | 16 | 32 | 138 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 63 | 15 | 23 | 101 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | 357 | 34 | 103 | 494 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | 432 | 90 | 127 | 649 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py | Python | 629 | 289 | 187 | 1,105 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py | Python | 43 | 1 | 16 | 60 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py | Python | 25 | 16 | 6 | 47 | +| /home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql | SQLite | 72 | 0 | 10 | 82 | +| /home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql | SQLite | 135 | 0 | 10 | 145 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py | Python | 2 | 1 | 2 | 5 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py | Python | 7 | 0 | 4 | 11 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py | Python | 152 | 15 | 28 | 195 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py | Python | 136 | 35 | 61 | 232 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py | Python | 92 | 16 | 30 | 138 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py | Python | 81 | 2 | 15 | 98 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py | Python | 104 | 4 | 17 | 125 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py | Python | 40 | 0 | 9 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py | Python | 40 | 0 | 13 | 53 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py | Python | 16 | 6 | 5 | 27 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py | Python | 40 | 1 | 8 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py | Python | 14 | 8 | 7 | 29 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py | Python | 19 | 0 | 6 | 25 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py | Python | 2 | 0 | 2 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py | Python | 4 | 0 | 4 | 8 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py | Python | 16 | 0 | 9 | 25 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py | Python | 36 | 9 | 14 | 59 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py | Python | 39 | 2 | 17 | 58 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py | Python | 1 | 0 | 2 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | 63 | 22 | 21 | 106 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py | Python | 16 | 5 | 11 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py | Python | 56 | 0 | 15 | 71 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py | Python | 69 | 0 | 21 | 90 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py | Python | 24 | 0 | 9 | 33 | +| /home/lars/Projects/music-downloader/src/music_kraken_cli.py | Python | 3 | 0 | 3 | 6 | +| /home/lars/Projects/music-downloader/src/music_kraken_gtk.py | Python | 3 | 0 | 2 | 5 | +| /home/lars/Projects/music-downloader/src/musify_search.py | Python | 38 | 0 | 14 | 52 | +| /home/lars/Projects/music-downloader/src/tests/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/tests/conftest.py | Python | 3 | 1 | 2 | 6 | +| /home/lars/Projects/music-downloader/src/tests/test_building_objects.py | Python | 81 | 1 | 13 | 95 | +| /home/lars/Projects/music-downloader/src/tests/test_download.py | Python | 30 | 1 | 12 | 43 | +| /home/lars/Projects/music-downloader/src/tests/test_objects.py | Python | 173 | 15 | 51 | 239 | +| Total | | 6,536 | 1,157 | 1,980 | 9,673 | ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 90a6976..7123bb6 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -21,7 +21,8 @@ from ..objects import ( FormattedText, Label, Options, - Target + Target, + DatabaseObject ) from ..utils.shared import MUSIFY_LOGGER from ..utils import string_processing, shared @@ -63,70 +64,81 @@ class MusifyUrl: name_with_id: str musify_id: str url: str + +ALBUM_TYPE_MAP = defaultdict(lambda: AlbumType.OTHER, { + 1: AlbumType.OTHER, # literally other xD + 2: AlbumType.STUDIO_ALBUM, + 3: AlbumType.EP, + 4: AlbumType.SINGLE, + 5: AlbumType.OTHER, # BOOTLEG + 6: AlbumType.LIVE_ALBUM, + 7: AlbumType.COMPILATION_ALBUM, # compilation of different artists + 8: AlbumType.MIXTAPE, + 9: AlbumType.DEMO, + 10: AlbumType.MIXTAPE, # DJ Mixes + 11: AlbumType.COMPILATION_ALBUM, # compilation of only this artist + 12: AlbumType.STUDIO_ALBUM, # split + 13: AlbumType.COMPILATION_ALBUM, # unofficial + 14: AlbumType.MIXTAPE # "Soundtracks" +}) -class Musify(Page): - API_SESSION: requests.Session = requests.Session() - API_SESSION.headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", - "Connection": "keep-alive", - "Referer": "https://musify.club/" - } - API_SESSION.proxies = shared.proxies - TIMEOUT = 7 - POST_TIMEOUT = 15 - TRIES = 5 - HOST = "https://musify.club" + +def parse_url(url: str) -> MusifyUrl: + parsed = urlparse(url) - CONNECTION = Connection( - host="https://musify.club/", - logger=MUSIFY_LOGGER + path = parsed.path.split("/") + + split_name = path[2].split("-") + url_id = split_name[-1] + name_for_url = "-".join(split_name[:-1]) + + try: + type_enum = MusifyTypes(path[1]) + except ValueError as e: + MUSIFY_LOGGER.warning(f"{path[1]} is not yet implemented, add it to MusifyTypes") + raise e + + return MusifyUrl( + source_type=type_enum, + name_without_id=name_for_url, + name_with_id=path[2], + musify_id=url_id, + url=url ) + +class Preset(Page): + # CHANGE SOURCE_TYPE = SourcePages.MUSIFY + LOGGER = MUSIFY_LOGGER - LOGGER = shared.MUSIFY_LOGGER - - @classmethod - def parse_url(cls, url: str) -> MusifyUrl: - parsed = urlparse(url) - - path = parsed.path.split("/") - - split_name = path[2].split("-") - url_id = split_name[-1] - name_for_url = "-".join(split_name[:-1]) - - try: - type_enum = MusifyTypes(path[1]) - except ValueError as e: - cls.LOGGER.warning(f"{path[1]} is not yet implemented, add it to MusifyTypes") - raise e - - return MusifyUrl( - source_type=type_enum, - name_without_id=name_for_url, - name_with_id=path[2], - musify_id=url_id, - url=url + HOST = "https://musify.club" + + def __init__(self): + self.connection: Connection = Connection( + host="https://musify.club/", + logger=self.LOGGER ) + + super().__init__() - @classmethod - def _raw_search(cls, query: str) -> Options: - query_obj = cls.Query(query) - - if query_obj.is_raw: - return cls.plaintext_search(query_obj.query) - return cls.plaintext_search(cls.get_plaintext_query(query_obj)) - - @classmethod - def get_plaintext_query(cls, query: Query) -> str: - if query.album is None: - return f"{query.artist or '*'} - {query.song or '*'}" - return f"{query.artist or '*'} - {query.album or '*'} - {query.song or '*'}" - - @classmethod - def parse_artist_contact(cls, contact: BeautifulSoup) -> Artist: + def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: + if source.url is None: + return None + + musify_url = parse_url(source.url) + + # Has no labels, because afaiak musify has no Labels + musify_type_to_database_type = { + MusifyTypes.SONG: Song, + MusifyTypes.RELEASE: Album, + MusifyTypes.ARTIST: Artist + } + + return musify_type_to_database_type.get(musify_url.source_type) + + def _parse_artist_contact(self, contact: BeautifulSoup) -> Artist: source_list: List[Source] = [] name = None _id = None @@ -140,7 +152,7 @@ class Musify(Page): if "-" in href: _id = href.split("-")[-1] - source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href)) + source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) # artist image image_soup = contact.find("img") @@ -156,9 +168,8 @@ class Musify(Page): name=name, source_list=source_list ) - - @classmethod - def parse_album_contact(cls, contact: BeautifulSoup) -> Album: + + def _parse_album_contact(self, contact: BeautifulSoup) -> Album: """
@@ -213,7 +224,7 @@ class Musify(Page): if "-" in href: _id = href.split("-")[-1] - source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href)) + source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) # cover art image_soup = contact.find("img") @@ -255,7 +266,7 @@ class Musify(Page): track_count_soup: BeautifulSoup = small_list[1] rating_soup: BeautifulSoup = small_list[2] else: - cls.LOGGER.warning("got an unequal ammount than 3 small elements") + self.LOGGER.warning("got an unequal ammount than 3 small elements") return Album( _id=_id, @@ -264,9 +275,8 @@ class Musify(Page): date=ID3Timestamp(year=year), artist_list=artist_list ) - - @classmethod - def parse_contact_container(cls, contact_container_soup: BeautifulSoup) -> List[Union[Artist, Album]]: + + def _parse_contact_container(self, contact_container_soup: BeautifulSoup) -> List[Union[Artist, Album]]: contacts = [] contact: BeautifulSoup @@ -279,13 +289,12 @@ class Musify(Page): if url is not None: if "artist" in url: - contacts.append(cls.parse_artist_contact(contact)) + contacts.append(self._parse_artist_contact(contact)) elif "release" in url: - contacts.append(cls.parse_album_contact(contact)) + contacts.append(self._parse_album_contact(contact)) return contacts - @classmethod - def parse_playlist_item(cls, playlist_item_soup: BeautifulSoup) -> Song: + def _parse_playlist_item(self, playlist_item_soup: BeautifulSoup) -> Song: _id = None song_title = playlist_item_soup.get("data-name") artist_list: List[Artist] = [] @@ -302,7 +311,7 @@ class Musify(Page): for artist_anchor in anchor_list[:-1]: _id = None href = artist_anchor.get("href") - artist_source: Source = Source(cls.SOURCE_TYPE, cls.HOST + href) + artist_source: Source = Source(self.SOURCE_TYPE, self.HOST + href) if "-" in href: _id = href.split("-")[-1] @@ -325,11 +334,11 @@ class Musify(Page): raw_id: str = href.split("-")[-1] if raw_id.isdigit(): _id = raw_id - source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href)) + source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) else: - cls.LOGGER.debug("there are not enough anchors (2) for artist and track") - cls.LOGGER.debug(str(artist_list)) + self.LOGGER.debug("there are not enough anchors (2) for artist and track") + self.LOGGER.debug(str(artist_list)) """ artist_name = playlist_item_soup.get("data-artist") @@ -349,390 +358,38 @@ class Musify(Page): source_list=source_list ) - @classmethod - def parse_playlist_soup(cls, playlist_soup: BeautifulSoup) -> List[Song]: + def _parse_playlist_soup(self, playlist_soup: BeautifulSoup) -> List[Song]: song_list = [] for playlist_item_soup in playlist_soup.find_all("div", {"class": "playlist__item"}): - song_list.append(cls.parse_playlist_item(playlist_item_soup)) + song_list.append(self._parse_playlist_item(playlist_item_soup)) return song_list - - @classmethod - def plaintext_search(cls, query: str) -> Options: + + def general_search(self, search_query: str) -> List[DatabaseObject]: search_results = [] - r = cls.CONNECTION.get(f"https://musify.club/search?searchText={query}") + r = self.connection.get(f"https://musify.club/search?searchText={search_query}") if r is None: - return Options() - search_soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser") + return [] + search_soup: BeautifulSoup = self.get_soup_from_response(r) # album and songs # child of div class: contacts row for contact_container_soup in search_soup.find_all("div", {"class": "contacts"}): - search_results.extend(cls.parse_contact_container(contact_container_soup)) + search_results.extend(self._parse_contact_container(contact_container_soup)) # song # div class: playlist__item for playlist_soup in search_soup.find_all("div", {"class": "playlist"}): - search_results.extend(cls.parse_playlist_soup(playlist_soup)) + search_results.extend(self._parse_playlist_soup(playlist_soup)) - return Options(search_results) + return search_results + + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: + return Song() - @classmethod - def parse_album_card(cls, album_card: BeautifulSoup, artist_name: str = None) -> Album: - """ -
- - Self Loather - - -
-

- Self Loather -

-
-

2021

-
-

- Depressive BlackPost-Black Metal

-
-
- 13.11.2021 - 5,88 -
-
- """ - - album_type_map = defaultdict(lambda: AlbumType.OTHER, { - 1: AlbumType.OTHER, # literally other xD - 2: AlbumType.STUDIO_ALBUM, - 3: AlbumType.EP, - 4: AlbumType.SINGLE, - 5: AlbumType.OTHER, # BOOTLEG - 6: AlbumType.LIVE_ALBUM, - 7: AlbumType.COMPILATION_ALBUM, # compilation of different artists - 8: AlbumType.MIXTAPE, - 9: AlbumType.DEMO, - 10: AlbumType.MIXTAPE, # DJ Mixes - 11: AlbumType.COMPILATION_ALBUM, # compilation of only this artist - 12: AlbumType.STUDIO_ALBUM, # split - 13: AlbumType.COMPILATION_ALBUM, # unofficial - 14: AlbumType.MIXTAPE # "Soundtracks" - }) - - _id: Optional[str] = None - name: str = None - source_list: List[Source] = [] - timestamp: Optional[ID3Timestamp] = None - album_status = None - - def set_name(new_name: str): - nonlocal name - nonlocal artist_name - - # example of just setting not working: - # https://musify.club/release/unjoy-eurythmie-psychonaut-4-tired-numb-still-alive-2012-324067 - if new_name.count(" - ") != 1: - name = new_name - return - - potential_artist_list, potential_name = new_name.split(" - ") - unified_artist_list = string_processing.unify(potential_artist_list) - if artist_name is not None: - if string_processing.unify(artist_name) not in unified_artist_list: - name = new_name - return - - name = potential_name - return - - name = new_name - - album_status_id = album_card.get("data-type") - if album_status_id.isdigit(): - album_status_id = int(album_status_id) - album_type = album_type_map[album_status_id] - - if album_status_id == 5: - album_status = AlbumStatus.BOOTLEG - - def parse_release_anchor(_anchor: BeautifulSoup, text_is_name=False): - nonlocal _id - nonlocal name - nonlocal source_list - - if _anchor is None: - return - - href = _anchor.get("href") - if href is not None: - # add url to sources - source_list.append(Source( - cls.SOURCE_TYPE, - cls.HOST + href - )) - - # split id from url - split_href = href.split("-") - if len(split_href) > 1: - _id = split_href[-1] - - if not text_is_name: - return - - set_name(_anchor.text) - - anchor_list = album_card.find_all("a", recursive=False) - if len(anchor_list) > 0: - anchor = anchor_list[0] - parse_release_anchor(anchor) - - thumbnail: BeautifulSoup = anchor.find("img") - if thumbnail is not None: - alt = thumbnail.get("alt") - if alt is not None: - set_name(alt) - - image_url = thumbnail.get("src") - else: - cls.LOGGER.debug("the card has no thumbnail or url") - - card_body = album_card.find("div", {"class": "card-body"}) - if card_body is not None: - parse_release_anchor(card_body.find("a"), text_is_name=True) - - def parse_small_date(small_soup: BeautifulSoup): - """ - - - 13.11.2021 - - """ - nonlocal timestamp - - italic_tagging_soup: BeautifulSoup = small_soup.find("i") - if italic_tagging_soup is None: - return - if italic_tagging_soup.get("title") != "Добавлено": - # "Добавлено" can be translated to "Added (at)" - return - - raw_time = small_soup.text.strip() - timestamp = ID3Timestamp.strptime(raw_time, "%d.%m.%Y") - - # parse small date - card_footer_list = album_card.find_all("div", {"class": "card-footer"}) - if len(card_footer_list) != 3: - cls.LOGGER.debug("there are not exactly 3 card footers in a card") - - if len(card_footer_list) > 0: - for any_small_soup in card_footer_list[-1].find_all("small"): - parse_small_date(any_small_soup) - else: - cls.LOGGER.debug("there is not even 1 footer in the album card") - - return Album( - _id=_id, - title=name, - source_list=source_list, - date=timestamp, - album_type=album_type, - album_status=album_status - ) - - @classmethod - def get_discography(cls, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> List[Album]: - """ - POST https://musify.club/artist/filteralbums - ArtistID: 280348 - SortOrder.Property: dateCreated - SortOrder.IsAscending: false - X-Requested-With: XMLHttpRequest - """ - - endpoint = cls.HOST + "/" + url.source_type.value + "/filteralbums" - - r = cls.post_request(url=endpoint, json={ - "ArtistID": str(url.musify_id), - "SortOrder.Property": "dateCreated", - "SortOrder.IsAscending": False, - "X-Requested-With": "XMLHttpRequest" - }) - if r is None: - return [] - soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser") - - discography: List[Album] = [] - for card_soup in soup.find_all("div", {"class": "card"}): - new_album: Album = cls.parse_album_card(card_soup, artist_name) - album_source: Source - - if stop_at_level > 1: - for album_source in new_album.source_collection.get_sources_from_page(cls.SOURCE_TYPE): - new_album.merge(cls._fetch_album_from_source(album_source, stop_at_level=stop_at_level-1)) - - discography.append(new_album) - - return discography - - @classmethod - def get_artist_attributes(cls, url: MusifyUrl) -> Artist: - """ - fetches the main Artist attributes from this endpoint - https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent - it needs to parse html - - :param url: - :return: - """ - - r = cls.CONNECTION.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent") - if r is None: - return Artist(_id=url.musify_id) - - soup = BeautifulSoup(r.content, "html.parser") - - """ -
    -
  1. Главная
    2. -
  2. Исполнители
    3. -
  3. Ghost Bath
    4. -
- - - -
-

Ghost Bath

-
- ... -
-
- -
    -
  • - - - Соединенные Штаты -
    • -
- """ - name = None - source_list: List[Source] = [] - country = None - notes: FormattedText = FormattedText() - - breadcrumbs: BeautifulSoup = soup.find("ol", {"class": "breadcrumb"}) - if breadcrumbs is not None: - breadcrumb_list: List[BeautifulSoup] = breadcrumbs.find_all("li", {"class": "breadcrumb-item"}, recursive=False) - if len(breadcrumb_list) == 3: - name = breadcrumb_list[-1].get_text(strip=True) - else: - cls.LOGGER.debug("breadcrumb layout on artist page changed") - - nav_tabs: BeautifulSoup = soup.find("ul", {"class": "nav-tabs"}) - if nav_tabs is not None: - list_item: BeautifulSoup - for list_item in nav_tabs.find_all("li", {"class": "nav-item"}, recursive=False): - if not list_item.get_text(strip=True).startswith("песни"): - # "песни" translates to "songs" - continue - - anchor: BeautifulSoup = list_item.find("a") - if anchor is None: - continue - href = anchor.get("href") - if href is None: - continue - - source_list.append(Source( - cls.SOURCE_TYPE, - cls.HOST + href - )) - - content_title: BeautifulSoup = soup.find("header", {"class": "content__title"}) - if content_title is not None: - h1_name: BeautifulSoup = content_title.find("h1", recursive=False) - if h1_name is not None: - name = h1_name.get_text(strip=True) - - # country and sources - icon_list: BeautifulSoup = soup.find("ul", {"class": "icon-list"}) - if icon_list is not None: - country_italic: BeautifulSoup = icon_list.find("i", {"class", "flag-icon"}) - if country_italic is not None: - style_classes: set = {'flag-icon', 'shadow'} - classes: set = set(country_italic.get("class")) - - country_set: set = classes.difference(style_classes) - if len(country_set) != 1: - cls.LOGGER.debug("the country set contains multiple values") - if len(country_set) != 0: - """ - This is the css file, where all flags that can be used on musify - are laid out and styled. - Every flag has two upper case letters, thus I assume they follow the alpha_2 - https://musify.club/content/flags.min.css - """ - - country = pycountry.countries.get(alpha_2=list(country_set)[0]) - - # get all additional sources - additional_source: BeautifulSoup - for additional_source in icon_list.find_all("a", {"class", "link"}): - href = additional_source.get("href") - if href is None: - continue - new_src = Source.match_url(href, referer_page=cls.SOURCE_TYPE) - if new_src is None: - continue - source_list.append(new_src) - - note_soup: BeautifulSoup = soup.find(id="text-main") - if note_soup is not None: - notes.html = note_soup.decode_contents() - - return Artist( - _id=url.musify_id, - name=name, - country=country, - source_list=source_list, - notes=notes - ) - - @classmethod - def _fetch_artist_from_source(cls, source: Source, stop_at_level: int = 1) -> Artist: - """ - fetches artist from source - - [x] discography - [x] attributes - [] picture gallery - - Args: - source (Source): the source to fetch - stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False. - - Returns: - Artist: the artist fetched - """ - - url = cls.parse_url(source.url) - - artist = cls.get_artist_attributes(url) - - discography: List[Album] = cls.get_discography(url, artist.name) - artist.main_album_collection.extend(discography) - - return artist - - @classmethod - def parse_song_card(cls, song_card: BeautifulSoup) -> Song: + def _parse_song_card(self, song_card: BeautifulSoup) -> Song: """
@@ -821,8 +478,8 @@ class Musify(Page): track_anchor: BeautifulSoup = anchor_list[-1] href: str = track_anchor.get("href") if href is not None: - current_url = cls.HOST + href - source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href)) + current_url = self.HOST + href + source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) song_name = parse_title(track_anchor.get_text(strip=True)) # artist @@ -834,7 +491,7 @@ class Musify(Page): if meta_artist_src is not None: meta_artist_url = meta_artist_src.get("content") if meta_artist_url is not None: - _artist_src = [Source(cls.SOURCE_TYPE, cls.HOST + meta_artist_url)] + _artist_src = [Source(self.SOURCE_TYPE, self.HOST + meta_artist_url)] meta_artist_name = artist_span.find("meta", {"itemprop": "name"}) if meta_artist_name is not None: @@ -861,9 +518,9 @@ class Musify(Page): download_href = download_anchor.get("href") if download_href is not None and current_url is not None: source_list.append(Source( - cls.SOURCE_TYPE, + self.SOURCE_TYPE, url=current_url, - adio_url=cls.HOST + download_href + adio_url=self.HOST + download_href )) return Song( @@ -873,8 +530,323 @@ class Musify(Page): source_list=source_list ) - @classmethod - def _parse_album(cls, soup: BeautifulSoup) -> Album: + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + """ + fetches album from source: + eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188' + + /html/musify/album_overview.html + - [x] tracklist + - [x] attributes + - [ ] ratings + + :param stop_at_level: + :param source: + :return: + """ + + url = parse_url(source.url) + + endpoint = self.HOST + "/release/" + url.name_with_id + r = self.connection.get(endpoint) + if r is None: + return Album() + + soup = BeautifulSoup(r.content, "html.parser") + + album = self._parse_album(soup) + + #
...
+ cards_soup: BeautifulSoup = soup.find("div", {"class": "card-body"}) + if cards_soup is not None: + card_soup: BeautifulSoup + for card_soup in cards_soup.find_all("div", {"class": "playlist__item"}): + new_song = self._parse_song_card(card_soup) + album.song_collection.append(new_song) + + if stop_at_level > 1: + song: Song + for song in album.song_collection: + sources = song.source_collection.get_sources_from_page(cls.SOURCE_TYPE) + for source in sources: + song.merge(self.fetch_song(source=source)) + + album.update_tracksort() + + return album + + def _get_artist_attributes(self, url: MusifyUrl) -> Artist: + """ + fetches the main Artist attributes from this endpoint + https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent + it needs to parse html + + :param url: + :return: + """ + + r = self.connection.get(f"https://musify.club/{url.source_type.value}/{url.name_with_id}?_pjax=#bodyContent") + if r is None: + return Artist() + + soup = self.get_soup_from_response(r) + + """ +
    +
  1. Главная
    2. +
  2. Исполнители
    3. +
  3. Ghost Bath
    4. +
+ + + +
+

Ghost Bath

+
+ ... +
+
+ +
    +
  • + + + Соединенные Штаты +
    • +
+ """ + name = None + source_list: List[Source] = [] + country = None + notes: FormattedText = FormattedText() + + breadcrumbs: BeautifulSoup = soup.find("ol", {"class": "breadcrumb"}) + if breadcrumbs is not None: + breadcrumb_list: List[BeautifulSoup] = breadcrumbs.find_all("li", {"class": "breadcrumb-item"}, recursive=False) + if len(breadcrumb_list) == 3: + name = breadcrumb_list[-1].get_text(strip=True) + else: + self.LOGGER.debug("breadcrumb layout on artist page changed") + + nav_tabs: BeautifulSoup = soup.find("ul", {"class": "nav-tabs"}) + if nav_tabs is not None: + list_item: BeautifulSoup + for list_item in nav_tabs.find_all("li", {"class": "nav-item"}, recursive=False): + if not list_item.get_text(strip=True).startswith("песни"): + # "песни" translates to "songs" + continue + + anchor: BeautifulSoup = list_item.find("a") + if anchor is None: + continue + href = anchor.get("href") + if href is None: + continue + + source_list.append(Source( + self.SOURCE_TYPE, + self.HOST + href + )) + + content_title: BeautifulSoup = soup.find("header", {"class": "content__title"}) + if content_title is not None: + h1_name: BeautifulSoup = content_title.find("h1", recursive=False) + if h1_name is not None: + name = h1_name.get_text(strip=True) + + # country and sources + icon_list: BeautifulSoup = soup.find("ul", {"class": "icon-list"}) + if icon_list is not None: + country_italic: BeautifulSoup = icon_list.find("i", {"class", "flag-icon"}) + if country_italic is not None: + style_classes: set = {'flag-icon', 'shadow'} + classes: set = set(country_italic.get("class")) + + country_set: set = classes.difference(style_classes) + if len(country_set) != 1: + self.LOGGER.debug("the country set contains multiple values") + if len(country_set) != 0: + """ + This is the css file, where all flags that can be used on musify + are laid out and styled. + Every flag has two upper case letters, thus I assume they follow the alpha_2 + https://musify.club/content/flags.min.css + """ + + country = pycountry.countries.get(alpha_2=list(country_set)[0]) + + # get all additional sources + additional_source: BeautifulSoup + for additional_source in icon_list.find_all("a", {"class", "link"}): + href = additional_source.get("href") + if href is None: + continue + new_src = Source.match_url(href, referer_page=self.SOURCE_TYPE) + if new_src is None: + continue + source_list.append(new_src) + + note_soup: BeautifulSoup = soup.find(id="text-main") + if note_soup is not None: + notes.html = note_soup.decode_contents() + + return Artist( + name=name, + country=country, + source_list=source_list, + notes=notes + ) + + def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None) -> Album: + """ +
+ + Self Loather + + +
+

+ Self Loather +

+
+

2021

+
+

+ Depressive BlackPost-Black Metal

+
+
+ 13.11.2021 + 5,88 +
+
+ """ + + _id: Optional[str] = None + name: str = None + source_list: List[Source] = [] + timestamp: Optional[ID3Timestamp] = None + album_status = None + + def set_name(new_name: str): + nonlocal name + nonlocal artist_name + + # example of just setting not working: + # https://musify.club/release/unjoy-eurythmie-psychonaut-4-tired-numb-still-alive-2012-324067 + if new_name.count(" - ") != 1: + name = new_name + return + + potential_artist_list, potential_name = new_name.split(" - ") + unified_artist_list = string_processing.unify(potential_artist_list) + if artist_name is not None: + if string_processing.unify(artist_name) not in unified_artist_list: + name = new_name + return + + name = potential_name + return + + name = new_name + + album_status_id = album_card.get("data-type") + if album_status_id.isdigit(): + album_status_id = int(album_status_id) + album_type = ALBUM_TYPE_MAP[album_status_id] + + if album_status_id == 5: + album_status = AlbumStatus.BOOTLEG + + def parse_release_anchor(_anchor: BeautifulSoup, text_is_name=False): + nonlocal _id + nonlocal name + nonlocal source_list + + if _anchor is None: + return + + href = _anchor.get("href") + if href is not None: + # add url to sources + source_list.append(Source( + self.SOURCE_TYPE, + self.HOST + href + )) + + # split id from url + split_href = href.split("-") + if len(split_href) > 1: + _id = split_href[-1] + + if not text_is_name: + return + + set_name(_anchor.text) + + anchor_list = album_card.find_all("a", recursive=False) + if len(anchor_list) > 0: + anchor = anchor_list[0] + parse_release_anchor(anchor) + + thumbnail: BeautifulSoup = anchor.find("img") + if thumbnail is not None: + alt = thumbnail.get("alt") + if alt is not None: + set_name(alt) + + image_url = thumbnail.get("src") + else: + self.LOGGER.debug("the card has no thumbnail or url") + + card_body = album_card.find("div", {"class": "card-body"}) + if card_body is not None: + parse_release_anchor(card_body.find("a"), text_is_name=True) + + def parse_small_date(small_soup: BeautifulSoup): + """ + + + 13.11.2021 + + """ + nonlocal timestamp + + italic_tagging_soup: BeautifulSoup = small_soup.find("i") + if italic_tagging_soup is None: + return + if italic_tagging_soup.get("title") != "Добавлено": + # "Добавлено" can be translated to "Added (at)" + return + + raw_time = small_soup.text.strip() + timestamp = ID3Timestamp.strptime(raw_time, "%d.%m.%Y") + + # parse small date + card_footer_list = album_card.find_all("div", {"class": "card-footer"}) + if len(card_footer_list) != 3: + self.LOGGER.debug("there are not exactly 3 card footers in a card") + + if len(card_footer_list) > 0: + for any_small_soup in card_footer_list[-1].find_all("small"): + parse_small_date(any_small_soup) + else: + self.LOGGER.debug("there is not even 1 footer in the album card") + + return Album( + _id=_id, + title=name, + source_list=source_list, + date=timestamp, + album_type=album_type, + album_status=album_status + ) + + def _parse_album(self, soup: BeautifulSoup) -> Album: name: str = None source_list: List[Source] = [] artist_list: List[Artist] = [] @@ -901,7 +873,7 @@ class Musify(Page): artist_source_list: List[Source] = [] if href is not None: - artist_source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + href.strip())) + artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href.strip())) span: BeautifulSoup = anchor.find("span") if span is not None: @@ -910,14 +882,14 @@ class Musify(Page): source_list=artist_source_list )) else: - cls.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case") + self.LOGGER.debug("there are not 4 breadcrumb items, which shouldn't be the case") # meta meta_url: BeautifulSoup = soup.find("meta", {"itemprop": "url"}) if meta_url is not None: url = meta_url.get("content") if url is not None: - source_list.append(Source(cls.SOURCE_TYPE, cls.HOST + url)) + source_list.append(Source(self.SOURCE_TYPE, self.HOST + url)) meta_name: BeautifulSoup = soup.find("meta", {"itemprop": "name"}) if meta_name is not None: @@ -937,7 +909,7 @@ class Musify(Page): if artist_url_meta is not None: artist_href = artist_url_meta.get("content") if artist_href is not None: - artist_source_list.append(Source(cls.SOURCE_TYPE, url=cls.HOST + artist_href)) + artist_source_list.append(Source(self.SOURCE_TYPE, url=self.HOST + artist_href)) artist_meta_name = artist_anchor.find("meta", {"itemprop": "name"}) if artist_meta_name is not None: @@ -955,7 +927,7 @@ class Musify(Page): try: date = ID3Timestamp.strptime(raw_datetime, "%Y-%m-%d") except ValueError: - cls.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}") + self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}") return Album( title=name, @@ -964,51 +936,107 @@ class Musify(Page): date=date ) - @classmethod - def _fetch_album_from_source(cls, source: Source, stop_at_level: int = 1) -> Album: + def _get_discography(self, url: MusifyUrl, artist_name: str = None, stop_at_level: int = 1) -> List[Album]: """ - fetches album from source: - eg. 'https://musify.club/release/linkin-park-hybrid-theory-2000-188' - - /html/musify/album_overview.html - - [x] tracklist - - [x] attributes - - [ ] ratings - - :param stop_at_level: - :param source: - :return: + POST https://musify.club/artist/filteralbums + ArtistID: 280348 + SortOrder.Property: dateCreated + SortOrder.IsAscending: false + X-Requested-With: XMLHttpRequest """ - url = cls.parse_url(source.url) + endpoint = self.HOST + "/" + url.source_type.value + "/filteralbums" - endpoint = cls.HOST + "/release/" + url.name_with_id - r = cls.CONNECTION.get(endpoint) + r = self.connection.post(url=endpoint, json={ + "ArtistID": str(url.musify_id), + "SortOrder.Property": "dateCreated", + "SortOrder.IsAscending": False, + "X-Requested-With": "XMLHttpRequest" + }) if r is None: - return Album() + return [] + soup: BeautifulSoup = BeautifulSoup(r.content, features="html.parser") - soup = BeautifulSoup(r.content, "html.parser") + discography: List[Album] = [] + for card_soup in soup.find_all("div", {"class": "card"}): + new_album: Album = self._parse_album_card(card_soup, artist_name) + album_source: Source + + if stop_at_level > 1: + for album_source in new_album.source_collection.get_sources_from_page(self.SOURCE_TYPE): + new_album.merge(self.fetch_album(album_source, stop_at_level=stop_at_level-1)) + + discography.append(new_album) - album = cls._parse_album(soup) + return discography - #
...
- cards_soup: BeautifulSoup = soup.find("div", {"class": "card-body"}) - if cards_soup is not None: - card_soup: BeautifulSoup - for card_soup in cards_soup.find_all("div", {"class": "playlist__item"}): - new_song = cls.parse_song_card(card_soup) - album.song_collection.append(new_song) + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + """ + fetches artist from source + + [x] discography + [x] attributes + [] picture gallery + + Args: + source (Source): the source to fetch + stop_at_level: int = 1: if it is false, every album from discograohy will be fetched. Defaults to False. + + Returns: + Artist: the artist fetched + """ + + url = parse_url(source.url) + + artist = self._get_artist_attributes(url) + + discography: List[Album] = self._get_discography(url, artist.name) + artist.main_album_collection.extend(discography) - if stop_at_level > 1: - song: Song - for song in album.song_collection: - sources = song.source_collection.get_sources_from_page(cls.SOURCE_TYPE) - for source in sources: - song.merge(cls._fetch_song_from_source(source=source)) - - album.update_tracksort() + return artist - return album + def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: + return Label() + + +class Musify(Page): + API_SESSION: requests.Session = requests.Session() + API_SESSION.headers = { + "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", + "Connection": "keep-alive", + "Referer": "https://musify.club/" + } + API_SESSION.proxies = shared.proxies + TIMEOUT = 7 + POST_TIMEOUT = 15 + TRIES = 5 + HOST = "https://musify.club" + + CONNECTION = Connection( + host="https://musify.club/", + logger=MUSIFY_LOGGER + ) + + SOURCE_TYPE = SourcePages.MUSIFY + + LOGGER = shared.MUSIFY_LOGGER + + + + + + @classmethod + def get_plaintext_query(cls, query: Query) -> str: + if query.album is None: + return f"{query.artist or '*'} - {query.song or '*'}" + return f"{query.artist or '*'} - {query.album or '*'} - {query.song or '*'}" + + + + + + + @classmethod def _get_type_of_url(cls, url: str) -> Optional[Union[Type[Song], Type[Album], Type[Artist], Type[Label]]]: From 8af48fb4d4db62a030d1b9fcdd7511bea29ae43e Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Wed, 24 May 2023 18:02:19 +0200 Subject: [PATCH 31/81] fixed borderline stupid bug --- src/actual_donwload.py | 9 +++++---- src/music_kraken/download/page_attributes.py | 1 + src/music_kraken/pages/musify.py | 6 +++--- src/music_kraken/pages/preset.py | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index b6fb29b..9db40b9 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -1,11 +1,12 @@ import music_kraken from music_kraken import pages +from music_kraken.download import Search from music_kraken.utils.enums.source import SourcePages from music_kraken.objects import Song, Target, Source, Album def search_pages(): - search = pages.Search() + search = Search() print("metadata", search.pages) print("audio", search.audio_pages) @@ -22,7 +23,7 @@ def search_pages(): def direct_download(): - search = pages.Search() + search = Search() search.search_url("https://www.metal-archives.com/bands/Ghost_Bath/3540372489") print(search) @@ -46,7 +47,7 @@ def download_audio(): def real_download(): - search = pages.Search() + search = Search() search.search_url("https://musify.club/release/children-of-the-night-2018-1079829") search.download_chosen() @@ -56,7 +57,7 @@ if __name__ == "__main__": # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", # "https://musify.club/release/ghost-bath-self-loather-2021-1554266", "#a Ghost Bath", - "0", + "1", "4", "ok" ]) diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index e51afec..861e7e6 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -10,6 +10,7 @@ SOURCE_PAGE_MAP: Dict[SourcePages, Page] = dict() ALL_PAGES: Tuple[Page, ...] = ( EncyclopaediaMetallum(), + Musify() ) AUDIO_PAGES: Tuple[Page, ...] = ( diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 7123bb6..4b6b390 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -108,7 +108,7 @@ def parse_url(url: str) -> MusifyUrl: ) -class Preset(Page): +class Musify(Page): # CHANGE SOURCE_TYPE = SourcePages.MUSIFY LOGGER = MUSIFY_LOGGER @@ -129,7 +129,7 @@ class Preset(Page): musify_url = parse_url(source.url) - # Has no labels, because afaiak musify has no Labels + # Has no labels, because afaik musify has no Labels musify_type_to_database_type = { MusifyTypes.SONG: Song, MusifyTypes.RELEASE: Album, @@ -999,7 +999,7 @@ class Preset(Page): return Label() -class Musify(Page): +class OldMusify(Page): API_SESSION: requests.Session = requests.Session() API_SESSION.headers = { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", diff --git a/src/music_kraken/pages/preset.py b/src/music_kraken/pages/preset.py index b2af7fd..e688074 100644 --- a/src/music_kraken/pages/preset.py +++ b/src/music_kraken/pages/preset.py @@ -3,7 +3,7 @@ from urllib.parse import urlparse import logging -from music_kraken.objects import Source, DatabaseObject +from ..objects import Source, DatabaseObject from .abstract import Page from ..objects import ( Artist, From a1650d560c1e84bdebf8de1cc2a7d75bdad63f7d Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Thu, 25 May 2023 01:27:05 +0200 Subject: [PATCH 32/81] much progress --- src/music_kraken/objects/collection.py | 2 +- src/music_kraken/objects/parents.py | 4 ++++ src/music_kraken/objects/song.py | 6 +++++ src/music_kraken/pages/abstract.py | 32 ++++++++++++++++++++++++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/music_kraken/objects/collection.py b/src/music_kraken/objects/collection.py index 3a35e0a..2daf0fa 100644 --- a/src/music_kraken/objects/collection.py +++ b/src/music_kraken/objects/collection.py @@ -96,7 +96,7 @@ class Collection: return AppendResult(True, existing_object, False) # if the object does already exist - # thus merging and don't add it afterwards + # thus merging and don't add it afterward if merge_into_existing: existing_object.merge(element) # in case any relevant data has been added (e.g. it remaps the old object) diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index f692dea..a6b50fb 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -11,6 +11,10 @@ class DatabaseObject: COLLECTION_ATTRIBUTES: tuple = tuple() SIMPLE_ATTRIBUTES: dict = dict() + # contains all collection attributes, which describe something "smaller" + # e.g. album has songs, but not artist. + DOWNWARDS_COLLECTION_ATTRIBUTES: tuple = tuple() + def __init__(self, _id: int = None, dynamic: bool = False, **kwargs) -> None: self.automatic_id: bool = False diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index 3716b03..e7b0194 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -204,6 +204,8 @@ class Album(MainObject): "notes": FormattedText() } + DOWNWARDS_COLLECTION_ATTRIBUTES = ("song_collection",) + def __init__( self, _id: int = None, @@ -422,6 +424,8 @@ class Artist(MainObject): "general_genre": "" } + DOWNWARDS_COLLECTION_ATTRIBUTES = ("feature_song_collection", "main_album_collection") + def __init__( self, _id: int = None, @@ -646,6 +650,8 @@ class Label(MainObject): "notes": FormattedText() } + DOWNWARDS_COLLECTION_ATTRIBUTES = COLLECTION_ATTRIBUTES + def __init__( self, _id: int = None, diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index d5cd8fa..a53526b 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -237,6 +237,38 @@ class Page(threading.Thread): def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() + def download(self, music_object: DatabaseObject, download_all: bool = False) -> DownloadResult: + self._download(music_object, {}, download_all) + + return DownloadResult() + + + def _download(self, music_object: DatabaseObject, naming_objects: Dict[Type[DatabaseObject], DatabaseObject], download_all: bool = False) -> list: + # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False + if isinstance(music_object, Album): + if not download_all and music_object.album_type in shared.ALBUM_TYPE_BLACKLIST: + return [] + + naming_objects[type(music_object)] = music_object + + if isinstance(music_object, Song): + print("Downloading", music_object) + return [music_object] + + return_values: List = [] + + for collection_name in music_object.DOWNWARDS_COLLECTION_ATTRIBUTES: + collection: Collection = getattr(self, collection_name) + + sub_ordered_music_object: DatabaseObject + for sub_ordered_music_object in collection: + return_values.extend(self._download(sub_ordered_music_object, naming_objects.copy(), download_all)) + + return return_values + + def download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], DatabaseObject]): + pass + @classmethod def download( cls, From 82a95f9b49a06ba0016621e2f72a310cc2667a9b Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 25 May 2023 09:21:37 +0200 Subject: [PATCH 33/81] fuuuuuck --- src/music_kraken/objects/parents.py | 1 + src/music_kraken/objects/song.py | 6 +++- src/music_kraken/pages/abstract.py | 52 ++++++++++++++++++++++++++--- 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index a6b50fb..d68463f 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -14,6 +14,7 @@ class DatabaseObject: # contains all collection attributes, which describe something "smaller" # e.g. album has songs, but not artist. DOWNWARDS_COLLECTION_ATTRIBUTES: tuple = tuple() + UPWARDS_COLLECTION_ATTRIBUTES: tuple = tuple() def __init__(self, _id: int = None, dynamic: bool = False, **kwargs) -> None: self.automatic_id: bool = False diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index e7b0194..b253539 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -46,6 +46,8 @@ class Song(MainObject): "genre": None, "notes": FormattedText() } + + UPWARDS_COLLECTION_ATTRIBUTES = ("album_collection", "main_artist_collection", "feature_artist_collection") def __init__( self, @@ -204,7 +206,8 @@ class Album(MainObject): "notes": FormattedText() } - DOWNWARDS_COLLECTION_ATTRIBUTES = ("song_collection",) + DOWNWARDS_COLLECTION_ATTRIBUTES = ("song_collection", ) + UPWARDS_COLLECTION_ATTRIBUTES = ("artist_collection", "label_collection") def __init__( self, @@ -425,6 +428,7 @@ class Artist(MainObject): } DOWNWARDS_COLLECTION_ATTRIBUTES = ("feature_song_collection", "main_album_collection") + UPWARDS_COLLECTION_ATTRIBUTES = ("label_collection", ) def __init__( self, diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index a53526b..80be169 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -23,6 +23,7 @@ from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType from ..audio import write_metadata_to_target, correct_codec from ..utils import shared +from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE from ..utils.support_classes import Query, DownloadResult, DefaultTarget @@ -237,8 +238,26 @@ class Page(threading.Thread): def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() - def download(self, music_object: DatabaseObject, download_all: bool = False) -> DownloadResult: - self._download(music_object, {}, download_all) + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + naming_objects = {"genre": genre} + + def fill_naming_objects(naming_music_object: DatabaseObject): + nonlocal naming_objects + + for collection_name in naming_music_object.UPWARDS_COLLECTION_ATTRIBUTES: + collection: Collection = getattr(self, collection_name) + + if collection.empty(): + continue + if collection.element_type in naming_objects: + continue + + dom_ordered_music_object: DatabaseObject = collection[0] + return fill_naming_objects(dom_ordered_music_object) + + fill_naming_objects(music_object) + + self._download(music_object, {}, genre, download_all) return DownloadResult() @@ -252,8 +271,7 @@ class Page(threading.Thread): naming_objects[type(music_object)] = music_object if isinstance(music_object, Song): - print("Downloading", music_object) - return [music_object] + return [self._download_song(music_object, naming_objects)] return_values: List = [] @@ -266,7 +284,31 @@ class Page(threading.Thread): return return_values - def download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], DatabaseObject]): + def _download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], DatabaseObject]): + name_attribute = DEFAULT_VALUES.copy() + + # song + name_attribute["genre"] = naming_objects["genre"] + name_attribute["song"] = song.title + + if Album in naming_objects: + album: Album = naming_objects[Album] + name_attribute["album"] = album.title + name_attribute["album_type"] = album.album_type.value + + if Artist in naming_objects: + artist: Artist = naming_objects[Artist] + naming_objects["artist"] = artist.name + + if Label in naming_objects: + label: Label = naming_objects[Label] + naming_objects["label"] = label.name + + new_target = Target( + relative_to_music_dir=True, + path=DOWNLOAD_PATH.format(**name_attribute), + file=DOWNLOAD_FILE.format(**name_attribute) + ) pass @classmethod From 1362f708e6807cad7a62a3676edddf1baf73c81d Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 25 May 2023 11:21:39 +0200 Subject: [PATCH 34/81] continued refactoring of downloads --- src/music_kraken/objects/target.py | 3 + src/music_kraken/pages/abstract.py | 261 +++-------------------------- src/music_kraken/pages/musify.py | 63 +------ 3 files changed, 29 insertions(+), 298 deletions(-) diff --git a/src/music_kraken/objects/target.py b/src/music_kraken/objects/target.py index 7fae49b..8c70755 100644 --- a/src/music_kraken/objects/target.py +++ b/src/music_kraken/objects/target.py @@ -98,3 +98,6 @@ class Target(DatabaseObject): except requests.exceptions.Timeout: shared.DOWNLOAD_LOGGER.error("Stream timed out.") return False + + def delete(self): + self.file_path.unlink(missing_ok=True) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 80be169..31b7dd2 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -257,32 +257,31 @@ class Page(threading.Thread): fill_naming_objects(music_object) - self._download(music_object, {}, genre, download_all) - - return DownloadResult() + return self._download(music_object, {}, genre, download_all) def _download(self, music_object: DatabaseObject, naming_objects: Dict[Type[DatabaseObject], DatabaseObject], download_all: bool = False) -> list: # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False if isinstance(music_object, Album): if not download_all and music_object.album_type in shared.ALBUM_TYPE_BLACKLIST: - return [] + return DownloadResult() + self.fetch_details(music_object=music_object, stop_at_level=2) naming_objects[type(music_object)] = music_object if isinstance(music_object, Song): - return [self._download_song(music_object, naming_objects)] + return self._download_song(music_object, naming_objects) - return_values: List = [] + download_result: DownloadResult = DownloadResult() for collection_name in music_object.DOWNWARDS_COLLECTION_ATTRIBUTES: collection: Collection = getattr(self, collection_name) sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: - return_values.extend(self._download(sub_ordered_music_object, naming_objects.copy(), download_all)) + download_result.merge(self._download(sub_ordered_music_object, naming_objects.copy(), download_all)) - return return_values + return download_result def _download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], DatabaseObject]): name_attribute = DEFAULT_VALUES.copy() @@ -309,246 +308,27 @@ class Page(threading.Thread): path=DOWNLOAD_PATH.format(**name_attribute), file=DOWNLOAD_FILE.format(**name_attribute) ) - pass - - @classmethod - def download( - cls, - music_object: Union[Song, Album, Artist, Label], - download_features: bool = True, - default_target: DefaultTarget = None, - genre: str = None, - override_existing: bool = False, - create_target_on_demand: bool = True, - download_all: bool = False, - exclude_album_type: Set[AlbumType] = shared.ALBUM_TYPE_BLACKLIST - ) -> DownloadResult: - """ - - :param genre: The downloader will download to THIS folder (set the value of default_target.genre to genre) - :param music_object: - :param download_features: - :param default_target: - :param override_existing: - :param create_target_on_demand: - :param download_all: - :param exclude_album_type: - :return total downloads, failed_downloads: - """ - if default_target is None: - default_target = DefaultTarget() - - if download_all: - exclude_album_type: Set[AlbumType] = set() - elif exclude_album_type is None: - exclude_album_type = { - AlbumType.COMPILATION_ALBUM, - AlbumType.LIVE_ALBUM, - AlbumType.MIXTAPE - } - - if type(music_object) is Song: - return cls.download_song( - music_object, - override_existing=override_existing, - create_target_on_demand=create_target_on_demand, - genre=genre - ) - if type(music_object) is Album: - return cls.download_album( - music_object, - default_target=default_target, - override_existing=override_existing, - genre=genre - ) - if type(music_object) is Artist: - return cls.download_artist( - music_object, - default_target=default_target, - download_features=download_features, - exclude_album_type=exclude_album_type, - genre=genre - ) - if type(music_object) is Label: - return cls.download_label( - music_object, - download_features=download_features, - default_target=default_target, - exclude_album_type=exclude_album_type, - genre=genre - ) - - return DownloadResult(error_message=f"{type(music_object)} can't be downloaded.") - - @classmethod - def download_label( - cls, - label: Label, - exclude_album_type: Set[AlbumType], - download_features: bool = True, - override_existing: bool = False, - default_target: DefaultTarget = None, - genre: str = None - ) -> DownloadResult: - - default_target = DefaultTarget() if default_target is None else copy(default_target) - default_target.label_object(label) - - r = DownloadResult() - - cls.fetch_details(label) - for artist in label.current_artist_collection: - r.merge(cls.download_artist( - artist, - download_features=download_features, - override_existing=override_existing, - default_target=default_target, - exclude_album_type=exclude_album_type, - genre=genre - )) - - album: Album - for album in label.album_collection: - if album.album_type == AlbumType.OTHER: - cls.fetch_details(album) - - if album.album_type in exclude_album_type: - cls.LOGGER.info(f"Skipping {album.option_string} due to the filter. ({album.album_type})") - continue - - r.merge(cls.download_album( - album, - override_existing=override_existing, - default_target=default_target, - genre=genre - )) - - return r - - @classmethod - def download_artist( - cls, - artist: Artist, - exclude_album_type: Set[AlbumType], - download_features: bool = True, - override_existing: bool = False, - default_target: DefaultTarget = None, - genre: str = None - ) -> DownloadResult: - - default_target = DefaultTarget() if default_target is None else copy(default_target) - default_target.artist_object(artist) - - r = DownloadResult() - - cls.fetch_details(artist) - - album: Album - for album in artist.main_album_collection: - if album.album_type in exclude_album_type: - cls.LOGGER.info(f"Skipping {album.option_string} due to the filter. ({album.album_type})") - continue - - r.merge(cls.download_album( - album, - override_existing=override_existing, - default_target=default_target, - genre=genre - )) - - if download_features: - for song in artist.feature_album.song_collection: - r.merge(cls.download_song( - song, - override_existing=override_existing, - default_target=default_target, - genre=genre - )) - - return r - - @classmethod - def download_album( - cls, - album: Album, - override_existing: bool = False, - default_target: DefaultTarget = None, - genre: str = None - ) -> DownloadResult: - - default_target = DefaultTarget() if default_target is None else copy(default_target) - default_target.album_object(album) - - r = DownloadResult() - - cls.fetch_details(album) - - album.update_tracksort() - - cls.LOGGER.info(f"downloading album: {album.title}") - for song in album.song_collection: - r.merge(cls.download_song( - song, - override_existing=override_existing, - default_target=default_target, - genre=genre - )) - - return r - - @classmethod - def download_song( - cls, - song: Song, - override_existing: bool = False, - create_target_on_demand: bool = True, - default_target: DefaultTarget = None, - genre: str = None - ) -> DownloadResult: - cls.LOGGER.debug(f"Setting genre of {song.option_string} to {genre}") - song.genre = genre - - default_target = DefaultTarget() if default_target is None else copy(default_target) - default_target.song_object(song) - - cls.fetch_details(song) - + if song.target_collection.empty: - if create_target_on_demand and not song.main_artist_collection.empty and not song.album_collection.empty: - song.target_collection.append(default_target.target) - else: - return DownloadResult(error_message=f"No target exists for {song.title}, but create_target_on_demand is False.") - - target: Target - if any(target.exists for target in song.target_collection) and not override_existing: - r = DownloadResult(total=1, fail=0) - - existing_target: Target - for existing_target in song.target_collection: - if existing_target.exists: - r.merge(cls._post_process_targets(song=song, temp_target=existing_target)) - break - - return r - - sources = song.source_collection.get_sources_from_page(cls.SOURCE_TYPE) + song.target_collection.append(new_target) + + sources = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) if len(sources) == 0: - return DownloadResult(error_message=f"No source found for {song.title} as {cls.__name__}.") + return DownloadResult(error_message=f"No source found for {song.title} as {self.__class__.__name__}.") temp_target: Target = Target( path=shared.TEMP_DIR, file=str(random.randint(0, 999999)) ) - - r = cls._download_song_to_targets(source=sources[0], target=temp_target, desc=song.title) + + r = self._download_song_to_targets(source=sources[0], target=temp_target, desc=song.title) if not r.is_fatal_error: - r.merge(cls._post_process_targets(song, temp_target)) + r.merge(self._post_process_targets(song, temp_target)) return r - - @classmethod - def _post_process_targets(cls, song: Song, temp_target: Target) -> DownloadResult: + + def _post_process_targets(self, song: Song, temp_target: Target) -> DownloadResult: correct_codec(temp_target) write_metadata_to_target(song.metadata, temp_target) @@ -559,9 +339,10 @@ class Page(threading.Thread): if temp_target is not target: temp_target.copy_content(target) r.add_target(target) + + temp_target.delete() return r - - @classmethod - def _download_song_to_targets(cls, source: Source, target: Target, desc: str = None) -> DownloadResult: + + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: return DownloadResult() diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 4b6b390..e4dea1a 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -567,7 +567,7 @@ class Musify(Page): if stop_at_level > 1: song: Song for song in album.song_collection: - sources = song.source_collection.get_sources_from_page(cls.SOURCE_TYPE) + sources = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) for source in sources: song.merge(self.fetch_song(source=source)) @@ -997,61 +997,8 @@ class Musify(Page): def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() - - -class OldMusify(Page): - API_SESSION: requests.Session = requests.Session() - API_SESSION.headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", - "Connection": "keep-alive", - "Referer": "https://musify.club/" - } - API_SESSION.proxies = shared.proxies - TIMEOUT = 7 - POST_TIMEOUT = 15 - TRIES = 5 - HOST = "https://musify.club" - - CONNECTION = Connection( - host="https://musify.club/", - logger=MUSIFY_LOGGER - ) - - SOURCE_TYPE = SourcePages.MUSIFY - LOGGER = shared.MUSIFY_LOGGER - - - - - - @classmethod - def get_plaintext_query(cls, query: Query) -> str: - if query.album is None: - return f"{query.artist or '*'} - {query.song or '*'}" - return f"{query.artist or '*'} - {query.album or '*'} - {query.song or '*'}" - - - - - - - - - @classmethod - def _get_type_of_url(cls, url: str) -> Optional[Union[Type[Song], Type[Album], Type[Artist], Type[Label]]]: - url: MusifyUrl = cls.parse_url(url) - - if url.source_type == MusifyTypes.ARTIST: - return Artist - if url.source_type == MusifyTypes.RELEASE: - return Album - if url.source_type == MusifyTypes.SONG: - return Song - return None - - @classmethod - def _download_song_to_targets(cls, source: Source, target: Target, desc: str = None) -> DownloadResult: + def download_song_to_targets(self, source: Source, target: Target, desc: str = None) -> DownloadResult: """ https://musify.club/track/im-in-a-coffin-life-never-was-waste-of-skin-16360302 https://musify.club/track/dl/16360302/im-in-a-coffin-life-never-was-waste-of-skin.mp3 @@ -1059,15 +1006,15 @@ class OldMusify(Page): endpoint = source.audio_url if source.audio_url is None: - url: MusifyUrl = cls.parse_url(source.url) + url: MusifyUrl = parse_url(source.url) if url.source_type != MusifyTypes.SONG: return DownloadResult(error_message=f"The url is not of the type Song: {source.url}") endpoint = f"https://musify.club/track/dl/{url.musify_id}/{url.name_without_id}.mp3" - cls.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") + self.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = cls.CONNECTION.get(endpoint, stream=True, raw_url=True) + r = self.connection.get(endpoint, stream=True, raw_url=True) if r is None: return DownloadResult(error_message=f"couldn't connect to {endpoint}") From 6caa6a28a66341df1b219165c23e6f4d8c46380f Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 25 May 2023 13:46:47 +0200 Subject: [PATCH 35/81] progress on threading --- src/music_kraken/download/page_attributes.py | 4 ++-- src/music_kraken/download/search.py | 2 +- src/music_kraken/pages/abstract.py | 8 ++++++-- src/music_kraken/pages/encyclopaedia_metallum.py | 4 ++-- src/music_kraken/pages/musify.py | 4 ++-- src/music_kraken/pages/preset.py | 4 ++-- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 861e7e6..cec3b5e 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -10,7 +10,7 @@ SOURCE_PAGE_MAP: Dict[SourcePages, Page] = dict() ALL_PAGES: Tuple[Page, ...] = ( EncyclopaediaMetallum(), - Musify() + Musify()) ) AUDIO_PAGES: Tuple[Page, ...] = ( @@ -27,7 +27,7 @@ for i, page in enumerate(ALL_PAGES): NAME_PAGE_MAP[type(page).__name__.lower()] = page NAME_PAGE_MAP[SHORTHANDS[i].lower()] = page - PAGE_NAME_MAP[page] = SHORTHANDS[i] + PAGE_NAME_MAP[type(page)] = SHORTHANDS[i] SOURCE_PAGE_MAP[page.SOURCE_TYPE] = page \ No newline at end of file diff --git a/src/music_kraken/download/search.py b/src/music_kraken/download/search.py index ba053c5..91e556c 100644 --- a/src/music_kraken/download/search.py +++ b/src/music_kraken/download/search.py @@ -203,7 +203,7 @@ class Search(Download): page = self._get_page_from_source(source=source) if page in self.audio_pages: - return page.download(music_object=self._current_option._derive_from, genre=genre, download_all=download_all, **kwargs) + return page.download(music_object=self._current_option._derive_from, genre=genre, download_all=download_all) return DownloadResult(error_message=f"Didn't find a source for {self._current_option._derive_from.option_string}.") diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 31b7dd2..65df60f 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -3,6 +3,7 @@ import random from copy import copy from typing import Optional, Union, Type, Dict, Set, List import threading +from queue import Queue import requests from bs4 import BeautifulSoup @@ -108,11 +109,14 @@ class Page(threading.Thread): This is an abstract class, laying out the functionality for every other class fetching something """ - + SOURCE_TYPE: SourcePages LOGGER = logging.getLogger("this shouldn't be used") - def __init__(self): + def __init__(self, search_queue: Queue, search_result_queue: Queue): + self.search_queue = search_queue + self.search_result_queue = search_result_queue + threading.Thread.__init__(self) def run(self) -> None: diff --git a/src/music_kraken/pages/encyclopaedia_metallum.py b/src/music_kraken/pages/encyclopaedia_metallum.py index 42e0e5f..5810dab 100644 --- a/src/music_kraken/pages/encyclopaedia_metallum.py +++ b/src/music_kraken/pages/encyclopaedia_metallum.py @@ -110,13 +110,13 @@ class EncyclopaediaMetallum(Page): SOURCE_TYPE = SourcePages.ENCYCLOPAEDIA_METALLUM LOGGER = ENCYCLOPAEDIA_METALLUM_LOGGER - def __init__(self): + def __init__(self, **kwargs): self.connection: Connection = Connection( host="https://www.metal-archives.com/", logger=ENCYCLOPAEDIA_METALLUM_LOGGER ) - super().__init__() + super().__init__(**kwargs) def song_search(self, song: Song) -> List[Song]: endpoint = "https://www.metal-archives.com/search/ajax-advanced/searching/songs/?songTitle={song}&bandName={" \ diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index e4dea1a..34c9fe6 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -115,13 +115,13 @@ class Musify(Page): HOST = "https://musify.club" - def __init__(self): + def __init__(self, *args, **kwargs): self.connection: Connection = Connection( host="https://musify.club/", logger=self.LOGGER ) - super().__init__() + super().__init__(*args, **kwargs) def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: if source.url is None: diff --git a/src/music_kraken/pages/preset.py b/src/music_kraken/pages/preset.py index e688074..7fc8212 100644 --- a/src/music_kraken/pages/preset.py +++ b/src/music_kraken/pages/preset.py @@ -20,13 +20,13 @@ class Preset(Page): SOURCE_TYPE = SourcePages.PRESET LOGGER = logging.getLogger("preset") - def __init__(self): + def __init__(self, *args, **kwargs): self.connection: Connection = Connection( host="https://www.preset.cum/", logger=self.LOGGER ) - super().__init__() + super().__init__(*args, **kwargs) def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: return super().get_source_type(source) From e996bd4e4ff095f782551f65d77dfef05f381596 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 26 May 2023 09:00:02 +0200 Subject: [PATCH 36/81] tried multithreading, not worth it at the current point --- src/music_kraken/download/page_attributes.py | 34 +++++++++++-------- src/music_kraken/pages/abstract.py | 33 ++++++++++++++---- src/music_kraken/utils/shared.py | 2 ++ .../utils/support_classes/__init__.py | 1 + .../utils/support_classes/thread_classes.py | 12 +++++++ 5 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 src/music_kraken/utils/support_classes/thread_classes.py diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index cec3b5e..4c06c06 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -1,26 +1,32 @@ -from typing import Tuple, Type, Dict +from typing import Tuple, Type, Dict, List +import threading +from queue import Queue from ..utils.enums.source import SourcePages +from ..utils.support_classes import Query, EndThread from ..pages import Page, EncyclopaediaMetallum, Musify - -NAME_PAGE_MAP: Dict[str, Page] = dict() -PAGE_NAME_MAP: Dict[Page, str] = dict() -SOURCE_PAGE_MAP: Dict[SourcePages, Page] = dict() - -ALL_PAGES: Tuple[Page, ...] = ( - EncyclopaediaMetallum(), - Musify()) +ALL_PAGES: Tuple[Type[Page], ...] = ( + EncyclopaediaMetallum, + Musify ) -AUDIO_PAGES: Tuple[Page, ...] = ( - Musify(), +AUDIO_PAGES: Tuple[Type[Page], ...] = ( + Musify, ) -SHADY_PAGES: Tuple[Page, ...] = ( - Musify(), +SHADY_PAGES: Tuple[Type[Page], ...] = ( + Musify, ) + +exit_threads = EndThread() +search_queue: Queue[Query] = Queue() + +_page_threads: Dict[Type[Page], List[Page]] = dict() + + +""" # this needs to be case-insensitive SHORTHANDS = ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z') for i, page in enumerate(ALL_PAGES): @@ -30,4 +36,4 @@ for i, page in enumerate(ALL_PAGES): PAGE_NAME_MAP[type(page)] = SHORTHANDS[i] SOURCE_PAGE_MAP[page.SOURCE_TYPE] = page - \ No newline at end of file +""" \ No newline at end of file diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 65df60f..fe6c3b0 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -24,8 +24,8 @@ from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType from ..audio import write_metadata_to_target, correct_codec from ..utils import shared -from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE -from ..utils.support_classes import Query, DownloadResult, DefaultTarget +from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE, THREADED +from ..utils.support_classes import Query, DownloadResult, DefaultTarget, EndThread, FinishedSearch INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] @@ -104,7 +104,14 @@ def merge_together(old_object: DatabaseObject, new_object: DatabaseObject) -> Da return old_object -class Page(threading.Thread): +class LoreIpsum: + pass + + +Parent = threading.Thread if THREADED else LoreIpsum + + +class Page(Parent): """ This is an abstract class, laying out the functionality for every other class fetching something @@ -113,14 +120,24 @@ class Page(threading.Thread): SOURCE_TYPE: SourcePages LOGGER = logging.getLogger("this shouldn't be used") - def __init__(self, search_queue: Queue, search_result_queue: Queue): + def __init__(self, end_event: EndThread, search_queue: Queue, search_result_queue: Queue): + self.end_event = end_event + self.search_queue = search_queue self.search_result_queue = search_result_queue - threading.Thread.__init__(self) + Parent.__init__(self) + + @property + def _empty_working_queues(self): + return self.search_queue.empty() def run(self) -> None: - pass + while bool(self.end_event) and self._empty_working_queues: + if not self.search_queue.empty(): + self.search(self.search_queue.get()) + self.search_result_queue.put(FinishedSearch()) + continue def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: return None @@ -146,7 +163,9 @@ class Page(threading.Thread): r = [] for default_query in query.default_search: - r.extend(self.general_search(default_query)) + for single_option in self.general_search(default_query): + r.append(single_option) + self.search_result_queue.put(single_option) return r diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index ef396e2..b17d3cd 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -103,3 +103,5 @@ SORT_BY_DATE = AUDIO_SECTION.SORT_BY_DATE.object_from_value SORT_BY_ALBUM_TYPE = AUDIO_SECTION.SORT_BY_ALBUM_TYPE.object_from_value ALBUM_TYPE_BLACKLIST: Set[AlbumType] = set(AUDIO_SECTION.ALBUM_TYPE_BLACKLIST.object_from_value) + +THREADED = False diff --git a/src/music_kraken/utils/support_classes/__init__.py b/src/music_kraken/utils/support_classes/__init__.py index 625ed41..0789e87 100644 --- a/src/music_kraken/utils/support_classes/__init__.py +++ b/src/music_kraken/utils/support_classes/__init__.py @@ -1,3 +1,4 @@ from .default_target import DefaultTarget from .download_result import DownloadResult from .query import Query +from .thread_classes import EndThread, FinishedSearch diff --git a/src/music_kraken/utils/support_classes/thread_classes.py b/src/music_kraken/utils/support_classes/thread_classes.py new file mode 100644 index 0000000..1a17e57 --- /dev/null +++ b/src/music_kraken/utils/support_classes/thread_classes.py @@ -0,0 +1,12 @@ +class EndThread: + _has_ended: bool = False + + def __bool__(self): + return self._has_ended + + def exit(self): + self._has_ended + +class FinishedSearch: + pass + \ No newline at end of file From 2fabd09a71579008f3e0fdf9d4ae7b07d44b5617 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 26 May 2023 10:11:36 +0200 Subject: [PATCH 37/81] much progress in cleaning up --- src/music_kraken/download/page_attributes.py | 44 ++++++++++++++------ src/music_kraken/download/results.py | 25 +++++++++++ src/music_kraken/pages/abstract.py | 21 +++++----- 3 files changed, 68 insertions(+), 22 deletions(-) create mode 100644 src/music_kraken/download/results.py diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 4c06c06..4ba72b5 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -1,29 +1,49 @@ -from typing import Tuple, Type, Dict, List -import threading -from queue import Queue +from typing import Tuple, Type, Dict, List, Set from ..utils.enums.source import SourcePages from ..utils.support_classes import Query, EndThread from ..pages import Page, EncyclopaediaMetallum, Musify -ALL_PAGES: Tuple[Type[Page], ...] = ( +ALL_PAGES: Set[Type[Page]] = { EncyclopaediaMetallum, Musify -) +} -AUDIO_PAGES: Tuple[Type[Page], ...] = ( +AUDIO_PAGES: Set[Type[Page]] = { Musify, -) +} -SHADY_PAGES: Tuple[Type[Page], ...] = ( +SHADY_PAGES: Set[Type[Page]] = { Musify, -) +} -exit_threads = EndThread() -search_queue: Queue[Query] = Queue() -_page_threads: Dict[Type[Page], List[Page]] = dict() +class Pages: + def __init__(self, exclude_pages: Set[Type[Page]] = None, exclude_shady: bool = False) -> None: + # initialize all page instances + self._page_instances: Dict[Type[Page], Page] = dict() + + exclude_pages = exclude_pages if exclude_pages is not None else set() + + if exclude_shady: + exclude_pages = exclude_pages.union(SHADY_PAGES) + + if not exclude_pages.issubset(ALL_PAGES): + raise ValueError(f"The excluded pages have to be a subset of all pages: {exclude_pages} | {ALL_PAGES}") + + def _set_to_tuple(page_set: Set[Type[Page]]) -> Tuple[Type[Page], ...]: + return tuple(sorted(page_set, key=lambda page: page.__name__)) + + self.pages: Tuple[Type[Page], ...] = _set_to_tuple(ALL_PAGES.difference(exclude_pages)) + self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(set(self.pages).intersection(AUDIO_PAGES)) + + for page_type in ALL_PAGES: + self._page_instances[page_type] = page_type() + + def search(self, query: Query): + for page_type in self.pages: + self._page_instances[page_type].search(query=query) """ diff --git a/src/music_kraken/download/results.py b/src/music_kraken/download/results.py new file mode 100644 index 0000000..a6a27fe --- /dev/null +++ b/src/music_kraken/download/results.py @@ -0,0 +1,25 @@ +from typing import Tuple, Type, Dict, List + +from ..objects import DatabaseObject +from ..utils.enums.source import SourcePages +from ..pages import Page, EncyclopaediaMetallum, Musify + +class SearchResults: + def __init__( + self, + pages: Tuple[Type[Page], ...] + + ) -> None: + self.pages = pages + # this would initialize a list for every page, which I don't think I want + # self.results = Dict[Type[Page], List[DatabaseObject]] = {page: [] for page in self.pages} + self.results = Dict[Type[Page], List[DatabaseObject]] = {} + + def add(self, page: Type[Page], search_result: List[DatabaseObject]): + self.results[page] = search_result + + def __str__(self) -> str: + for page in self.pages: + if page not in self.results: + continue + \ No newline at end of file diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index fe6c3b0..2562f5e 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -104,14 +104,7 @@ def merge_together(old_object: DatabaseObject, new_object: DatabaseObject) -> Da return old_object -class LoreIpsum: - pass - - -Parent = threading.Thread if THREADED else LoreIpsum - - -class Page(Parent): +class Page(): """ This is an abstract class, laying out the functionality for every other class fetching something @@ -120,14 +113,21 @@ class Page(Parent): SOURCE_TYPE: SourcePages LOGGER = logging.getLogger("this shouldn't be used") + + def __init__(self): + super().__init__() + + """ + CODE I NEED WHEN I START WITH MULTITHREADING + def __init__(self, end_event: EndThread, search_queue: Queue, search_result_queue: Queue): self.end_event = end_event self.search_queue = search_queue self.search_result_queue = search_result_queue - Parent.__init__(self) - + super().__init__() + @property def _empty_working_queues(self): return self.search_queue.empty() @@ -138,6 +138,7 @@ class Page(Parent): self.search(self.search_queue.get()) self.search_result_queue.put(FinishedSearch()) continue + """ def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: return None From 10f6153199f555beff9c9a2b04f39eb05665bc9a Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 26 May 2023 11:41:20 +0200 Subject: [PATCH 38/81] added wrapper methods, for the basic fuctionalities of the webscraper --- src/music_kraken/download/page_attributes.py | 56 ++++++++++++++++---- src/music_kraken/objects/source.py | 6 ++- src/music_kraken/pages/__init__.py | 2 +- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 4ba72b5..a4630a8 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -1,8 +1,10 @@ from typing import Tuple, Type, Dict, List, Set +from .results import SearchResults +from ..objects import DatabaseObject from ..utils.enums.source import SourcePages -from ..utils.support_classes import Query, EndThread -from ..pages import Page, EncyclopaediaMetallum, Musify +from ..utils.support_classes import Query, DownloadResult +from ..pages import Page, EncyclopaediaMetallum, Musify, INDEPENDENT_DB_OBJECTS ALL_PAGES: Set[Type[Page]] = { EncyclopaediaMetallum, @@ -23,6 +25,7 @@ class Pages: def __init__(self, exclude_pages: Set[Type[Page]] = None, exclude_shady: bool = False) -> None: # initialize all page instances self._page_instances: Dict[Type[Page], Page] = dict() + self._source_to_page: Dict[SourcePages, Type[Page]] = dict() exclude_pages = exclude_pages if exclude_pages is not None else set() @@ -35,15 +38,50 @@ class Pages: def _set_to_tuple(page_set: Set[Type[Page]]) -> Tuple[Type[Page], ...]: return tuple(sorted(page_set, key=lambda page: page.__name__)) - self.pages: Tuple[Type[Page], ...] = _set_to_tuple(ALL_PAGES.difference(exclude_pages)) - self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(set(self.pages).intersection(AUDIO_PAGES)) + self._pages_set: Set[Type[Page]] = ALL_PAGES.difference(exclude_pages) + self.pages: Tuple[Type[Page], ...] = _set_to_tuple(ALL_PAGES.difference(self.pages)) + + self._audio_pages_set: Set[Type[Page]] = self._pages_set.intersection(AUDIO_PAGES) + self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(self._audio_pages_set) - for page_type in ALL_PAGES: - self._page_instances[page_type] = page_type() - - def search(self, query: Query): for page_type in self.pages: - self._page_instances[page_type].search(query=query) + self._page_instances[page_type] = page_type() + self._source_to_page[page_type.SOURCE_TYPE] = page_type + + def search(self, query: Query) -> SearchResults: + result = SearchResults() + + for page_type in self.pages: + result.add( + page=page_type, + search_result=self._page_instances[page_type].search(query=query) + ) + + return result + + def fetch_details(self, music_object: DatabaseObject, stop_at_level: int = 1) -> DatabaseObject: + if not isinstance(music_object, INDEPENDENT_DB_OBJECTS): + return music_object + + for source_page in music_object.source_collection.source_pages: + page_type = self._source_to_page[source_page] + + if page_type in self._pages_set: + music_object.merge(self._page_instances[page_type].fetch_details(music_object=music_object, stop_at_level=stop_at_level)) + + return music_object + + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + if not isinstance(music_object, INDEPENDENT_DB_OBJECTS): + return DownloadResult(error_message=f"{type(music_object).__name__} can't be downloaded.") + + _page_types = set(self._source_to_page[src] for src in music_object.source_collection.source_pages) + audio_pages = self._audio_pages_set.intersection(_page_types) + + for download_page in audio_pages: + return self._page_instances[download_page].download(genre=genre, download_all=download_all) + + return DownloadResult(error_message=f"No audio source has been found for {music_object}.") """ diff --git a/src/music_kraken/objects/source.py b/src/music_kraken/objects/source.py index 2d37f59..1680821 100644 --- a/src/music_kraken/objects/source.py +++ b/src/music_kraken/objects/source.py @@ -1,6 +1,6 @@ from collections import defaultdict from enum import Enum -from typing import List, Dict, Tuple, Optional +from typing import List, Dict, Set, Tuple, Optional from urllib.parse import urlparse from ..utils.enums.source import SourcePages, SourceTypes @@ -128,6 +128,10 @@ class SourceCollection(Collection): super().map_element(source) self._page_to_source_list[source.page_enum].append(source) + + @property + def source_pages(self) -> Set[SourcePages]: + return set(source.page_enum for source in self._data) def get_sources_from_page(self, source_page: SourcePages) -> List[Source]: """ diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index 0423f94..a7e2a61 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -1,3 +1,3 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum from .musify import Musify -from .abstract import Page +from .abstract import Page, INDEPENDENT_DB_OBJECTS From ff5a79a3c76e7c4fdf8c6b6c655d681ef8b56c00 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 12 Jun 2023 10:16:56 +0200 Subject: [PATCH 39/81] added a new custom exception, and an api, to fetch direct url --- src/music_kraken/download/download.py | 26 +++----------------- src/music_kraken/download/page_attributes.py | 25 +++++++++---------- src/music_kraken/download/results.py | 5 ++++ src/music_kraken/utils/exception/download.py | 11 +++++++++ 4 files changed, 32 insertions(+), 35 deletions(-) create mode 100644 src/music_kraken/utils/exception/download.py diff --git a/src/music_kraken/download/download.py b/src/music_kraken/download/download.py index 195b04c..2df3176 100644 --- a/src/music_kraken/download/download.py +++ b/src/music_kraken/download/download.py @@ -1,6 +1,6 @@ from typing import Optional, Tuple, Type, Set, Union, List -from . import page_attributes +from .page_attributes import Pages from ..pages import Page from ..objects import Song, Album, Artist, Label, Source @@ -10,29 +10,11 @@ MusicObject = Union[Song, Album, Artist, Label] class Download: def __init__( self, - pages: Tuple[Page] = page_attributes.ALL_PAGES, - exclude_pages=None, - exclude_shady: bool = False, + exclude_pages: Set[Type[Page]] = None, + exclude_shady: bool = False ) -> None: - if exclude_pages is None: - exclude_pages = set() - _page_list: List[Page] = [] - _audio_page_list: List[Page] = [] - - for page in pages: - if exclude_shady and page in page_attributes.SHADY_PAGES: - continue - if page in exclude_pages: - continue - - _page_list.append(page) - - if page in page_attributes.AUDIO_PAGES: - _audio_page_list.append(page) - - self.pages: Tuple[Page] = tuple(_page_list) - self.audio_pages: Tuple[Page] = tuple(_audio_page_list) + self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) def fetch_details(self, music_object: MusicObject) -> MusicObject: for page in self.pages: diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index a4630a8..c3ef381 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -1,9 +1,10 @@ from typing import Tuple, Type, Dict, List, Set from .results import SearchResults -from ..objects import DatabaseObject +from ..objects import DatabaseObject, Source from ..utils.enums.source import SourcePages from ..utils.support_classes import Query, DownloadResult +from ..utils.exception.download import UrlNotFoundException from ..pages import Page, EncyclopaediaMetallum, Musify, INDEPENDENT_DB_OBJECTS ALL_PAGES: Set[Type[Page]] = { @@ -83,15 +84,13 @@ class Pages: return DownloadResult(error_message=f"No audio source has been found for {music_object}.") - -""" -# this needs to be case-insensitive -SHORTHANDS = ('a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z') -for i, page in enumerate(ALL_PAGES): - NAME_PAGE_MAP[type(page).__name__.lower()] = page - NAME_PAGE_MAP[SHORTHANDS[i].lower()] = page - - PAGE_NAME_MAP[type(page)] = SHORTHANDS[i] - - SOURCE_PAGE_MAP[page.SOURCE_TYPE] = page -""" \ No newline at end of file + def fetch_url(self, url: str, stop_at_level: int = 2) -> DatabaseObject: + source = Source.match_url(url, SourcePages.MANUAL) + + if source is None: + raise UrlNotFoundException(url=url) + + _actual_page = self._source_to_page[source.page_enum] + + + return self._page_instances[_actual_page].fetch_object_from_source(source=source, stop_at_level=stop_at_level) \ No newline at end of file diff --git a/src/music_kraken/download/results.py b/src/music_kraken/download/results.py index a6a27fe..194425a 100644 --- a/src/music_kraken/download/results.py +++ b/src/music_kraken/download/results.py @@ -16,6 +16,11 @@ class SearchResults: self.results = Dict[Type[Page], List[DatabaseObject]] = {} def add(self, page: Type[Page], search_result: List[DatabaseObject]): + """ + adds a list of found music objects to the according page + WARNING: if a page already has search results, they are just gonna be overwritten + """ + self.results[page] = search_result def __str__(self) -> str: diff --git a/src/music_kraken/utils/exception/download.py b/src/music_kraken/utils/exception/download.py new file mode 100644 index 0000000..8ee4fe3 --- /dev/null +++ b/src/music_kraken/utils/exception/download.py @@ -0,0 +1,11 @@ +class DownloadException(Exception): + pass + + +class UrlNotFoundException(DownloadException): + def __init__(self, url: str, *args: object) -> None: + self.url = url + super().__init__(*args) + + def __str__(self) -> str: + return f"Couldn't find the page of {self.url}" From 58dbb5a9c7cae9ca298902ed2c6d259f5b615d03 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 12 Jun 2023 11:02:05 +0200 Subject: [PATCH 40/81] added classes to querry through the results --- src/music_kraken/download/download.py | 30 ----------- src/music_kraken/download/results.py | 74 ++++++++++++++++++++++++--- src/music_kraken/download/search.py | 14 ++--- 3 files changed, 71 insertions(+), 47 deletions(-) delete mode 100644 src/music_kraken/download/download.py diff --git a/src/music_kraken/download/download.py b/src/music_kraken/download/download.py deleted file mode 100644 index 2df3176..0000000 --- a/src/music_kraken/download/download.py +++ /dev/null @@ -1,30 +0,0 @@ -from typing import Optional, Tuple, Type, Set, Union, List - -from .page_attributes import Pages -from ..pages import Page -from ..objects import Song, Album, Artist, Label, Source - -MusicObject = Union[Song, Album, Artist, Label] - - -class Download: - def __init__( - self, - exclude_pages: Set[Type[Page]] = None, - exclude_shady: bool = False - ) -> None: - - self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) - - def fetch_details(self, music_object: MusicObject) -> MusicObject: - for page in self.pages: - page.fetch_details(music_object=music_object) - return music_object - - def fetch_source(self, source: Source) -> Optional[MusicObject]: - source_page = page_attributes.SOURCE_PAGE_MAP[source.page_enum] - - if source_page not in self.pages: - return - - return source_page.fetch_object_from_source(source) diff --git a/src/music_kraken/download/results.py b/src/music_kraken/download/results.py index 194425a..4d9d72e 100644 --- a/src/music_kraken/download/results.py +++ b/src/music_kraken/download/results.py @@ -1,15 +1,42 @@ -from typing import Tuple, Type, Dict, List +from typing import Tuple, Type, Dict, List, Generator +from dataclasses import dataclass from ..objects import DatabaseObject from ..utils.enums.source import SourcePages from ..pages import Page, EncyclopaediaMetallum, Musify -class SearchResults: + +@dataclass +class Option: + index: int + music_object: DatabaseObject + + +class Results: + def __init__(self) -> None: + self._by_index = Dict[int, DatabaseObject] = dict() + + def __iter__(self) -> Generator[DatabaseObject]: + for option in self.formated_generator(): + if isinstance(option, Option): + yield option.music_object + + def formated_generator(self, max_items_per_page: int = 10) -> Generator[Type[Page], Option]: + self._by_index = dict() + + def get_music_object_by_index(self, index: int) -> DatabaseObject: + # if this throws a key error, either the formated generator needs to be iterated, or the option doesn't exist. + return self._by_index[index] + + +class SearchResults(Results): def __init__( self, pages: Tuple[Type[Page], ...] ) -> None: + super().__init() + self.pages = pages # this would initialize a list for every page, which I don't think I want # self.results = Dict[Type[Page], List[DatabaseObject]] = {page: [] for page in self.pages} @@ -22,9 +49,42 @@ class SearchResults: """ self.results[page] = search_result + + def get_page_results(self, page: Type[Page]) -> "PageResults": + return PageResults(page, self.results.get(page, [])) + + def formated_generator(self, max_items_per_page: int = 10): + super().formated_generator() + i = 0 - def __str__(self) -> str: - for page in self.pages: - if page not in self.results: - continue - \ No newline at end of file + for page in self.results: + yield page + + j = 0 + for option in self.results[page]: + yield Option(i, option) + self._by_index[i] = option + i += 1 + j += 1 + + if j >= max_items_per_page: + break + + +class PageResults(Results): + def __init__(self, page: Type[Page], results: List[DatabaseObject]) -> None: + super().__init() + + self.page: Type[Page] = page + self.results: List[DatabaseObject] = results + + def formated_generator(self, max_items_per_page: int = 10): + super().formated_generator() + i = 0 + + yield self.page + + for option in self.results: + yield Option(i, option) + self._by_index[i] = option + i += 1 diff --git a/src/music_kraken/download/search.py b/src/music_kraken/download/search.py index 91e556c..ce667a2 100644 --- a/src/music_kraken/download/search.py +++ b/src/music_kraken/download/search.py @@ -1,7 +1,6 @@ from typing import Tuple, List, Set, Type, Optional, Dict -from . import page_attributes -from .download import Download +from .page_attributes import Pages from .multiple_options import MultiPageOptions from ..pages.abstract import Page from ..utils.support_classes import DownloadResult, Query @@ -9,20 +8,15 @@ from ..objects import DatabaseObject, Source, Artist, Song, Album from ..utils.enums.source import SourcePages -class Search(Download): +class Search: def __init__( self, - pages: Tuple[Type[Page]] = page_attributes.ALL_PAGES, - exclude_pages: Set[Type[Page]] = set(), + exclude_pages: Set[Type[Page]] = None, exclude_shady: bool = False, max_displayed_options: int = 10, option_digits: int = 3, ) -> None: - super().__init__( - pages=pages, - exclude_pages=exclude_pages, - exclude_shady=exclude_shady - ) + self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) self.max_displayed_options = max_displayed_options self.option_digits: int = option_digits From 4cf902b05b0262c987301546212f4bd910338e02 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 12 Jun 2023 14:56:14 +0200 Subject: [PATCH 41/81] started reimplementing the shell --- src/music_kraken/__init__.py | 141 +-------- src/music_kraken/cli/__init__.py | 1 + src/music_kraken/cli/download/__init__.py | 0 src/music_kraken/cli/download/shell.py | 316 +++++++++++++++++++ src/music_kraken/cli/options/__init__.py | 0 src/music_kraken/download/__init__.py | 1 - src/music_kraken/download/page_attributes.py | 2 +- src/music_kraken/download/results.py | 26 +- src/music_kraken/objects/parents.py | 4 +- src/music_kraken/objects/song.py | 18 +- src/music_kraken/pages/abstract.py | 1 - 11 files changed, 356 insertions(+), 154 deletions(-) create mode 100644 src/music_kraken/cli/__init__.py create mode 100644 src/music_kraken/cli/download/__init__.py create mode 100644 src/music_kraken/cli/download/shell.py create mode 100644 src/music_kraken/cli/options/__init__.py diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index f4debe2..147be6a 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -6,6 +6,7 @@ from typing import List import gc import musicbrainzngs +from .cli import Shell from . import objects, pages, download from .utils import exception, shared, path_manager from .utils.config import config, read, write, PATHS_SECTION @@ -132,138 +133,18 @@ def cli( direct_download_url: str = None, command_list: List[str] = None ): - def get_existing_genre() -> List[str]: - """ - gets the name of all subdirectories of shared.MUSIC_DIR, - but filters out all directories, where the name matches with any patern - from shared.NOT_A_GENRE_REGEX. - """ - existing_genres: List[str] = [] - - # get all subdirectories of MUSIC_DIR, not the files in the dir. - existing_subdirectories: List[Path] = [f for f in MUSIC_DIR.iterdir() if f.is_dir()] - - for subdirectory in existing_subdirectories: - name: str = subdirectory.name - - if not any(re.match(regex_pattern, name) for regex_pattern in NOT_A_GENRE_REGEX): - existing_genres.append(name) - - existing_genres.sort() - - return existing_genres - - def get_genre(): - existing_genres = get_existing_genre() - for i, genre_option in enumerate(existing_genres): - print(f"{i + 1:0>2}: {genre_option}") - - while True: - genre = input("Id or new genre: ") - - if genre.isdigit(): - genre_id = int(genre) - 1 - if genre_id >= len(existing_genres): - print(f"No genre under the id {genre_id + 1}.") - continue - - return existing_genres[genre_id] - - new_genre = fit_to_file_system(genre) - - agree_inputs = {"y", "yes", "ok"} - verification = input(f"create new genre \"{new_genre}\"? (Y/N): ").lower() - if verification in agree_inputs: - return new_genre - - def next_search(_search: download.Search, query: str) -> bool: - """ - :param _search: - :param query: - :return exit in the next step: - """ - nonlocal genre - nonlocal download_all - - query: str = query.strip() - parsed: str = query.lower() - - if parsed in EXIT_COMMANDS: - return True - - if parsed == ".": - return False - if parsed == "..": - _search.goto_previous() - return False - - if parsed.isdigit(): - _search.choose_index(int(parsed)) - return False - - if parsed in DOWNLOAD_COMMANDS: - r = _search.download_chosen(genre=genre, download_all=download_all) - - print() - print(r) - print() - - return not r.is_mild_failure - - url = re.match(URL_REGEX, query) - if url is not None: - if not _search.search_url(url.string): - print("The given url couldn't be found.") - return False - - page = _search.get_page_from_query(parsed) - if page is not None: - _search.choose_page(page) - return False - - # if everything else is not valid search - _search.search(query) - return False - - if genre is None: - genre = get_genre() - print() - - print_cute_message() - print() - print(f"Downloading to: \"{genre}\"") - print() - - search = download.Search() - - # directly download url - if direct_download_url is not None: - if search.search_url(direct_download_url): - r = search.download_chosen(genre=genre, download_all=download_all) - print() - print(r) - print() - else: - print(f"Sorry, could not download the url: {direct_download_url}") - - exit_message() - return - - # run one command after another from the command list + shell = Shell(genre=genre) + if command_list is not None: for command in command_list: - print(f">> {command}") - if next_search(search, command): - break - print(search) - - exit_message() + shell.process_input(command) return - # the actual cli - while True: - if next_search(search, input(">> ")): - break - print(search) - + if direct_download_url is not None: + if shell.download(direct_download_url, download_all=download_all): + exit_message() + return + + shell.mainloop() + exit_message() diff --git a/src/music_kraken/cli/__init__.py b/src/music_kraken/cli/__init__.py new file mode 100644 index 0000000..c21af81 --- /dev/null +++ b/src/music_kraken/cli/__init__.py @@ -0,0 +1 @@ +from .download.shell import Shell \ No newline at end of file diff --git a/src/music_kraken/cli/download/__init__.py b/src/music_kraken/cli/download/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/download/shell.py new file mode 100644 index 0000000..8cb627a --- /dev/null +++ b/src/music_kraken/cli/download/shell.py @@ -0,0 +1,316 @@ +from typing import Set, Type, Dict, List +from pathlib import Path +import re + +from ...utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX +from ...utils.regex import URL_PATTERN +from ...utils.string_processing import fit_to_file_system +from ...utils.support_classes import Query +from ...download.results import Results, SearchResults, Option, PageResults +from ...download.page_attributes import Pages +from ...pages import Page +from ...objects import Song, Album, Artist, DatabaseObject + + +""" +This is the implementation of the Shell + +# Behaviour + +## Searching + +```mkshell +> s: {querry or url} + +# examples +> s: https://musify.club/release/some-random-release-183028492 +> s: r: #a an Artist #r some random Release +``` + +Searches for an url, or an query + +### Query Syntax + +``` +#a {artist} #r {release} #t {track} +``` + +You can escape stuff like `#` doing this: `\#` + +## Downloading + +To download something, you either need a direct link, or you need to have already searched for options + +```mkshell +> d: {option ids or direct url} + +# examples +> d: 0, 3, 4 +> d: 1 +> d: https://musify.club/release/some-random-release-183028492 +``` + +## Misc + +### Exit + +```mkshell +> q +> quit +> exit +> abort +``` + +### Current Options + +```mkshell +> . +``` + +### Previous Options + +``` +> .. +``` + +""" + +EXIT_COMMANDS = {"q", "quit", "exit", "abort"} +ALPHABET = "abcdefghijklmnopqrstuvwxyz" +PAGE_NAME_FILL = "-" +MAX_PAGE_LEN = 21 + + +def get_existing_genre() -> List[str]: + """ + gets the name of all subdirectories of shared.MUSIC_DIR, + but filters out all directories, where the name matches with any patern + from shared.NOT_A_GENRE_REGEX. + """ + existing_genres: List[str] = [] + + # get all subdirectories of MUSIC_DIR, not the files in the dir. + existing_subdirectories: List[Path] = [f for f in MUSIC_DIR.iterdir() if f.is_dir()] + + for subdirectory in existing_subdirectories: + name: str = subdirectory.name + + if not any(re.match(regex_pattern, name) for regex_pattern in NOT_A_GENRE_REGEX): + existing_genres.append(name) + + existing_genres.sort() + + return existing_genres + +def get_genre(): + existing_genres = get_existing_genre() + for i, genre_option in enumerate(existing_genres): + print(f"{i + 1:0>2}: {genre_option}") + + while True: + genre = input("Id or new genre: ") + + if genre.isdigit(): + genre_id = int(genre) - 1 + if genre_id >= len(existing_genres): + print(f"No genre under the id {genre_id + 1}.") + continue + + return existing_genres[genre_id] + + new_genre = fit_to_file_system(genre) + + agree_inputs = {"y", "yes", "ok"} + verification = input(f"create new genre \"{new_genre}\"? (Y/N): ").lower() + if verification in agree_inputs: + return new_genre + + +def help_message(): + print() + print(""" +to search: +> s: {query or url} +> s: https://musify.club/release/some-random-release-183028492 +> s: #a {artist} #r {release} #t {track} + +to download: +> d: {option ids or direct url} +> d: 0, 3, 4 +> d: 1 +> d: https://musify.club/release/some-random-release-183028492 + +have fun :3 + """.strip()) + print() + + +class Shell: + """ + TODO: + + - Implement search and download for direct urls + """ + + def __init__( + self, + exclude_pages: Set[Type[Page]] = None, + exclude_shady: bool = False, + max_displayed_options: int = 10, + option_digits: int = 3, + genre: str = None + ) -> None: + self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) + + self.page_dict: Dict[str, Type[Page]] = dict() + + self.max_displayed_options = max_displayed_options + self.option_digits: int = option_digits + + self.current_results: Results = SearchResults + + self.genre = genre or get_genre() + + print() + print(f"Downloading to: \"{self.genre}\"") + print() + + + def print_current_options(self): + self.page_dict = dict() + + page_count = 0 + for option in self.current_results.formated_generator(max_items_per_page=self.max_displayed_options): + if isinstance(option, Option): + print(f"{option.index:0{self.option_digits}} {option.music_object.option_string}") + else: + prefix = ALPHABET[page_count%len(ALPHABET)] + print(f"({prefix}) ------------------------{option.__name__:{PAGE_NAME_FILL}<{MAX_PAGE_LEN}}------------") + + self.page_dict[prefix] = option + self.page_dict[option.__name__] = option + + page_count += 1 + + def set_current_options(self, current_options: Results): + self.current_results = current_options + + def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: + song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) + album = None if not "r" in key_text else Album(title=key_text["r"], dynamic=True) + artist = None if not "a" in key_text else Artist(name=key_text["a"], dynamic=True) + + if song is not None: + song.album_collection.append(album) + song.main_artist_collection.append(artist) + return Query(raw_query=query, music_object=song) + + if album is not None: + album.artist_collection.append(artist) + return Query(raw_query=query, music_object=album) + + if artist is not None: + return Query(raw_query=query, music_object=artist) + + def search(self, query: str): + special_characters = "#\\" + query = query + " " + + key_text = {} + + skip_next = False + escape_next = False + new_text = "" + latest_key: str = None + for i in range(len(query) - 1): + current_char = query[i] + next_char = query[i+1] + + if skip_next: + skip_next = False + continue + + if escape_next: + new_text += current_char + escape_next = False + + # escaping + if current_char == "\\": + if next_char in special_characters: + escape_next = True + continue + + if current_char == "#": + if latest_key is not None: + key_text[latest_key] = new_text + new_text = "" + + latest_key = next_char + skip_next = True + continue + + new_text += current_char + + if latest_key is not None: + key_text[latest_key] = new_text + + + parsed_query: Query = self._process_parsed(key_text, query) + + self.set_current_options(self.pages.search(parsed_query)) + self.print_current_options() + + def goto(self, index: int): + page: Type[Page] + music_object: DatabaseObject + + try: + page, music_object = self.current_results.get_music_object_by_index(index) + except IndexError: + print() + print(f"The option {index} doesn't exist.") + print() + return + + self.pages.fetch_details(music_object) + + self.set_current_options(PageResults(page, music_object.options)) + + self.print_current_options() + + + def download(self, download_str: str, download_all: bool = False) -> bool: + return False + + def process_input(self, input_str: str) -> bool: + input_str = input_str.strip() + processed_input: str = input_str.lower() + + if processed_input in EXIT_COMMANDS: + return True + + if processed_input == ".": + self.print_current_options() + return False + + if processed_input.startswith("s: "): + self.search(input_str[3:]) + return False + + if processed_input.startswith("d: "): + return self.download(input_str[3:]) + + if processed_input.isdigit(): + self.goto(int(processed_input)) + return False + + if processed_input != "help": + print("Invalid input.") + help_message() + return False + + def mainloop(self): + while True: + if self.process_input(input("> ")): + return + \ No newline at end of file diff --git a/src/music_kraken/cli/options/__init__.py b/src/music_kraken/cli/options/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/music_kraken/download/__init__.py b/src/music_kraken/download/__init__.py index e670c65..327a571 100644 --- a/src/music_kraken/download/__init__.py +++ b/src/music_kraken/download/__init__.py @@ -1,2 +1 @@ from .search import Search -from .download import Download diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index c3ef381..bb392f8 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -40,7 +40,7 @@ class Pages: return tuple(sorted(page_set, key=lambda page: page.__name__)) self._pages_set: Set[Type[Page]] = ALL_PAGES.difference(exclude_pages) - self.pages: Tuple[Type[Page], ...] = _set_to_tuple(ALL_PAGES.difference(self.pages)) + self.pages: Tuple[Type[Page], ...] = _set_to_tuple(self._pages_set) self._audio_pages_set: Set[Type[Page]] = self._pages_set.intersection(AUDIO_PAGES) self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(self._audio_pages_set) diff --git a/src/music_kraken/download/results.py b/src/music_kraken/download/results.py index 4d9d72e..fa41369 100644 --- a/src/music_kraken/download/results.py +++ b/src/music_kraken/download/results.py @@ -1,4 +1,4 @@ -from typing import Tuple, Type, Dict, List, Generator +from typing import Tuple, Type, Dict, List, Generator, Union from dataclasses import dataclass from ..objects import DatabaseObject @@ -14,33 +14,35 @@ class Option: class Results: def __init__(self) -> None: - self._by_index = Dict[int, DatabaseObject] = dict() + self._by_index: Dict[int, DatabaseObject] = dict() + self._page_by_index: Dict[int: Type[Page]] = dict() - def __iter__(self) -> Generator[DatabaseObject]: + def __iter__(self) -> Generator[DatabaseObject, None, None]: for option in self.formated_generator(): if isinstance(option, Option): yield option.music_object - def formated_generator(self, max_items_per_page: int = 10) -> Generator[Type[Page], Option]: + def formated_generator(self, max_items_per_page: int = 10) -> Generator[Union[Type[Page], Option], None, None]: self._by_index = dict() + self._page_by_index = dict() - def get_music_object_by_index(self, index: int) -> DatabaseObject: + def get_music_object_by_index(self, index: int) -> Tuple[Type[Page], DatabaseObject]: # if this throws a key error, either the formated generator needs to be iterated, or the option doesn't exist. - return self._by_index[index] + return self._page_by_index[index], self._by_index[index] class SearchResults(Results): def __init__( self, - pages: Tuple[Type[Page], ...] + pages: Tuple[Type[Page], ...] = None ) -> None: - super().__init() + super().__init__() - self.pages = pages + self.pages = pages or [] # this would initialize a list for every page, which I don't think I want # self.results = Dict[Type[Page], List[DatabaseObject]] = {page: [] for page in self.pages} - self.results = Dict[Type[Page], List[DatabaseObject]] = {} + self.results: Dict[Type[Page], List[DatabaseObject]] = {} def add(self, page: Type[Page], search_result: List[DatabaseObject]): """ @@ -64,6 +66,7 @@ class SearchResults(Results): for option in self.results[page]: yield Option(i, option) self._by_index[i] = option + self._page_by_index[i] = page i += 1 j += 1 @@ -73,7 +76,7 @@ class SearchResults(Results): class PageResults(Results): def __init__(self, page: Type[Page], results: List[DatabaseObject]) -> None: - super().__init() + super().__init__() self.page: Type[Page] = page self.results: List[DatabaseObject] = results @@ -87,4 +90,5 @@ class PageResults(Results): for option in self.results: yield Option(i, option) self._by_index[i] = option + self._page_by_index[i] = self.page i += 1 diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index d68463f..20c8303 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -95,8 +95,8 @@ class DatabaseObject: return Metadata() @property - def options(self) -> Options: - return Options([self]) + def options(self) -> List["DatabaseObject"]: + return [self] @property def option_string(self) -> str: diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index b253539..e984376 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -14,7 +14,7 @@ from .metadata import ( Metadata ) from .option import Options -from .parents import MainObject +from .parents import MainObject, DatabaseObject from .source import Source, SourceCollection from .target import Target from ..utils.string_processing import unify @@ -162,7 +162,7 @@ class Song(MainObject): f"feat. Artist({OPTION_STRING_DELIMITER.join(artist.name for artist in self.feature_artist_collection)})" @property - def options(self) -> Options: + def options(self) -> List[DatabaseObject]: """ Return a list of related objects including the song object, album object, main artist objects, and feature artist objects. @@ -173,7 +173,7 @@ class Song(MainObject): options.extend(self.feature_artist_collection) options.extend(self.album_collection) options.append(self) - return Options(options) + return options @property def tracksort_str(self) -> str: @@ -309,12 +309,12 @@ class Album(MainObject): f"under Label({OPTION_STRING_DELIMITER.join([label.name for label in self.label_collection])})" @property - def options(self) -> Options: + def options(self) -> List[DatabaseObject]: options = self.artist_collection.shallow_list options.append(self) options.extend(self.song_collection) - return Options(options) + return options def update_tracksort(self): """ @@ -600,11 +600,11 @@ class Artist(MainObject): f"under Label({OPTION_STRING_DELIMITER.join([label.name for label in self.label_collection])})" @property - def options(self) -> Options: + def options(self) -> List[DatabaseObject]: options = [self] options.extend(self.main_album_collection) options.extend(self.feature_song_collection) - return Options(options) + return options @property def country_string(self): @@ -704,7 +704,9 @@ class Label(MainObject): ] @property - def options(self) -> Options: + def options(self) -> List[DatabaseObject]: options = [self] options.extend(self.current_artist_collection.shallow_list) options.extend(self.album_collection.shallow_list) + + return options diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 2562f5e..2895cf0 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -166,7 +166,6 @@ class Page(): for default_query in query.default_search: for single_option in self.general_search(default_query): r.append(single_option) - self.search_result_queue.put(single_option) return r From 7743ff1b9e5170000a48da6863c6b6c1ec0c701f Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 12 Jun 2023 17:40:54 +0200 Subject: [PATCH 42/81] continued download --- documentation/shell.md | 57 ++++++++++++++++++++ src/music_kraken/cli/download/shell.py | 46 +++++++++++++++- src/music_kraken/connection/connection.py | 1 + src/music_kraken/download/page_attributes.py | 7 ++- src/music_kraken/pages/abstract.py | 2 +- 5 files changed, 106 insertions(+), 7 deletions(-) create mode 100644 documentation/shell.md diff --git a/documentation/shell.md b/documentation/shell.md new file mode 100644 index 0000000..de72743 --- /dev/null +++ b/documentation/shell.md @@ -0,0 +1,57 @@ +# Shell + +## Searching + +```mkshell +> s: {querry or url} + +# examples +> s: https://musify.club/release/some-random-release-183028492 +> s: r: #a an Artist #r some random Release +``` + +Searches for an url, or an query + +### Query Syntax + +``` +#a {artist} #r {release} #t {track} +``` + +You can escape stuff like `#` doing this: `\#` + +## Downloading + +To download something, you either need a direct link, or you need to have already searched for options + +```mkshell +> d: {option ids or direct url} + +# examples +> d: 0, 3, 4 +> d: 1 +> d: https://musify.club/release/some-random-release-183028492 +``` + +## Misc + +### Exit + +```mkshell +> q +> quit +> exit +> abort +``` + +### Current Options + +```mkshell +> . +``` + +### Previous Options + +``` +> .. +``` diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/download/shell.py index 8cb627a..adb7bfb 100644 --- a/src/music_kraken/cli/download/shell.py +++ b/src/music_kraken/cli/download/shell.py @@ -5,7 +5,7 @@ import re from ...utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX from ...utils.regex import URL_PATTERN from ...utils.string_processing import fit_to_file_system -from ...utils.support_classes import Query +from ...utils.support_classes import Query, DownloadResult from ...download.results import Results, SearchResults, Option, PageResults from ...download.page_attributes import Pages from ...pages import Page @@ -211,8 +211,15 @@ class Shell: if artist is not None: return Query(raw_query=query, music_object=artist) + + return Query(raw_query=query) def search(self, query: str): + if re.match(URL_PATTERN, query) is not None: + self.set_current_options(*self.pages.fetch_url(re.match(URL_PATTERN, query))) + self.print_current_options() + return + special_characters = "#\\" query = query + " " @@ -266,7 +273,7 @@ class Shell: try: page, music_object = self.current_results.get_music_object_by_index(index) - except IndexError: + except KeyError: print() print(f"The option {index} doesn't exist.") print() @@ -280,6 +287,41 @@ class Shell: def download(self, download_str: str, download_all: bool = False) -> bool: + to_download: List[DatabaseObject] = [] + + if re.match(URL_PATTERN, download_str) is not None: + _, music_objects = self.pages.fetch_url(re.match(URL_PATTERN, download_str)) + to_download.append(music_objects) + + else: + index: str + for index in download_str.split(", "): + if not index.strip().isdigit(): + print() + print(f"Every download thingie has to be an index, not {index}.") + print() + return False + + for index in download_str.split(", "): + to_download.append(self.current_results.get_music_object_by_index(int(index))[1]) + + print() + print("Downloading:") + for download_object in to_download: + print(download_object.option_string) + print() + + _result_map: Dict[DatabaseObject, DownloadResult] = dict() + + for database_object in to_download: + r = self.pages.download(music_object=database_object, genre=self.genre, download_all=download_all) + _result_map[database_object] = r + + for music_object, result in _result_map.items(): + print() + print(music_object.option_string) + print(result) + return False def process_input(self, input_str: str) -> bool: diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 70c77ea..02eba18 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -131,6 +131,7 @@ class Connection: accepted_response_code=accepted_response_code, url=url, timeout=timeout, + headers=headers, **kwargs ) diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index bb392f8..11598d0 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -80,11 +80,11 @@ class Pages: audio_pages = self._audio_pages_set.intersection(_page_types) for download_page in audio_pages: - return self._page_instances[download_page].download(genre=genre, download_all=download_all) + return self._page_instances[download_page].download(music_object=music_object, genre=genre, download_all=download_all) return DownloadResult(error_message=f"No audio source has been found for {music_object}.") - def fetch_url(self, url: str, stop_at_level: int = 2) -> DatabaseObject: + def fetch_url(self, url: str, stop_at_level: int = 2) -> Tuple[Type[Page], DatabaseObject]: source = Source.match_url(url, SourcePages.MANUAL) if source is None: @@ -92,5 +92,4 @@ class Pages: _actual_page = self._source_to_page[source.page_enum] - - return self._page_instances[_actual_page].fetch_object_from_source(source=source, stop_at_level=stop_at_level) \ No newline at end of file + return _actual_page, self._page_instances[_actual_page].fetch_object_from_source(source=source, stop_at_level=stop_at_level) \ No newline at end of file diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 2895cf0..d2c72b5 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -268,7 +268,7 @@ class Page(): nonlocal naming_objects for collection_name in naming_music_object.UPWARDS_COLLECTION_ATTRIBUTES: - collection: Collection = getattr(self, collection_name) + collection: Collection = getattr(naming_music_object, collection_name) if collection.empty(): continue From c869eea6b2671c94c23cb62d38b9eb18ef688415 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Mon, 12 Jun 2023 17:46:31 +0200 Subject: [PATCH 43/81] test script --- src/actual_donwload.py | 53 ++---------------------------------------- 1 file changed, 2 insertions(+), 51 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 9db40b9..1a2bcce 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -5,59 +5,10 @@ from music_kraken.utils.enums.source import SourcePages from music_kraken.objects import Song, Target, Source, Album -def search_pages(): - search = Search() - print("metadata", search.pages) - print("audio", search.audio_pages) - - print() - print(search) - - search.choose_page(pages.Musify) - - print() - print(search) - - search.choose_index(0) - print(search) - - -def direct_download(): - search = Search() - - search.search_url("https://www.metal-archives.com/bands/Ghost_Bath/3540372489") - print(search) - - search.search_url("https://musify.club/artist/ghost-bath-280348") - print(search) - - -def download_audio(): - song = Song( - source_list=[ - Source(SourcePages.MUSIFY, "https://musify.club/track/im-in-a-coffin-life-never-was-waste-of-skin-16360302") - ], - target_list=[ - Target(relative_to_music_dir=True, path="example", file="waste_of_skin_1"), - Target(relative_to_music_dir=True, path="example", file="waste_of_skin_2") - ] - ) - - pages.Musify.download_song(song) - - -def real_download(): - search = Search() - search.search_url("https://musify.club/release/children-of-the-night-2018-1079829") - search.download_chosen() - - if __name__ == "__main__": music_kraken.cli(genre="test", command_list=[ # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", # "https://musify.club/release/ghost-bath-self-loather-2021-1554266", - "#a Ghost Bath", - "1", - "4", - "ok" + "s: #a Ghost Bath", + ]) From 4b4e05239fb097810b00784cc191d4ee3dd34aae Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Mon, 12 Jun 2023 19:46:46 +0200 Subject: [PATCH 44/81] completed new cli --- src/actual_donwload.py | 17 +++++++++-------- src/music_kraken/cli/download/shell.py | 15 ++++++++++----- src/music_kraken/objects/parents.py | 5 +++++ src/music_kraken/pages/abstract.py | 17 +++++++++-------- src/music_kraken/pages/musify.py | 7 +------ 5 files changed, 34 insertions(+), 27 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 1a2bcce..bb53da2 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -1,14 +1,15 @@ import music_kraken -from music_kraken import pages -from music_kraken.download import Search -from music_kraken.utils.enums.source import SourcePages -from music_kraken.objects import Song, Target, Source, Album if __name__ == "__main__": - music_kraken.cli(genre="test", command_list=[ - # "https://musify.club/release/molchat-doma-etazhi-2018-1092949", - # "https://musify.club/release/ghost-bath-self-loather-2021-1554266", + normally_download = [ "s: #a Ghost Bath", + "1", + "d: 1, 5" + ] - ]) + direct_download = [ + "d: https://musify.club/release/crystal-f-x-2012-795181" + ] + + music_kraken.cli(genre="test", command_list=direct_download) diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/download/shell.py index adb7bfb..1db4e6d 100644 --- a/src/music_kraken/cli/download/shell.py +++ b/src/music_kraken/cli/download/shell.py @@ -178,7 +178,9 @@ class Shell: def print_current_options(self): self.page_dict = dict() - + + print() + page_count = 0 for option in self.current_results.formated_generator(max_items_per_page=self.max_displayed_options): if isinstance(option, Option): @@ -191,7 +193,9 @@ class Shell: self.page_dict[option.__name__] = option page_count += 1 - + + print() + def set_current_options(self, current_options: Results): self.current_results = current_options @@ -216,7 +220,8 @@ class Shell: def search(self, query: str): if re.match(URL_PATTERN, query) is not None: - self.set_current_options(*self.pages.fetch_url(re.match(URL_PATTERN, query))) + page, data_object = self.pages.fetch_url(query) + self.set_current_options(PageResults(page, data_object.options)) self.print_current_options() return @@ -290,7 +295,7 @@ class Shell: to_download: List[DatabaseObject] = [] if re.match(URL_PATTERN, download_str) is not None: - _, music_objects = self.pages.fetch_url(re.match(URL_PATTERN, download_str)) + _, music_objects = self.pages.fetch_url(download_str) to_download.append(music_objects) else: @@ -322,7 +327,7 @@ class Shell: print(music_object.option_string) print(result) - return False + return True def process_input(self, input_str: str) -> bool: input_str = input_str.strip() diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index 20c8303..9185a82 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -36,6 +36,11 @@ class DatabaseObject: self.build_version = -1 + def __hash__(self): + if self.dynamic: + raise TypeError("Dynamic DatabaseObjects are unhashable.") + return self.id + def __eq__(self, other) -> bool: if not isinstance(other, type(self)): return False diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index d2c72b5..cbeb46a 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -104,7 +104,7 @@ def merge_together(old_object: DatabaseObject, new_object: DatabaseObject) -> Da return old_object -class Page(): +class Page: """ This is an abstract class, laying out the functionality for every other class fetching something @@ -270,7 +270,7 @@ class Page(): for collection_name in naming_music_object.UPWARDS_COLLECTION_ATTRIBUTES: collection: Collection = getattr(naming_music_object, collection_name) - if collection.empty(): + if collection.empty: continue if collection.element_type in naming_objects: continue @@ -280,10 +280,10 @@ class Page(): fill_naming_objects(music_object) - return self._download(music_object, {}, genre, download_all) + return self._download(music_object, naming_objects, download_all) - def _download(self, music_object: DatabaseObject, naming_objects: Dict[Type[DatabaseObject], DatabaseObject], download_all: bool = False) -> list: + def _download(self, music_object: DatabaseObject, naming_objects: Dict[Union[Type[DatabaseObject], str], DatabaseObject], download_all: bool = False) -> DownloadResult: # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False if isinstance(music_object, Album): if not download_all and music_object.album_type in shared.ALBUM_TYPE_BLACKLIST: @@ -298,7 +298,7 @@ class Page(): download_result: DownloadResult = DownloadResult() for collection_name in music_object.DOWNWARDS_COLLECTION_ATTRIBUTES: - collection: Collection = getattr(self, collection_name) + collection: Collection = getattr(music_object, collection_name) sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: @@ -306,11 +306,12 @@ class Page(): return download_result - def _download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], DatabaseObject]): + def _download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], Union[DatabaseObject, str]]): name_attribute = DEFAULT_VALUES.copy() # song - name_attribute["genre"] = naming_objects["genre"] + if "genre" in naming_objects: + name_attribute["genre"] = naming_objects["genre"] name_attribute["song"] = song.title if Album in naming_objects: @@ -344,7 +345,7 @@ class Page(): file=str(random.randint(0, 999999)) ) - r = self._download_song_to_targets(source=sources[0], target=temp_target, desc=song.title) + r = self.download_song_to_target(source=sources[0], target=temp_target, desc=song.title) if not r.is_fatal_error: r.merge(self._post_process_targets(song, temp_target)) diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 34c9fe6..46c8429 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -164,7 +164,6 @@ class Musify(Page): artist_thumbnail = image_soup.get("src") return Artist( - _id=_id, name=name, source_list=source_list ) @@ -269,7 +268,6 @@ class Musify(Page): self.LOGGER.warning("got an unequal ammount than 3 small elements") return Album( - _id=_id, title=title, source_list=source_list, date=ID3Timestamp(year=year), @@ -316,7 +314,6 @@ class Musify(Page): _id = href.split("-")[-1] artist_list.append(Artist( - _id=_id, name=artist_anchor.get_text(strip=True), source_list=[artist_source] )) @@ -352,7 +349,6 @@ class Musify(Page): _id = raw_id return Song( - _id=_id, title=song_title, main_artist_list=artist_list, source_list=source_list @@ -838,7 +834,6 @@ class Musify(Page): self.LOGGER.debug("there is not even 1 footer in the album card") return Album( - _id=_id, title=name, source_list=source_list, date=timestamp, @@ -998,7 +993,7 @@ class Musify(Page): def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() - def download_song_to_targets(self, source: Source, target: Target, desc: str = None) -> DownloadResult: + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: """ https://musify.club/track/im-in-a-coffin-life-never-was-waste-of-skin-16360302 https://musify.club/track/dl/16360302/im-in-a-coffin-life-never-was-waste-of-skin.mp3 From 5156ae715be21948b4304b522ea14f7379ff2009 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:53:40 +0200 Subject: [PATCH 45/81] completed new cli --- README.md | 20 +- contribute.md | 2 + src/music_kraken/download/page_attributes.py | 6 +- src/music_kraken/not_used_anymore/__init__.py | 2 - .../not_used_anymore/fetch_audio.py | 106 ----- .../not_used_anymore/fetch_source.py | 70 ---- .../not_used_anymore/metadata/__init__.py | 7 - .../metadata/metadata_fetch.py | 345 ----------------- .../metadata/metadata_search.py | 364 ------------------ .../metadata/sources/__init__.py | 4 - .../metadata/sources/musicbrainz.py | 59 --- .../not_used_anymore/sources/__init__.py | 0 .../not_used_anymore/sources/genius.py | 172 --------- .../not_used_anymore/sources/local_files.py | 57 --- .../not_used_anymore/sources/musify.py | 181 --------- .../not_used_anymore/sources/source.py | 23 -- .../not_used_anymore/sources/youtube.py | 98 ----- src/music_kraken/pages/__init__.py | 2 + src/music_kraken/pages/musify.py | 2 - src/music_kraken/pages/preset.py | 5 + src/music_kraken/pages/youtube.py | 92 +++-- 21 files changed, 87 insertions(+), 1530 deletions(-) delete mode 100644 src/music_kraken/not_used_anymore/__init__.py delete mode 100644 src/music_kraken/not_used_anymore/fetch_audio.py delete mode 100644 src/music_kraken/not_used_anymore/fetch_source.py delete mode 100644 src/music_kraken/not_used_anymore/metadata/__init__.py delete mode 100644 src/music_kraken/not_used_anymore/metadata/metadata_fetch.py delete mode 100644 src/music_kraken/not_used_anymore/metadata/metadata_search.py delete mode 100644 src/music_kraken/not_used_anymore/metadata/sources/__init__.py delete mode 100644 src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py delete mode 100644 src/music_kraken/not_used_anymore/sources/__init__.py delete mode 100644 src/music_kraken/not_used_anymore/sources/genius.py delete mode 100644 src/music_kraken/not_used_anymore/sources/local_files.py delete mode 100644 src/music_kraken/not_used_anymore/sources/musify.py delete mode 100644 src/music_kraken/not_used_anymore/sources/source.py delete mode 100644 src/music_kraken/not_used_anymore/sources/youtube.py diff --git a/README.md b/README.md index a82d5ff..8bccd6d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Music Kraken - +music kraken logo - [Music Kraken](#music-kraken) - [Installation](#installation) @@ -52,9 +52,19 @@ If you choose to run it in WSL, make sure ` ~/.local/bin` is added to your `$PAT ## Quick-Guide -**Genre:** First, the cli asks you to input a genre you want to download to. The options it gives you (if it gives you any) are all the folders you have in the music directory. You can also just input a new one. +The **Genre** you define at the start is the folder, my programm will download the files into, AS WELL as the value of the ID3 genre field. -**What to download:** After that it prompts you for a search. Here are a couple examples how you can search: +When it drops you into the **shell** 2 main things are important: + +1. You search with `s: ` +2. You choose an option with just the index number of the option +3. You download with `d: `, where the options are comma seperated + +Trust me it WILL make sense, once you see it. + +### Query + +The syntax for the query is like really simple. ``` > #a @@ -67,9 +77,9 @@ searches for the release (album) by the artist searches for the track from the release ``` -After searching with this syntax, it prompts you with multiple results. You can either choose one of those by inputing its id `int`, or you can search for a new query. +For a more detailed guid of the downloading shell, see [here](documentation/shell.md). -After you chose either an artist, a release group, a release, or a track by its id, download it by inputting the string `ok`. My downloader will download it automatically for you. +LOVE YALL *(except nazis ;-;)* --- diff --git a/contribute.md b/contribute.md index 1750781..34ba719 100644 --- a/contribute.md +++ b/contribute.md @@ -1,3 +1,5 @@ +> IMPORTANT NOTE: heavily outdated sorryyyyy + # How to contribute I am always happy about pull requests. diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index 11598d0..ee7c18d 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -5,15 +5,17 @@ from ..objects import DatabaseObject, Source from ..utils.enums.source import SourcePages from ..utils.support_classes import Query, DownloadResult from ..utils.exception.download import UrlNotFoundException -from ..pages import Page, EncyclopaediaMetallum, Musify, INDEPENDENT_DB_OBJECTS +from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, INDEPENDENT_DB_OBJECTS ALL_PAGES: Set[Type[Page]] = { EncyclopaediaMetallum, - Musify + Musify, + YouTube, } AUDIO_PAGES: Set[Type[Page]] = { Musify, + YouTube, } SHADY_PAGES: Set[Type[Page]] = { diff --git a/src/music_kraken/not_used_anymore/__init__.py b/src/music_kraken/not_used_anymore/__init__.py deleted file mode 100644 index 139597f..0000000 --- a/src/music_kraken/not_used_anymore/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/src/music_kraken/not_used_anymore/fetch_audio.py b/src/music_kraken/not_used_anymore/fetch_audio.py deleted file mode 100644 index 1793f12..0000000 --- a/src/music_kraken/not_used_anymore/fetch_audio.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import List -import mutagen.id3 -import requests -import os.path -from mutagen.easyid3 import EasyID3 -from pydub import AudioSegment - -from ..utils.shared import * -from .sources import ( - youtube, - musify, - local_files -) -from ..database.song import ( - Song as song_object, - Target as target_object, - Source as source_object -) -from ..database.temp_database import temp_database - -logger = DOWNLOAD_LOGGER - -# maps the classes to get data from to the source name -sources = { - 'Youtube': youtube.Youtube, - 'Musify': musify.Musify -} - -""" -https://en.wikipedia.org/wiki/ID3 -https://mutagen.readthedocs.io/en/latest/user/id3.html - -# to get all valid keys -from mutagen.easyid3 import EasyID3 -print("\n".join(EasyID3.valid_keys.keys())) -print(EasyID3.valid_keys.keys()) -""" - - -class Download: - def __init__(self): - Download.fetch_audios(temp_database.get_tracks_to_download()) - - @classmethod - def fetch_audios(cls, songs: List[song_object], override_existing: bool = False): - for song in songs: - if not cls.path_stuff(song.target) and not override_existing: - cls.write_metadata(song) - continue - - is_downloaded = False - for source in song.sources: - download_success = Download.download_from_src(song, source) - - if download_success == -1: - logger.warning(f"couldn't download {song['url']} from {song['src']}") - else: - is_downloaded = True - break - - if is_downloaded: - cls.write_metadata(song) - - @classmethod - def download_from_src(cls, song: song_object, source: source_object): - if source.src not in sources: - raise ValueError(f"source {source.src} seems to not exist") - source_subclass = sources[source.src] - - return source_subclass.fetch_audio(song, source) - - @classmethod - def write_metadata(cls, song: song_object): - if not os.path.exists(song.target.file): - logger.warning(f"file {song.target.file} doesn't exist") - return False - - # only convert the file to the proper format if mutagen doesn't work with it due to time - try: - audiofile = EasyID3(song.target.file) - except mutagen.id3.ID3NoHeaderError: - AudioSegment.from_file(song.target.file).export(song.target.file, format="mp3") - audiofile = EasyID3(song.target.file) - - for key, value in song.get_metadata(): - if type(value) != list: - value = str(value) - audiofile[key] = value - - logger.info("saving") - audiofile.save(song.target.file, v1=2) - - @classmethod - def path_stuff(cls, target: target_object) -> bool: - # returns true if it should be downloaded - if os.path.exists(target.file): - logger.info(f"'{target.file}' does already exist, thus not downloading.") - return False - os.makedirs(target.path, exist_ok=True) - return True - - -if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) - s = requests.Session() - Download() diff --git a/src/music_kraken/not_used_anymore/fetch_source.py b/src/music_kraken/not_used_anymore/fetch_source.py deleted file mode 100644 index e28479c..0000000 --- a/src/music_kraken/not_used_anymore/fetch_source.py +++ /dev/null @@ -1,70 +0,0 @@ -from typing import List - -from ..utils.shared import * -from .sources import ( - youtube, - musify, - local_files -) -from ..database.song import Song as song_object -from ..database.temp_database import temp_database - -logger = URL_DOWNLOAD_LOGGER - -# maps the classes to get data from to the source name -sources = { - 'Youtube': youtube.Youtube, - 'Musify': musify.Musify -} - - -class Download: - def __init__(self) -> None: - for song in temp_database.get_tracks_without_src(): - id_ = song['id'] - if os.path.exists(song.target.file): - logger.info(f"skipping the fetching of the download links, cuz {song.target.file} already exists.") - continue - - success = False - for src in AUDIO_SOURCES: - res = Download.fetch_from_src(song, src) - if res is not None: - success = True - Download.add_url(res, src, id_) - - if not success: - logger.warning(f"Didn't find any sources for {song}") - - @classmethod - def fetch_sources(cls, songs: List[song_object], skip_existing_files: bool = False): - for song in songs: - if song.target.exists_on_disc and skip_existing_files: - logger.info(f"skipping the fetching of the download links, cuz {song.target.file} already exists.") - continue - - success = False - for src in AUDIO_SOURCES: - res = cls.fetch_from_src(song, src) - if res is not None: - success = True - cls.add_url(res, src, song.id) - - if not success: - logger.warning(f"Didn't find any sources for {song}") - - @classmethod - def fetch_from_src(cls, song, src): - if src not in sources: - raise ValueError(f"source {src} seems to not exist") - - source_subclass = sources[src] - return source_subclass.fetch_source(song) - - @classmethod - def add_url(cls, url: str, src: str, id_: str): - temp_database.set_download_data(id_, url, src) - - -if __name__ == "__main__": - download = Download() diff --git a/src/music_kraken/not_used_anymore/metadata/__init__.py b/src/music_kraken/not_used_anymore/metadata/__init__.py deleted file mode 100644 index ec153f9..0000000 --- a/src/music_kraken/not_used_anymore/metadata/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -from . import ( - metadata_search, - metadata_fetch -) - -MetadataSearch = metadata_search.Search -MetadataDownload = metadata_fetch.MetadataDownloader diff --git a/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py b/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py deleted file mode 100644 index 7f71c86..0000000 --- a/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py +++ /dev/null @@ -1,345 +0,0 @@ -from src.music_kraken.utils.shared import * -from src.music_kraken.utils.object_handeling import get_elem_from_obj, parse_music_brainz_date - -from src.music_kraken.database.temp_database import temp_database - -from typing import List -import musicbrainzngs -import logging - -# I don't know if it would be feesable to set up my own mb instance -# https://github.com/metabrainz/musicbrainz-docker - - -# IMPORTANT DOCUMENTATION WHICH CONTAINS FOR EXAMPLE THE INCLUDES -# https://python-musicbrainzngs.readthedocs.io/en/v0.7.1/api/#getting-data - -logger = METADATA_DOWNLOAD_LOGGER - - -class MetadataDownloader: - def __init__(self): - pass - - class Artist: - def __init__( - self, - musicbrainz_artistid: str, - release_groups: List = [], - new_release_groups: bool = True - ): - """ - release_groups: list - """ - self.release_groups = release_groups - - self.musicbrainz_artistid = musicbrainz_artistid - - try: - result = musicbrainzngs.get_artist_by_id(self.musicbrainz_artistid, includes=["release-groups", "releases"]) - except musicbrainzngs.musicbrainz.NetworkError: - return - artist_data = get_elem_from_obj(result, ['artist'], return_if_none={}) - - self.artist = get_elem_from_obj(artist_data, ['name']) - - self.save() - - # STARTING TO FETCH' RELEASE GROUPS. IMPORTANT: DON'T WRITE ANYTHING BESIDES THAT HERE - if not new_release_groups: - return - # sort all release groups by date and add album sort to have them in chronological order. - release_groups = artist_data['release-group-list'] - for i, release_group in enumerate(release_groups): - release_groups[i]['first-release-date'] = parse_music_brainz_date(release_group['first-release-date']) - release_groups.sort(key=lambda x: x['first-release-date']) - - for i, release_group in enumerate(release_groups): - self.release_groups.append(MetadataDownloader.ReleaseGroup( - musicbrainz_releasegroupid=release_group['id'], - artists=[self], - albumsort=i + 1 - )) - - def __str__(self): - newline = "\n" - return f"artist: \"{self.artist}\"" - - def save(self): - logger.info(f"caching {self}") - temp_database.add_artist( - musicbrainz_artistid=self.musicbrainz_artistid, - artist=self.artist - ) - - class ReleaseGroup: - def __init__( - self, - musicbrainz_releasegroupid: str, - artists=[], - albumsort: int = None, - only_download_distinct_releases: bool = True, - fetch_further: bool = True - ): - """ - split_artists: list -> if len > 1: album_artist=VariousArtists - releases: list - """ - - self.musicbrainz_releasegroupid = musicbrainz_releasegroupid - self.artists = artists - self.releases = [] - - try: - result = musicbrainzngs.get_release_group_by_id(musicbrainz_releasegroupid, - includes=["artist-credits", "releases"]) - except musicbrainzngs.musicbrainz.NetworkError: - return - release_group_data = get_elem_from_obj(result, ['release-group'], return_if_none={}) - artist_datas = get_elem_from_obj(release_group_data, ['artist-credit'], return_if_none={}) - release_datas = get_elem_from_obj(release_group_data, ['release-list'], return_if_none={}) - - # only for printing the release - self.name = get_elem_from_obj(release_group_data, ['title']) - - for artist_data in artist_datas: - artist_id = get_elem_from_obj(artist_data, ['artist', 'id']) - if artist_id is None: - continue - self.append_artist(artist_id) - self.albumartist = "Various Artists" if len(self.artists) > 1 else self.artists[0].artist - self.album_artist_id = None if self.albumartist == "Various Artists" else self.artists[ - 0].musicbrainz_artistid - - self.albumsort = albumsort - self.musicbrainz_albumtype = get_elem_from_obj(release_group_data, ['primary-type']) - self.compilation = "1" if self.musicbrainz_albumtype == "Compilation" else None - - self.save() - - if not fetch_further: - return - - if only_download_distinct_releases: - self.append_distinct_releases(release_datas) - else: - self.append_all_releases(release_datas) - - def __str__(self): - return f"release group: \"{self.name}\"" - - def save(self): - logger.info(f"caching {self}") - temp_database.add_release_group( - musicbrainz_releasegroupid=self.musicbrainz_releasegroupid, - artist_ids=[artist.musicbrainz_artistid for artist in self.artists], - albumartist=self.albumartist, - albumsort=self.albumsort, - musicbrainz_albumtype=self.musicbrainz_albumtype, - compilation=self.compilation, - album_artist_id=self.album_artist_id - ) - - def append_artist(self, artist_id: str): - for existing_artist in self.artists: - if artist_id == existing_artist.musicbrainz_artistid: - return existing_artist - new_artist = MetadataDownloader.Artist(artist_id, release_groups=[self], - new_release_groups=False) - self.artists.append(new_artist) - return new_artist - - def append_release(self, release_data: dict): - musicbrainz_albumid = get_elem_from_obj(release_data, ['id']) - if musicbrainz_albumid is None: - return - self.releases.append( - MetadataDownloader.Release(musicbrainz_albumid, release_group=self)) - - def append_distinct_releases(self, release_datas: List[dict]): - titles = {} - - for release_data in release_datas: - title = get_elem_from_obj(release_data, ['title']) - if title is None: - continue - titles[title] = release_data - - for key in titles: - self.append_release(titles[key]) - - def append_all_releases(self, release_datas: List[dict]): - for release_data in release_datas: - self.append_release(release_data) - - class Release: - def __init__( - self, - musicbrainz_albumid: str, - release_group=None, - fetch_furter: bool = True - ): - """ - release_group: ReleaseGroup - tracks: list - """ - self.musicbrainz_albumid = musicbrainz_albumid - self.release_group = release_group - self.tracklist = [] - - try: - result = musicbrainzngs.get_release_by_id(self.musicbrainz_albumid, - includes=["recordings", "labels", "release-groups"]) - except musicbrainzngs.musicbrainz.NetworkError: - return - release_data = get_elem_from_obj(result, ['release'], return_if_none={}) - label_data = get_elem_from_obj(release_data, ['label-info-list'], return_if_none={}) - recording_datas = get_elem_from_obj(release_data, ['medium-list', 0, 'track-list'], return_if_none=[]) - release_group_data = get_elem_from_obj(release_data, ['release-group'], return_if_none={}) - if self.release_group is None: - self.release_group = MetadataDownloader.ReleaseGroup( - musicbrainz_releasegroupid=get_elem_from_obj( - release_group_data, ['id']), - fetch_further=False) - - self.title = get_elem_from_obj(release_data, ['title']) - self.copyright = get_elem_from_obj(label_data, [0, 'label', 'name']) - - self.album_status = get_elem_from_obj(release_data, ['status']) - self.language = get_elem_from_obj(release_data, ['text-representation', 'language']) - self.year = get_elem_from_obj(release_data, ['date'], lambda x: x.split("-")[0]) - self.date = get_elem_from_obj(release_data, ['date']) - self.country = get_elem_from_obj(release_data, ['country']) - self.barcode = get_elem_from_obj(release_data, ['barcode']) - - self.save() - if fetch_furter: - self.append_recordings(recording_datas) - - def __str__(self): - return f"release: {self.title} ©{self.copyright} {self.album_status}" - - def save(self): - logger.info(f"caching {self}") - temp_database.add_release( - musicbrainz_albumid=self.musicbrainz_albumid, - release_group_id=self.release_group.musicbrainz_releasegroupid, - title=self.title, - copyright_=self.copyright, - album_status=self.album_status, - language=self.language, - year=self.year, - date=self.date, - country=self.country, - barcode=self.barcode - ) - - def append_recordings(self, recording_datas: dict): - for i, recording_data in enumerate(recording_datas): - musicbrainz_releasetrackid = get_elem_from_obj(recording_data, ['recording', 'id']) - if musicbrainz_releasetrackid is None: - continue - - self.tracklist.append( - MetadataDownloader.Track(musicbrainz_releasetrackid, self, - track_number=str(i + 1))) - - class Track: - def __init__( - self, - musicbrainz_releasetrackid: str, - release=None, - track_number: str = None - ): - """ - release: Release - feature_artists: list - """ - - self.musicbrainz_releasetrackid = musicbrainz_releasetrackid - self.release = release - self.artists = [] - - self.track_number = track_number - - try: - result = musicbrainzngs.get_recording_by_id(self.musicbrainz_releasetrackid, - includes=["artists", "releases", "recording-rels", "isrcs", - "work-level-rels"]) - except musicbrainzngs.musicbrainz.NetworkError: - return - recording_data = result['recording'] - release_data = get_elem_from_obj(recording_data, ['release-list', -1]) - if self.release is None: - self.release = MetadataDownloader.Release(get_elem_from_obj(release_data, ['id']), fetch_furter=False) - - for artist_data in get_elem_from_obj(recording_data, ['artist-credit'], return_if_none=[]): - self.append_artist(get_elem_from_obj(artist_data, ['artist', 'id'])) - - self.isrc = get_elem_from_obj(recording_data, ['isrc-list', 0]) - self.title = recording_data['title'] - - self.lenth = get_elem_from_obj(recording_data, ['length']) - - self.save() - - def __str__(self): - return f"track: \"{self.title}\" {self.isrc or ''}" - - def save(self): - logger.info(f"caching {self}") - - temp_database.add_track( - musicbrainz_releasetrackid=self.musicbrainz_releasetrackid, - musicbrainz_albumid=self.release.musicbrainz_albumid, - feature_aritsts=[artist.musicbrainz_artistid for artist in self.artists], - tracknumber=self.track_number, - track=self.title, - isrc=self.isrc, - length=int(self.lenth) - ) - - def append_artist(self, artist_id: str): - if artist_id is None: - return - - for existing_artist in self.artists: - if artist_id == existing_artist.musicbrainz_artistid: - return existing_artist - new_artist = MetadataDownloader.Artist(artist_id, new_release_groups=False) - self.artists.append(new_artist) - return new_artist - - def download(self, option: dict): - type_ = option['type'] - mb_id = option['id'] - - if type_ == "artist": - return self.Artist(mb_id) - if type_ == "release_group": - return self.ReleaseGroup(mb_id) - if type_ == "release": - return self.Release(mb_id) - if type_ == "recording": - return self.Track(mb_id) - - logger.error(f"download type {type_} doesn't exists :(") - - - -if __name__ == "__main__": - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[ - logging.FileHandler(os.path.join(TEMP_DIR, LOG_FILE)), - logging.StreamHandler() - ] - ) - - downloader = MetadataDownloader() - - downloader.download({'id': 'd2006339-9e98-4624-a386-d503328eb854', 'type': 'recording'}) - downloader.download({'id': 'cdd16860-35fd-46af-bd8c-5de7b15ebc31', 'type': 'release'}) - # download({'id': '4b9af532-ef7e-42ab-8b26-c466327cb5e0', 'type': 'release'}) - #download({'id': 'c24ed9e7-6df9-44de-8570-975f1a5a75d1', 'type': 'track'}) diff --git a/src/music_kraken/not_used_anymore/metadata/metadata_search.py b/src/music_kraken/not_used_anymore/metadata/metadata_search.py deleted file mode 100644 index bae25e4..0000000 --- a/src/music_kraken/not_used_anymore/metadata/metadata_search.py +++ /dev/null @@ -1,364 +0,0 @@ -from typing import List -import musicbrainzngs - -from src.music_kraken.utils.shared import * -from src.music_kraken.utils.object_handeling import get_elem_from_obj, parse_music_brainz_date - -logger = SEARCH_LOGGER - -MAX_PARAMETERS = 3 -OPTION_TYPES = ['artist', 'release_group', 'release', 'recording'] - - -class Option: - def __init__(self, type_: str, id_: str, name: str, additional_info: str = "") -> None: - # print(type_, id_, name) - if type_ not in OPTION_TYPES: - raise ValueError(f"type: {type_} doesn't exist. Legal Values: {OPTION_TYPES}") - self.type = type_ - self.name = name - self.id = id_ - - self.additional_info = additional_info - - def __getitem__(self, item): - map_ = { - "id": self.id, - "type": self.type, - "kind": self.type, - "name": self.name - } - return map_[item] - - def __repr__(self) -> str: - type_repr = { - 'artist': 'artist\t\t', - 'release_group': 'release group\t', - 'release': 'release\t\t', - 'recording': 'recording\t' - } - return f"{type_repr[self.type]}: \"{self.name}\"{self.additional_info}" - - -class MultipleOptions: - def __init__(self, option_list: List[Option]) -> None: - self.option_list = option_list - - def __repr__(self) -> str: - return "\n".join([f"{str(i).zfill(2)}) {choice.__repr__()}" for i, choice in enumerate(self.option_list)]) - - -class Search: - def __init__(self) -> None: - self.options_history = [] - self.current_option: Option - - def append_new_choices(self, new_choices: List[Option]) -> MultipleOptions: - self.options_history.append(new_choices) - return MultipleOptions(new_choices) - - def get_previous_options(self): - self.options_history.pop(-1) - return MultipleOptions(self.options_history[-1]) - - @staticmethod - def fetch_new_options_from_artist(artist: Option): - """ - returning list of artist and every release group - """ - result = musicbrainzngs.get_artist_by_id(artist.id, includes=["release-groups", "releases"]) - artist_data = get_elem_from_obj(result, ['artist'], return_if_none={}) - - result = [artist] - - # sort all release groups by date and add album sort to have them in chronological order. - release_group_list = artist_data['release-group-list'] - for i, release_group in enumerate(release_group_list): - release_group_list[i]['first-release-date'] = parse_music_brainz_date(release_group['first-release-date']) - release_group_list.sort(key=lambda x: x['first-release-date']) - release_group_list = [Option("release_group", get_elem_from_obj(release_group_, ['id']), - get_elem_from_obj(release_group_, ['title']), - additional_info=f" ({get_elem_from_obj(release_group_, ['type'])}) from {get_elem_from_obj(release_group_, ['first-release-date'])}") - for release_group_ in release_group_list] - - result.extend(release_group_list) - return result - - @staticmethod - def fetch_new_options_from_release_group(release_group: Option): - """ - returning list including the artists, the releases and the tracklist of the first release - """ - results = [] - - result = musicbrainzngs.get_release_group_by_id(release_group.id, - includes=["artist-credits", "releases"]) - release_group_data = get_elem_from_obj(result, ['release-group'], return_if_none={}) - artist_datas = get_elem_from_obj(release_group_data, ['artist-credit'], return_if_none={}) - release_datas = get_elem_from_obj(release_group_data, ['release-list'], return_if_none={}) - - # appending all the artists to results - for artist_data in artist_datas: - results.append(Option('artist', get_elem_from_obj(artist_data, ['artist', 'id']), - get_elem_from_obj(artist_data, ['artist', 'name']))) - - # appending initial release group - results.append(release_group) - - # appending all releases - first_release = None - for i, release_data in enumerate(release_datas): - results.append( - Option('release', get_elem_from_obj(release_data, ['id']), get_elem_from_obj(release_data, ['title']), - additional_info=f" ({get_elem_from_obj(release_data, ['status'])})")) - if i == 0: - first_release = results[-1] - - # append tracklist of first release - if first_release is not None: - results.extend(Search.fetch_new_options_from_release(first_release, only_tracklist=True)) - - return results - - @staticmethod - def fetch_new_options_from_release(release: Option, only_tracklist: bool = False): - """ - artists - release group - release - tracklist - """ - results = [] - result = musicbrainzngs.get_release_by_id(release.id, - includes=["recordings", "labels", "release-groups", "artist-credits"]) - release_data = get_elem_from_obj(result, ['release'], return_if_none={}) - label_data = get_elem_from_obj(release_data, ['label-info-list'], return_if_none={}) - recording_datas = get_elem_from_obj(release_data, ['medium-list', 0, 'track-list'], return_if_none=[]) - release_group_data = get_elem_from_obj(release_data, ['release-group'], return_if_none={}) - artist_datas = get_elem_from_obj(release_data, ['artist-credit'], return_if_none={}) - - # appending all the artists to results - for artist_data in artist_datas: - results.append(Option('artist', get_elem_from_obj(artist_data, ['artist', 'id']), - get_elem_from_obj(artist_data, ['artist', 'name']))) - - # appending the according release group - results.append(Option("release_group", get_elem_from_obj(release_group_data, ['id']), - get_elem_from_obj(release_group_data, ['title']), - additional_info=f" ({get_elem_from_obj(release_group_data, ['type'])}) from {get_elem_from_obj(release_group_data, ['first-release-date'])}")) - - # appending the release - results.append(release) - - # appending the tracklist, but first putting it in a list, in case of only_tracklist being True to - # return this instead - tracklist = [] - for i, recording_data in enumerate(recording_datas): - recording_data = recording_data['recording'] - tracklist.append(Option('recording', get_elem_from_obj(recording_data, ['id']), - get_elem_from_obj(recording_data, ['title']), - f" ({get_elem_from_obj(recording_data, ['length'])}) from {get_elem_from_obj(recording_data, ['artist-credit-phrase'])}")) - - if only_tracklist: - return tracklist - results.extend(tracklist) - return results - - @staticmethod - def fetch_new_options_from_record(recording: Option): - """ - artists, release, record - """ - results = [] - - result = musicbrainzngs.get_recording_by_id(recording.id, includes=["artists", "releases"]) - recording_data = result['recording'] - release_datas = get_elem_from_obj(recording_data, ['release-list']) - artist_datas = get_elem_from_obj(recording_data, ['artist-credit'], return_if_none={}) - - # appending all the artists to results - for artist_data in artist_datas: - results.append(Option('artist', get_elem_from_obj(artist_data, ['artist', 'id']), - get_elem_from_obj(artist_data, ['artist', 'name']))) - - # appending all releases - for i, release_data in enumerate(release_datas): - results.append( - Option('release', get_elem_from_obj(release_data, ['id']), get_elem_from_obj(release_data, ['title']), - additional_info=f" ({get_elem_from_obj(release_data, ['status'])})")) - - results.append(recording) - - return results - - def fetch_new_options(self) -> MultipleOptions: - if self.current_option is None: - return -1 - - result = [] - if self.current_option.type == 'artist': - result = self.fetch_new_options_from_artist(self.current_option) - elif self.current_option.type == 'release_group': - result = self.fetch_new_options_from_release_group(self.current_option) - elif self.current_option.type == 'release': - result = self.fetch_new_options_from_release(self.current_option) - elif self.current_option.type == 'recording': - result = self.fetch_new_options_from_record(self.current_option) - - return self.append_new_choices(result) - - def choose(self, index: int) -> MultipleOptions: - if len(self.options_history) == 0: - logging.error("initial query neaded before choosing") - return MultipleOptions([]) - - latest_options = self.options_history[-1] - if index >= len(latest_options): - logging.error("index outside of options") - return MultipleOptions([]) - - self.current_option = latest_options[index] - return self.fetch_new_options() - - @staticmethod - def search_recording_from_text(artist: str = None, release_group: str = None, recording: str = None, - query: str = None): - result = musicbrainzngs.search_recordings(artist=artist, release=release_group, recording=recording, - query=query) - recording_list = get_elem_from_obj(result, ['recording-list'], return_if_none=[]) - - resulting_options = [ - Option("recording", get_elem_from_obj(recording_, ['id']), get_elem_from_obj(recording_, ['title']), - additional_info=f" of {get_elem_from_obj(recording_, ['release-list', 0, 'title'])} by {get_elem_from_obj(recording_, ['artist-credit', 0, 'name'])}") - for recording_ in recording_list] - return resulting_options - - @staticmethod - def search_release_group_from_text(artist: str = None, release_group: str = None, query: str = None): - result = musicbrainzngs.search_release_groups(artist=artist, releasegroup=release_group, query=query) - release_group_list = get_elem_from_obj(result, ['release-group-list'], return_if_none=[]) - - resulting_options = [Option("release_group", get_elem_from_obj(release_group_, ['id']), - get_elem_from_obj(release_group_, ['title']), - additional_info=f" by {get_elem_from_obj(release_group_, ['artist-credit', 0, 'name'])}") - for release_group_ in release_group_list] - return resulting_options - - @staticmethod - def search_artist_from_text(artist: str = None, query: str = None): - result = musicbrainzngs.search_artists(artist=artist, query=query) - artist_list = get_elem_from_obj(result, ['artist-list'], return_if_none=[]) - - resulting_options = [Option("artist", get_elem_from_obj(artist_, ['id']), get_elem_from_obj(artist_, ['name']), - additional_info=f": {', '.join([i['name'] for i in get_elem_from_obj(artist_, ['tag-list'], return_if_none=[])])}") - for artist_ in artist_list] - return resulting_options - - def search_from_text(self, artist: str = None, release_group: str = None, recording: str = None) -> MultipleOptions: - logger.info( - f"searching specified artist: \"{artist}\", release group: \"{release_group}\", recording: \"{recording}\"") - if artist is None and release_group is None and recording is None: - logger.error("either artist, release group or recording has to be set") - return MultipleOptions([]) - - if recording is not None: - logger.info("search for recording") - results = self.search_recording_from_text(artist=artist, release_group=release_group, recording=recording) - elif release_group is not None: - logger.info("search for release group") - results = self.search_release_group_from_text(artist=artist, release_group=release_group) - else: - logger.info("search for artist") - results = self.search_artist_from_text(artist=artist) - - return self.append_new_choices(results) - - def search_from_text_unspecified(self, query: str) -> MultipleOptions: - logger.info(f"searching unspecified: \"{query}\"") - - results = [] - results.extend(self.search_artist_from_text(query=query)) - results.extend(self.search_release_group_from_text(query=query)) - results.extend(self.search_recording_from_text(query=query)) - - return self.append_new_choices(results) - - def search_from_query(self, query: str) -> MultipleOptions: - if query is None: - return MultipleOptions([]) - """ - mit # wird ein neuer Parameter gestartet - der Buchstabe dahinter legt die Art des Parameters fest - "#a Psychonaut 4 #r Tired, Numb and #t Drop by Drop" - if no # is in the query it gets treated as "unspecified query" - :param query: - :return: - """ - - if not '#' in query: - return self.search_from_text_unspecified(query) - - artist = None - release_group = None - recording = None - - query = query.strip() - parameters = query.split('#') - parameters.remove('') - - if len(parameters) > MAX_PARAMETERS: - raise ValueError(f"too many parameters. Only {MAX_PARAMETERS} are allowed") - - for parameter in parameters: - splitted = parameter.split(" ") - type_ = splitted[0] - input_ = " ".join(splitted[1:]).strip() - - if type_ == "a": - artist = input_ - continue - if type_ == "r": - release_group = input_ - continue - if type_ == "t": - recording = input_ - continue - - return self.search_from_text(artist=artist, release_group=release_group, recording=recording) - - -def automated_demo(): - search = Search() - search.search_from_text(artist="I Prevail") - - # choose an artist - search.choose(0) - # choose a release group - search.choose(9) - # choose a release - search.choose(2) - # choose a recording - search.choose(4) - - -def interactive_demo(): - search = Search() - while True: - input_ = input( - "q to quit, .. for previous options, int for this element, str to search for query, ok to download: ") - input_.strip() - if input_.lower() == "ok": - break - if input_.lower() == "q": - break - if input_.lower() == "..": - search.get_previous_options() - continue - if input_.isdigit(): - search.choose(int(input_)) - continue - search.search_from_query(input_) - - -if __name__ == "__main__": - interactive_demo() diff --git a/src/music_kraken/not_used_anymore/metadata/sources/__init__.py b/src/music_kraken/not_used_anymore/metadata/sources/__init__.py deleted file mode 100644 index e9dba5f..0000000 --- a/src/music_kraken/not_used_anymore/metadata/sources/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from enum import Enum - -class Providers(Enum): - musicbrainz = "musicbrainz" diff --git a/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py b/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py deleted file mode 100644 index b329bcc..0000000 --- a/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py +++ /dev/null @@ -1,59 +0,0 @@ -from typing import List -import musicbrainzngs - -from src.music_kraken.database import ( - Artist, - Album, - Song -) -from src.music_kraken.utils.object_handeling import ( - get_elem_from_obj -) - - -def get_artist(flat: bool = False) -> Artist: - # getting the flat artist - artist_object = Artist() - if flat: - return artist_object - # get additional stuff like discography - return artist_object - - -def get_album(flat: bool = False) -> Album: - # getting the flat album object - album_object = Album() - if flat: - return album_object - # get additional stuff like tracklist - return album_object - - -def get_song(mb_id: str, flat: bool = False) -> Song: - # getting the flat song object - try: - result = musicbrainzngs.get_recording_by_id(mb_id, - includes=["artists", "releases", "recording-rels", "isrcs", - "work-level-rels"]) - except musicbrainzngs.musicbrainz.NetworkError: - return - - recording_data = result['recording'] - - song_object = Song( - mb_id=mb_id, - title=recording_data['title'], - length=get_elem_from_obj(recording_data, ['length']), - isrc=get_elem_from_obj(recording_data, ['isrc-list', 0]) - ) - if flat: - return song_object - - # fetch additional stuff - artist_data_list = get_elem_from_obj(recording_data, ['artist-credit'], return_if_none=[]) - for artist_data in artist_data_list: - mb_artist_id = get_elem_from_obj(artist_data, ['artist', 'id']) - - release_data = get_elem_from_obj(recording_data, ['release-list', -1]) - mb_release_id = get_elem_from_obj(release_data, ['id']) - return song_object diff --git a/src/music_kraken/not_used_anymore/sources/__init__.py b/src/music_kraken/not_used_anymore/sources/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/music_kraken/not_used_anymore/sources/genius.py b/src/music_kraken/not_used_anymore/sources/genius.py deleted file mode 100644 index 798b30c..0000000 --- a/src/music_kraken/not_used_anymore/sources/genius.py +++ /dev/null @@ -1,172 +0,0 @@ -import requests -from typing import List -from bs4 import BeautifulSoup -import pycountry - -from src.music_kraken.database import ( - Lyrics, - Song, - Artist -) -from src.music_kraken.utils.shared import * -from src.music_kraken.utils import phonetic_compares -from src.music_kraken.utils.object_handeling import get_elem_from_obj - -TIMEOUT = 10 - -# search doesn't support isrc -# https://genius.com/api/search/multi?q=I Prevail - Breaking Down -# https://genius.com/api/songs/6192944 -# https://docs.genius.com/ - -session = requests.Session() -session.headers = { - "Connection": "keep-alive", - "Referer": "https://genius.com/search/embed" -} -session.proxies = proxies - -logger = GENIUS_LOGGER - - -class LyricsSong: - def __init__(self, raw_data: dict, desirered_data: dict): - self.raw_data = raw_data - self.desired_data = desirered_data - - song_data = get_elem_from_obj(self.raw_data, ['result'], return_if_none={}) - self.id = get_elem_from_obj(song_data, ['id']) - self.artist = get_elem_from_obj(song_data, ['primary_artist', 'name']) - self.title = get_elem_from_obj(song_data, ['title']) - - lang_code = get_elem_from_obj(song_data, ['language']) or "en" - self.language = pycountry.languages.get(alpha_2=lang_code) - self.lang = self.language.alpha_3 - self.url = get_elem_from_obj(song_data, ['url']) - - # maybe could be implemented - self.lyricist: str - - if get_elem_from_obj(song_data, ['lyrics_state']) != "complete": - logger.warning( - f"lyrics state of {self.title} by {self.artist} is not complete but {get_elem_from_obj(song_data, ['lyrics_state'])}") - - self.valid = self.is_valid() - if self.valid: - logger.info(f"found lyrics for \"{self.__repr__()}\"") - else: - return - - self.lyrics = self.fetch_lyrics() - if self.lyrics is None: - self.valid = False - - def is_valid(self) -> bool: - title_match, title_distance = phonetic_compares.match_titles(self.title, self.desired_data['track']) - artist_match, artist_distance = phonetic_compares.match_artists(self.desired_data['artist'], self.artist) - - return not title_match and not artist_match - - def __repr__(self) -> str: - return f"{self.title} by {self.artist} ({self.url})" - - def fetch_lyrics(self) -> str | None: - if not self.valid: - logger.warning(f"{self.__repr__()} is invalid but the lyrics still get fetched. Something could be wrong.") - - try: - r = session.get(self.url, timeout=TIMEOUT) - except requests.exceptions.Timeout: - logger.warning(f"{self.url} timed out after {TIMEOUT} seconds") - return None - if r.status_code != 200: - logger.warning(f"{r.url} returned {r.status_code}:\n{r.content}") - return None - - soup = BeautifulSoup(r.content, "html.parser") - lyrics_soups = soup.find_all('div', {'data-lyrics-container': "true"}) - if len(lyrics_soups) == 0: - logger.warning(f"didn't found lyrics on {self.url}") - return None - # if len(lyrics_soups) != 1: - # logger.warning(f"number of lyrics_soups doesn't equals 1, but {len(lyrics_soups)} on {self.url}") - - lyrics = "\n".join([lyrics_soup.getText(separator="\n", strip=True) for lyrics_soup in lyrics_soups]) - - #
With the soundle - self.lyrics = lyrics - return lyrics - - def get_lyrics_object(self) -> Lyrics | None: - if self.lyrics is None: - return None - return Lyrics(text=self.lyrics, language=self.lang or "en") - - lyrics_object = property(fget=get_lyrics_object) - - -def process_multiple_songs(song_datas: list, desired_data: dict) -> List[LyricsSong]: - all_songs = [LyricsSong(song_data, desired_data) for song_data in song_datas] - return all_songs - - -def search_song_list(artist: str, track: str) -> List[LyricsSong]: - endpoint = "https://genius.com/api/search/multi?q=" - url = f"{endpoint}{artist} - {track}" - logging.info(f"requesting {url}") - - desired_data = { - 'artist': artist, - 'track': track - } - - try: - r = session.get(url, timeout=TIMEOUT) - except requests.exceptions.Timeout: - logger.warning(f"{url} timed out after {TIMEOUT} seconds") - return [] - if r.status_code != 200: - logging.warning(f"{r.url} returned {r.status_code}:\n{r.content}") - return [] - content = r.json() - if get_elem_from_obj(content, ['meta', 'status']) != 200: - logging.warning(f"{r.url} returned {get_elem_from_obj(content, ['meta', 'status'])}:\n{content}") - return [] - - sections = get_elem_from_obj(content, ['response', 'sections']) - for section in sections: - section_type = get_elem_from_obj(section, ['type']) - if section_type == "song": - return process_multiple_songs(get_elem_from_obj(section, ['hits'], return_if_none=[]), desired_data) - - return [] - - -def fetch_lyrics_from_artist(song: Song, artist: Artist) -> List[Lyrics]: - lyrics_list: List[Lyrics] = [] - lyrics_song_list = search_song_list(artist.name, song.title) - - for lyrics_song in lyrics_song_list: - if lyrics_song.valid: - lyrics_list.append(lyrics_song.lyrics_object) - - return lyrics_list - - -def fetch_lyrics(song: Song) -> List[Lyrics]: - lyrics: List[Lyrics] = [] - - for artist in song.artists: - lyrics.extend(fetch_lyrics_from_artist(song, artist)) - - return lyrics - - -""" -if __name__ == "__main__": - logging.basicConfig(level=logging.DEBUG) - - songs = search("Zombiez", "WALL OF Z") - for song in songs: - print(song) -""" diff --git a/src/music_kraken/not_used_anymore/sources/local_files.py b/src/music_kraken/not_used_anymore/sources/local_files.py deleted file mode 100644 index 358d413..0000000 --- a/src/music_kraken/not_used_anymore/sources/local_files.py +++ /dev/null @@ -1,57 +0,0 @@ -import os - -from ...utils.shared import * -from ...utils import phonetic_compares - - -def is_valid(a1, a2, t1, t2) -> bool: - title_match, title_distance = phonetic_compares.match_titles(t1, t2) - artist_match, artist_distance = phonetic_compares.match_artists(a1, a2) - - return not title_match and not artist_match - - -def get_metadata(file): - artist = None - title = None - - audiofile = EasyID3(file) - artist = audiofile['artist'] - title = audiofile['title'] - - return artist, title - - -def check_for_song(folder, artists, title): - if not os.path.exists(folder): - return False - files = [os.path.join(folder, i) for i in os.listdir(folder)] - - for file in files: - artists_, title_ = get_metadata(file) - if is_valid(artists, artists_, title, title_): - return True - return False - - -def get_path(row): - title = row['title'] - artists = row['artists'] - path_ = os.path.join(MUSIC_DIR, row['path']) - - print(artists, title, path_) - check_for_song(path_, artists, title) - - return None - - -if __name__ == "__main__": - row = {'artists': ['Psychonaut 4'], 'id': '6b40186b-6678-4328-a4b8-eb7c9806a9fb', 'tracknumber': None, - 'titlesort ': None, 'musicbrainz_releasetrackid': '6b40186b-6678-4328-a4b8-eb7c9806a9fb', - 'musicbrainz_albumid': '0d229a02-74f6-4c77-8c20-6612295870ae', 'title': 'Sweet Decadance', 'isrc': None, - 'album': 'Neurasthenia', 'copyright': 'Talheim Records', 'album_status': 'Official', 'language': 'eng', - 'year': '2016', 'date': '2016-10-07', 'country': 'AT', 'barcode': None, 'albumartist': 'Psychonaut 4', - 'albumsort': None, 'musicbrainz_albumtype': 'Album', 'compilation': None, - 'album_artist_id': 'c0c720b5-012f-4204-a472-981403f37b12', 'path': 'dsbm/Psychonaut 4/Neurasthenia', - 'file': 'dsbm/Psychonaut 4/Neurasthenia/Sweet Decadance.mp3', 'genre': 'dsbm', 'url': None, 'src': None} - print(get_path(row)) diff --git a/src/music_kraken/not_used_anymore/sources/musify.py b/src/music_kraken/not_used_anymore/sources/musify.py deleted file mode 100644 index bc8851b..0000000 --- a/src/music_kraken/not_used_anymore/sources/musify.py +++ /dev/null @@ -1,181 +0,0 @@ -import time - -import requests -import bs4 - -from ...utils.shared import * -from ...utils import phonetic_compares - -from .source import AudioSource -from ...database import song as song_objects - - -TRIES = 5 -TIMEOUT = 10 - -logger = MUSIFY_LOGGER - -session = requests.Session() -session.headers = { - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:106.0) Gecko/20100101 Firefox/106.0", - "Connection": "keep-alive", - "Referer": "https://musify.club/" -} -session.proxies = proxies - - -class Musify(AudioSource): - @classmethod - def fetch_source(cls, song: dict) -> str | None: - super().fetch_source(song) - - title = song.title - artists = song.get_artist_names() - - # trying to get a download link via the autocomplete api - for artist in artists: - url = cls.fetch_source_from_autocomplete(title=title, artist=artist) - if url is not None: - logger.info(f"found download link {url}") - return url - - # trying to get a download link via the html of the direct search page - for artist in artists: - url = cls.fetch_source_from_search(title=title, artist=artist) - if url is not None: - logger.info(f"found download link {url}") - return url - - logger.warning(f"Didn't find the audio on {cls.__name__}") - - @classmethod - def get_download_link(cls, track_url: str) -> str | None: - # https://musify.club/track/dl/18567672/rauw-alejandro-te-felicito-feat-shakira.mp3 - # /track/sundenklang-wenn-mein-herz-schreit-3883217' - - file_ = track_url.split("/")[-1] - if len(file_) == 0: - return None - musify_id = file_.split("-")[-1] - musify_name = "-".join(file_.split("-")[:-1]) - - return f"https://musify.club/track/dl/{musify_id}/{musify_name}.mp3" - - @classmethod - def fetch_source_from_autocomplete(cls, title: str, artist: str) -> str | None: - url = f"https://musify.club/search/suggestions?term={artist} - {title}" - - try: - logger.info(f"calling {url}") - r = session.get(url=url) - except requests.exceptions.ConnectionError: - logger.info("connection error occurred") - return None - if r.status_code == 200: - autocomplete = r.json() - for song in autocomplete: - if artist in song['label'] and "/track" in song['url']: - return cls.get_download_link(song['url']) - - return None - - @classmethod - def get_soup_of_search(cls, query: str, trie=0) -> bs4.BeautifulSoup | None: - url = f"https://musify.club/search?searchText={query}" - logger.debug(f"Trying to get soup from {url}") - try: - r = session.get(url, timeout=15) - except requests.exceptions.Timeout: - return None - if r.status_code != 200: - if r.status_code in [503] and trie < TRIES: - logging.warning(f"youtube blocked downloading. ({trie}-{TRIES})") - logging.warning(f"retrying in {TIMEOUT} seconds again") - time.sleep(TIMEOUT) - return cls.get_soup_of_search(query, trie=trie + 1) - - logging.warning("too many tries, returning") - return None - return bs4.BeautifulSoup(r.content, features="html.parser") - - @classmethod - def fetch_source_from_search(cls, title: str, artist: str) -> str | None: - query: str = f"{artist[0]} - {title}" - search_soup = cls.get_soup_of_search(query=query) - if search_soup is None: - return None - - # get the soup of the container with all track results - tracklist_container_soup = search_soup.find_all("div", {"class": "playlist"}) - if len(tracklist_container_soup) == 0: - return None - if len(tracklist_container_soup) != 1: - logger.warning("HTML Layout of https://musify.club changed. (or bug)") - tracklist_container_soup = tracklist_container_soup[0] - - tracklist_soup = tracklist_container_soup.find_all("div", {"class": "playlist__details"}) - - def parse_track_soup(_track_soup): - anchor_soups = _track_soup.find_all("a") - artist_ = anchor_soups[0].text.strip() - track_ = anchor_soups[1].text.strip() - url_ = anchor_soups[1]['href'] - return artist_, track_, url_ - - # check each track in the container, if they match - for track_soup in tracklist_soup: - artist_option, title_option, track_url = parse_track_soup(track_soup) - - title_match, title_distance = phonetic_compares.match_titles(title, title_option) - artist_match, artist_distance = phonetic_compares.match_artists(artist, artist_option) - - logging.debug(f"{(title, title_option, title_match, title_distance)}") - logging.debug(f"{(artist, artist_option, artist_match, artist_distance)}") - - if not title_match and not artist_match: - return cls.get_download_link(track_url) - - return None - - @classmethod - def download_from_musify(cls, target: song_objects.Target, url): - # returns if target hasn't been set - if target.path is None or target.file is None: - logger.warning(f"target hasn't been set. Can't download. Most likely a bug.") - return False - - # download the audio data - logger.info(f"downloading: '{url}'") - try: - r = session.get(url, timeout=TIMEOUT) - except requests.exceptions.ConnectionError: - return False - except requests.exceptions.ReadTimeout: - logger.warning(f"musify server didn't respond after {TIMEOUT} seconds") - return False - if r.status_code != 200: - if r.status_code == 404: - logger.warning(f"{r.url} was not found") - return False - if r.status_code == 503: - logger.warning(f"{r.url} raised an internal server error") - return False - logger.error(f"\"{url}\" returned {r.status_code}: {r.text}") - return False - - # write to the file and create folder if it doesn't exist - if not os.path.exists(target.path): - os.makedirs(target.path, exist_ok=True) - with open(target.file, "wb") as mp3_file: - mp3_file.write(r.content) - logger.info("finished") - return True - - @classmethod - def fetch_audio(cls, song: song_objects.Song, src: song_objects.Source): - super().fetch_audio(song, src) - return cls.download_from_musify(song.target, src.url) - - -if __name__ == "__main__": - pass diff --git a/src/music_kraken/not_used_anymore/sources/source.py b/src/music_kraken/not_used_anymore/sources/source.py deleted file mode 100644 index 9883866..0000000 --- a/src/music_kraken/not_used_anymore/sources/source.py +++ /dev/null @@ -1,23 +0,0 @@ -from ...utils.shared import * -from typing import Tuple - -from ...database import song as song_objects - - -logger = URL_DOWNLOAD_LOGGER - -""" -The class "Source" is the superclass every class for specific audio -sources inherits from. This gives the advantage of a consistent -calling of the functions do search for a song and to download it. -""" - - -class AudioSource: - @classmethod - def fetch_source(cls, row: dict): - logger.info(f"try getting source {row.title} from {cls.__name__}") - - @classmethod - def fetch_audio(cls, song: song_objects.Song, src: song_objects.Source): - logger.info(f"downloading {song}: {cls.__name__} {src.url} -> {song.target.file}") diff --git a/src/music_kraken/not_used_anymore/sources/youtube.py b/src/music_kraken/not_used_anymore/sources/youtube.py deleted file mode 100644 index f0051d3..0000000 --- a/src/music_kraken/not_used_anymore/sources/youtube.py +++ /dev/null @@ -1,98 +0,0 @@ -from typing import List - -import youtube_dl -import time - -from ...utils.shared import * -from ...utils import phonetic_compares -from .source import AudioSource - -from ...database import song as song_objects - - -logger = YOUTUBE_LOGGER - -YDL_OPTIONS = {'format': 'bestaudio', 'noplaylist': 'True'} -YOUTUBE_URL_KEY = 'webpage_url' -YOUTUBE_TITLE_KEY = 'title' -WAIT_BETWEEN_BLOCK = 10 -MAX_TRIES = 3 - -def youtube_length_to_mp3_length(youtube_len: float) -> int: - return int(youtube_len * 1000) - - -class Youtube(AudioSource): - @classmethod - def get_youtube_from_isrc(cls, isrc: str) -> List[dict]: - # https://stackoverflow.com/questions/63388364/searching-youtube-videos-using-youtube-dl - with youtube_dl.YoutubeDL(YDL_OPTIONS) as ydl: - try: - videos = ydl.extract_info(f"ytsearch:{isrc}", download=False)['entries'] - except youtube_dl.utils.DownloadError: - return [] - - return [{ - 'url': video[YOUTUBE_URL_KEY], - 'title': video[YOUTUBE_TITLE_KEY], - 'length': youtube_length_to_mp3_length(float(videos[0]['duration'])) - } for video in videos] - - @classmethod - def fetch_source(cls, song: song_objects.Song): - # https://stackoverflow.com/questions/63388364/searching-youtube-videos-using-youtube-dl - super().fetch_source(song) - - if not song.has_isrc(): - return None - - real_title = song.title.lower() - - final_result = None - results = cls.get_youtube_from_isrc(song.isrc) - for result in results: - video_title = result['title'].lower() - match, distance = phonetic_compares.match_titles(video_title, real_title) - - if match: - continue - - if not phonetic_compares.match_length(song.length, result['length']): - logger.warning(f"{song.length} doesn't match with {result}") - continue - - final_result = result - - if final_result is None: - return None - logger.info(f"found video {final_result}") - return final_result['url'] - - @classmethod - def fetch_audio(cls, song: song_objects.Song, src: song_objects.Source, trie: int=0): - super().fetch_audio(song, src) - if song.target.file is None or song.target.path is None: - logger.warning(f"target hasn't been set. Can't download. Most likely a bug.") - return False - - options = { - 'format': 'bestaudio/best', - 'keepvideo': False, - 'outtmpl': song.target.file - } - - # downloading - try: - with youtube_dl.YoutubeDL(options) as ydl: - ydl.download([src.url]) - - except youtube_dl.utils.DownloadError: - # retry when failing - logger.warning(f"youtube blocked downloading. ({trie}-{MAX_TRIES})") - if trie >= MAX_TRIES: - logger.warning("too many tries, returning") - return False - logger.warning(f"retrying in {WAIT_BETWEEN_BLOCK} seconds again") - time.sleep(WAIT_BETWEEN_BLOCK) - return cls.fetch_audio(song, src, trie=trie + 1) - diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index a7e2a61..b562686 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -1,3 +1,5 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum from .musify import Musify +from .youtube import YouTube + from .abstract import Page, INDEPENDENT_DB_OBJECTS diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 46c8429..ed045b6 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -5,7 +5,6 @@ from typing import List, Optional, Type, Union from urllib.parse import urlparse import pycountry -import requests from bs4 import BeautifulSoup from ..connection import Connection @@ -20,7 +19,6 @@ from ..objects import ( ID3Timestamp, FormattedText, Label, - Options, Target, DatabaseObject ) diff --git a/src/music_kraken/pages/preset.py b/src/music_kraken/pages/preset.py index 7fc8212..5e940ba 100644 --- a/src/music_kraken/pages/preset.py +++ b/src/music_kraken/pages/preset.py @@ -12,8 +12,10 @@ from ..objects import ( Song, Album, Label, + Target ) from ..connection import Connection +from ..utils.support_classes import DownloadResult class Preset(Page): # CHANGE @@ -57,3 +59,6 @@ class Preset(Page): def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() + + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: + return DownloadResult() diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index a5d6736..b23b1f8 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -1,46 +1,72 @@ -from typing import List -import requests -from bs4 import BeautifulSoup -import pycountry - -from ..utils.shared import ( - ENCYCLOPAEDIA_METALLUM_LOGGER as LOGGER -) +from typing import List, Optional, Type +from urllib.parse import urlparse +import logging +from ..objects import Source, DatabaseObject from .abstract import Page -from ..database import ( - MusicObject, +from ..objects import ( Artist, Source, SourcePages, Song, Album, - ID3Timestamp, - FormattedText -) -from ..utils import ( - string_processing + Label, + Target ) +from ..connection import Connection +from ..utils.support_classes import DownloadResult +from ..utils.shared import YOUTUBE_LOGGER -INVIDIOUS_INSTANCE = "https://yewtu.be/feed/popular" -class Youtube(Page): - """ - The youtube downloader should use https://invidious.io/ - to make the request. - They are an alternative frontend. +""" +- https://y.com.sb/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance +- https://y.com.sb/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA +- https://y.com.sb/api/v1/playlists/OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM +""" - To find an artist filter for chanel and search for - `{artist.name} - Topic` - and then ofc check for viable results. - - Ofc you can also implement searching songs by isrc. - - NOTE: I didn't look at the invidious api yet. If it sucks, - feel free to use projects like youtube-dl. - But don't implement you're own youtube client. - I don't wanna maintain that shit. - """ - API_SESSION: requests.Session = requests.Session() +class YouTube(Page): + # CHANGE SOURCE_TYPE = SourcePages.YOUTUBE + LOGGER = YOUTUBE_LOGGER + + def __init__(self, *args, **kwargs): + self.connection: Connection = Connection( + host="https://www.preset.cum/", + logger=self.LOGGER + ) + + super().__init__(*args, **kwargs) + + def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: + return super().get_source_type(source) + + def general_search(self, search_query: str) -> List[DatabaseObject]: + return [Artist(name="works")] + + def label_search(self, label: Label) -> List[Label]: + return [] + + def artist_search(self, artist: Artist) -> List[Artist]: + return [] + + def album_search(self, album: Album) -> List[Album]: + return [] + + def song_search(self, song: Song) -> List[Song]: + return [] + + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: + return Song() + + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + return Album() + + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + return Artist() + + def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: + return Label() + + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: + return DownloadResult() From 6cd54724d2078f6f4fa8b4a783e27a25e6b29640 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Mon, 12 Jun 2023 22:54:04 +0200 Subject: [PATCH 46/81] completed new cli --- src/music_kraken/utils/config/connection.py | 6 +++--- src/music_kraken/utils/config/misc.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index d6bae7a..1dfb4bf 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -1,7 +1,7 @@ from urllib.parse import urlparse import re -from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute +from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute, StringAttribute from ..regex import URL_PATTERN from ..exception.config import SettingValueError @@ -15,7 +15,7 @@ class ProxAttribute(ListAttribute): } -class UrlListAttribute(ListAttribute): +class UrlStringAttribute(StringAttribute): def validate(self, value: str): v = value.strip() url = re.match(URL_PATTERN, v) @@ -64,7 +64,7 @@ class ConnectionSection(Section): ) # INVIDIOUS INSTANCES LIST - self.INVIDIOUS_INSTANCE_LIST = UrlListAttribute( + self.INVIDIOUS_INSTANCE = UrlListAttribute( name="invidious_instances", description="This is a List, where you can define the invidious instances,\n" "the youtube downloader should use.\n" diff --git a/src/music_kraken/utils/config/misc.py b/src/music_kraken/utils/config/misc.py index 469c0af..021f154 100644 --- a/src/music_kraken/utils/config/misc.py +++ b/src/music_kraken/utils/config/misc.py @@ -12,11 +12,11 @@ class MiscSection(Section): "Support the artist.", "Star Me: https://github.com/HeIIow2/music-downloader", "🏳️‍⚧️🏳️‍⚧️ Trans rights are human rights. 🏳️‍⚧️🏳️‍⚧️", - "🏳️‍⚧️🏳️‍⚧️ Trans women are women, trans men are men. 🏳️‍⚧️🏳️‍⚧️", - "🏴‍☠️🏴‍☠️ Unite under one flag, fuck borders. 🏴‍☠️🏴‍☠️", + "🏳️‍⚧️🏳️‍⚧️ Trans women are women, trans men are men, and enbies are enbies. 🏳️‍⚧️🏳️‍⚧️", + "🏴‍☠️🏴‍☠️ Unite under one flag, fck borders. 🏴‍☠️🏴‍☠️", "Join my Matrix Space: https://matrix.to/#/#music-kraken:matrix.org", - "Gotta love the BPJM!! >:(", - "🏳️‍⚧️🏳️‍⚧️ Protect trans youth. 🏳️‍⚧️🏳️‍⚧️" + "Gotta love the BPJM ;-;", + "🏳️‍⚧️🏳️‍⚧️ Protect trans youth. 🏳️‍⚧️🏳️‍⚧️", ] ) From f0fa05dc92f06c7b3fcaddcb5b78e8d220ce880e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 13 Jun 2023 11:45:02 +0200 Subject: [PATCH 47/81] added a module to easily config the invidious instance --- src/music_kraken/__main__.py | 12 +++ src/music_kraken/cli/__init__.py | 3 +- src/music_kraken/cli/options/__init__.py | 4 + .../cli/options/invidious/__init__.py | 0 .../cli/options/invidious/shell.py | 102 ++++++++++++++++++ src/music_kraken/utils/config/connection.py | 16 ++- 6 files changed, 127 insertions(+), 10 deletions(-) create mode 100644 src/music_kraken/cli/options/invidious/__init__.py create mode 100644 src/music_kraken/cli/options/invidious/shell.py diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 6ba0cff..4aa30ef 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -65,6 +65,13 @@ if __name__ == "__main__": help="Resets the config file to the default one.", action="store_true" ) + + parser.add_argument( + "--invidious", + "-i", + help="Set a good and fast invidious instance from your homecountry, to reduce the latency.", + action="store_true" + ) arguments = parser.parse_args() @@ -94,6 +101,11 @@ if __name__ == "__main__": if os.path.exists(music_kraken.shared.CONFIG_FILE): os.remove(music_kraken.shared.CONFIG_FILE) music_kraken.read() + + if arguments.invidious: + from .cli.options import invidious + invidious() + exit() # getting the genre genre: str = arguments.genre diff --git a/src/music_kraken/cli/__init__.py b/src/music_kraken/cli/__init__.py index c21af81..b4e57ef 100644 --- a/src/music_kraken/cli/__init__.py +++ b/src/music_kraken/cli/__init__.py @@ -1 +1,2 @@ -from .download.shell import Shell \ No newline at end of file +from .download.shell import Shell +from .options import invidious \ No newline at end of file diff --git a/src/music_kraken/cli/options/__init__.py b/src/music_kraken/cli/options/__init__.py index e69de29..0ab4d7f 100644 --- a/src/music_kraken/cli/options/__init__.py +++ b/src/music_kraken/cli/options/__init__.py @@ -0,0 +1,4 @@ +from .invidious.shell import InvidiousShell + +def invidious(): + shell = InvidiousShell() \ No newline at end of file diff --git a/src/music_kraken/cli/options/invidious/__init__.py b/src/music_kraken/cli/options/invidious/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/music_kraken/cli/options/invidious/shell.py b/src/music_kraken/cli/options/invidious/shell.py new file mode 100644 index 0000000..1d83d9c --- /dev/null +++ b/src/music_kraken/cli/options/invidious/shell.py @@ -0,0 +1,102 @@ +from typing import Dict, List +import requests +from dataclasses import dataclass +from collections import defaultdict + +from ....utils import config, write +from ....utils import exception + + +INSTANCES_ENDPOINT = "https://api.invidious.io/instances.json" + +@dataclass +class Instance: + """ + Attributes which influence the quality of an instance: + + - users + """ + name: str + uri: str + region: str + users: int + + def __str__(self) -> str: + return f"{self.name} with {self.users} users." + + +class InvidiousShell: + def __init__(self): + self.region_flags = dict() + self.region_instances: Dict[str, List[Instance]] = defaultdict(list) + + self.download_instances() + self.region = self.get_country() + print() + self.choose_instance() + + + def process_instance(self, all_instance_data: dict): + instance_data = all_instance_data[1] + stats = instance_data["stats"] + + if not instance_data["api"]: + return + if instance_data["type"] != "https": + return + + region = instance_data["region"] + flag = instance_data["flag"] + + self.region_flags[region] = flag + + instance = Instance( + name=all_instance_data[0], + uri=instance_data["uri"], + region=region, + users=stats["usage"]["users"]["total"] + ) + + self.region_instances[region].append(instance) + + def download_instances(self): + print("Download idonvidious instances...") + r = requests.get(INSTANCES_ENDPOINT) + + for instance in r.json(): + self.process_instance(all_instance_data=instance) + + def get_country(self): + print("Input the country code, an example would be \"US\"") + print(f"({' | '.join(f'{region}-{flag}' for region, flag in self.region_flags.items())})") + + chosen_region = "" + + while chosen_region.upper() not in self.region_instances: + chosen_region = input("nearest country: ").strip().upper() + + return chosen_region + + def choose_instance(self): + instance_list = self.region_instances[self.region] + instance_list.sort(key=lambda x: x.users, reverse=True) + + # output the options + print("Choose your instance (input needs to be a digit):") + for i, instance in enumerate(instance_list): + print(f"{i}) {instance}") + + print() + + # ask for index + index = "" + while not index.isdigit() or int(index) >= len(instance_list): + index = input("> ").strip() + + instance = instance_list[int(index)] + print() + print(f"Setting the instance to {instance}") + + config.set_name_to_value("invidious_instance", instance.uri) + write() + \ No newline at end of file diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index 1dfb4bf..4f35f6a 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -19,7 +19,7 @@ class UrlStringAttribute(StringAttribute): def validate(self, value: str): v = value.strip() url = re.match(URL_PATTERN, v) - if v != url: + if url is None: raise SettingValueError( setting_name=self.name, setting_value=v, @@ -64,18 +64,14 @@ class ConnectionSection(Section): ) # INVIDIOUS INSTANCES LIST - self.INVIDIOUS_INSTANCE = UrlListAttribute( - name="invidious_instances", + self.INVIDIOUS_INSTANCE = UrlStringAttribute( + name="invidious_instance", description="This is a List, where you can define the invidious instances,\n" "the youtube downloader should use.\n" "Here is a list of active ones: https://docs.invidious.io/instances/\n" "Instances that use cloudflare or have source code changes could cause issues.\n" "Hidden instances (.onion) will only work, when setting 'tor=true'.", - value=[ - "https://yt.artemislena.eu/", - "https://watch.thekitty.zone/", - "https://y.com.sb/" - ] + value="https://yt.artemislena.eu/" ) # INVIDIOUS PROXY self.INVIDIOUS_PROXY_VIDEOS = BoolAttribute( @@ -88,7 +84,9 @@ class ConnectionSection(Section): self.USE_TOR, self.TOR_PORT, self.CHUNK_SIZE, - self.SHOW_DOWNLOAD_ERRORS_THRESHOLD + self.SHOW_DOWNLOAD_ERRORS_THRESHOLD, + self.INVIDIOUS_INSTANCE, + self.INVIDIOUS_PROXY_VIDEOS ] super().__init__() From 500f82d079cd5e05ce527fedc32a026e56b7fbd2 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 13 Jun 2023 13:20:49 +0200 Subject: [PATCH 48/81] added url parsing for youtube --- .../cli/options/invidious/shell.py | 1 - src/music_kraken/pages/youtube.py | 119 ++++++++++++++++-- src/music_kraken/utils/shared.py | 2 + 3 files changed, 114 insertions(+), 8 deletions(-) diff --git a/src/music_kraken/cli/options/invidious/shell.py b/src/music_kraken/cli/options/invidious/shell.py index 1d83d9c..131a9d8 100644 --- a/src/music_kraken/cli/options/invidious/shell.py +++ b/src/music_kraken/cli/options/invidious/shell.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from collections import defaultdict from ....utils import config, write -from ....utils import exception INSTANCES_ENDPOINT = "https://api.invidious.io/instances.json" diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index b23b1f8..1a84eef 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -1,6 +1,8 @@ from typing import List, Optional, Type -from urllib.parse import urlparse +from urllib.parse import urlparse, urlunparse, parse_qs import logging +from dataclasses import dataclass +from enum import Enum from ..objects import Source, DatabaseObject from .abstract import Page @@ -15,16 +17,111 @@ from ..objects import ( ) from ..connection import Connection from ..utils.support_classes import DownloadResult -from ..utils.shared import YOUTUBE_LOGGER +from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE """ -- https://y.com.sb/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance -- https://y.com.sb/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA -- https://y.com.sb/api/v1/playlists/OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM +- https://yt.artemislena.eu/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance +- https://yt.artemislena.eu/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA +- https://yt.artemislena.eu/api/v1/playlists/OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM +- https://yt.artemislena.eu/api/v1/videos/SULFl39UjgY """ +def get_invidious_url(path: str = "", query: str = "", fragment: str = "") -> str: + return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, query, fragment)) + + +class YouTubeUrlType(Enum): + CHANNEL = "channel" + PLAYLIST = "playlist" + VIDEO = "watch" + NONE = "" + + +class YouTubeUrl: + """ + Artist + https://yt.artemislena.eu/channel/UCV0Ntl3lVR7xDXKoCU6uUXA + https://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA + + Release + https://yt.artemislena.eu/playlist?list=OLAK5uy_nEg5joAyFjHBPwnS_ADHYtgSqAjFMQKLw + https://www.youtube.com/playlist?list=OLAK5uy_nEg5joAyFjHBPwnS_ADHYtgSqAjFMQKLw + + Track + https://yt.artemislena.eu/watch?v=SULFl39UjgY&list=OLAK5uy_nEg5joAyFjHBPwnS_ADHYtgSqAjFMQKLw&index=1 + https://www.youtube.com/watch?v=SULFl39UjgY + """ + + def __init__(self, url: str) -> None: + """ + Raises Index exception for wrong url, and value error for not found enum type + """ + self.id = "" + parsed = urlparse(url=url) + + self.url_type: YouTubeUrlType + + type_frag_list = parsed.path.split("/") + if len(type_frag_list) < 2: + self.url_type = YouTubeUrlType.NONE + else: + try: + self.url_type = YouTubeUrlType(type_frag_list[1].strip()) + except ValueError: + self.url_type = YouTubeUrlType.NONE + + if self.url_type == YouTubeUrlType.CHANNEL: + if len(type_frag_list) < 3: + self.couldnt_find_id(url) + else: + self.id = type_frag_list[2] + + elif self.url_type == YouTubeUrlType.PLAYLIST: + query_stuff = parse_qs(parsed.query) + if "list" not in query_stuff: + self.couldnt_find_id(url) + else: + self.id = query_stuff["list"] + + elif self.url_type == YouTubeUrlType.VIDEO: + query_stuff = parse_qs(parsed.query) + if "v" not in query_stuff: + self.couldnt_find_id(url) + else: + self.id = query_stuff["v"] + + + def couldnt_find_id(self, url: str): + YOUTUBE_LOGGER.warning(f"The id is missing: {url}") + self.url_type = YouTubeUrlType.NONE + + @property + def api(self) -> str: + if self.url_type == YouTubeUrlType.CHANNEL: + return get_invidious_url(path=f"/api/v1/channels/playlists/{self.id}") + + if self.url_type == YouTubeUrlType.PLAYLIST: + return get_invidious_url(path=f"/api/v1/playlists/{id}") + + if self.url_type == YouTubeUrlType.VIDEO: + return get_invidious_url(path=f"/api/v1/videos/{self.id}") + + return get_invidious_url() + + @property + def normal(self) -> str: + if self.url_type.CHANNEL: + return get_invidious_url(path=f"/channel/{self.id}") + + if self.url_type.PLAYLIST: + return get_invidious_url(path="/playlist", query=f"list={self.id}") + + if self.url_type.VIDEO: + return get_invidious_url(path="/watch", query=f"v={self.id}") + + class YouTube(Page): # CHANGE SOURCE_TYPE = SourcePages.YOUTUBE @@ -32,14 +129,22 @@ class YouTube(Page): def __init__(self, *args, **kwargs): self.connection: Connection = Connection( - host="https://www.preset.cum/", + host=urlunparse(INVIDIOUS_INSTANCE), logger=self.LOGGER ) super().__init__(*args, **kwargs) def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: - return super().get_source_type(source) + _url_type = { + YouTubeUrlType.CHANNEL: Artist, + YouTubeUrlType.PLAYLIST: Album, + YouTubeUrlType.VIDEO: Song, + } + + parsed = YouTubeUrl(source.url) + if parsed.url_type in _url_type: + return _url_type[parsed.url_type] def general_search(self, search_query: str) -> List[DatabaseObject]: return [Artist(name="works")] diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index b17d3cd..97c2009 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -2,6 +2,7 @@ import logging import random from pathlib import Path from typing import List, Tuple, Set, Dict +from urllib.parse import ParseResult from .path_manager import LOCATIONS from .config import LOGGING_SECTION, AUDIO_SECTION, CONNECTION_SECTION, MISC_SECTION, PATHS_SECTION @@ -90,6 +91,7 @@ if TOR: 'http': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}', 'https': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}' } +INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.value # size of the chunks that are streamed CHUNK_SIZE = CONNECTION_SECTION.CHUNK_SIZE.object_from_value From 65f7121837caa9d43ded1be91f0c4824495c5b7a Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 13 Jun 2023 13:29:24 +0200 Subject: [PATCH 49/81] fixed parsing --- src/music_kraken/pages/youtube.py | 9 +++++++-- src/music_kraken/utils/config/connection.py | 7 ++++--- src/music_kraken/utils/shared.py | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 1a84eef..86df1be 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -29,7 +29,8 @@ from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE def get_invidious_url(path: str = "", query: str = "", fragment: str = "") -> str: - return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, query, fragment)) + _ = "" + return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, query, fragment, _)) class YouTubeUrlType(Enum): @@ -129,7 +130,7 @@ class YouTube(Page): def __init__(self, *args, **kwargs): self.connection: Connection = Connection( - host=urlunparse(INVIDIOUS_INSTANCE), + host=get_invidious_url(), logger=self.LOGGER ) @@ -147,12 +148,16 @@ class YouTube(Page): return _url_type[parsed.url_type] def general_search(self, search_query: str) -> List[DatabaseObject]: + return [Artist(name="works")] def label_search(self, label: Label) -> List[Label]: return [] def artist_search(self, artist: Artist) -> List[Artist]: + # https://yt.artemislena.eu/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance + endpoint = get_invidious_url(path="/api/v1/search", query=f"q={artist.name.replace(' ', '+')}+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance") + print(endpoint) return [] def album_search(self, album: Album) -> List[Album]: diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index 4f35f6a..b02326b 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -1,4 +1,4 @@ -from urllib.parse import urlparse +from urllib.parse import urlparse, ParseResult import re from .base_classes import Section, FloatAttribute, IntAttribute, BoolAttribute, ListAttribute, StringAttribute @@ -26,8 +26,9 @@ class UrlStringAttribute(StringAttribute): rule="has to be a valid url" ) - def single_object_from_element(self, value: str): - return urlparse(value) + @property + def object_from_value(self) -> ParseResult: + return urlparse(self.value) class ConnectionSection(Section): diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 97c2009..713f8ff 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -91,7 +91,7 @@ if TOR: 'http': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}', 'https': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}' } -INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.value +INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.object_from_value # size of the chunks that are streamed CHUNK_SIZE = CONNECTION_SECTION.CHUNK_SIZE.object_from_value From 79c38387c81e160f0e55e82c162559b101045a33 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 13 Jun 2023 15:03:11 +0200 Subject: [PATCH 50/81] huuuuge progress on youtube scraper --- src/actual_donwload.py | 7 +++- src/music_kraken/pages/youtube.py | 63 +++++++++++++++++++++++++++---- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index bb53da2..08b072d 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -11,5 +11,10 @@ if __name__ == "__main__": direct_download = [ "d: https://musify.club/release/crystal-f-x-2012-795181" ] + + youtube_search = [ + "s: #a Zombiez", + "10" + ] - music_kraken.cli(genre="test", command_list=direct_download) + music_kraken.cli(genre="test", command_list=youtube_search) diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 86df1be..f166b4c 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -28,9 +28,8 @@ from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE """ -def get_invidious_url(path: str = "", query: str = "", fragment: str = "") -> str: - _ = "" - return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, query, fragment, _)) +def get_invidious_url(path: str = "", params: str = "", query: str = "", fragment: str = "") -> str: + return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, params, query, fragment)) class YouTubeUrlType(Enum): @@ -153,12 +152,32 @@ class YouTube(Page): def label_search(self, label: Label) -> List[Label]: return [] + + def _json_to_artist(self, artist_json: dict) -> Artist:# + return Artist( + name=artist_json["author"].replace(" - Topic", ""), + source_list=[ + Source(self.SOURCE_TYPE, get_invidious_url(path=artist_json["authorUrl"])) + ] + ) def artist_search(self, artist: Artist) -> List[Artist]: # https://yt.artemislena.eu/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance endpoint = get_invidious_url(path="/api/v1/search", query=f"q={artist.name.replace(' ', '+')}+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance") - print(endpoint) - return [] + + artist_list = [] + + r = self.connection.get(endpoint) + for search_result in r.json(): + if search_result["type"] != "channel": + continue + author: str = search_result["author"] + if not author.endswith(" - Topic"): + continue + + artist_list.append(self._json_to_artist(search_result)) + + return artist_list def album_search(self, album: Album) -> List[Album]: return [] @@ -170,10 +189,40 @@ class YouTube(Page): return Song() def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: - return Album() + return Album() def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: - return Artist() + parsed = YouTubeUrl(source.url) + if parsed.url_type != YouTubeUrlType.CHANNEL: + return Artist(source_list=[source]) + + artist_name = None + album_list = [] + + # playlist + # https://yt.artemislena.eu/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA + r = self.connection.get(get_invidious_url(f"/api/v1/channels/playlists/{parsed.id}")) + for playlist_json in r.json()["playlists"]: + if playlist_json["type"] != "playlist": + continue + + artist_name = playlist_json["author"].replace(" - Topic", "") + + # /playlist?list=OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw + album_list.append(Album( + title=playlist_json["title"], + source_list=[Source( + self.SOURCE_TYPE, get_invidious_url(path="/playlist", query=f"list={playlist_json['playlistId']}") + )], + artist_list=[Artist( + name=artist_name, + source_list=[ + Source(self.SOURCE_TYPE, get_invidious_url(path=playlist_json["authorUrl"])) + ] + )] + )) + + return Artist(name=artist_name, main_album_list=album_list, source_list=[source]) def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() From 73576b04538bb86cbe9bbcbebb5a634a3b8dc8b5 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 13 Jun 2023 20:10:11 +0200 Subject: [PATCH 51/81] completely implemented downloading from youtube --- src/actual_donwload.py | 3 +- src/music_kraken/connection/connection.py | 7 +- src/music_kraken/objects/metadata.py | 13 ++ src/music_kraken/pages/youtube.py | 165 +++++++++++++++++++--- 4 files changed, 168 insertions(+), 20 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 08b072d..2c24a28 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -14,7 +14,8 @@ if __name__ == "__main__": youtube_search = [ "s: #a Zombiez", - "10" + "10", + "8" ] music_kraken.cli(genre="test", command_list=youtube_search) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 02eba18..46c22de 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -1,3 +1,4 @@ +import time from typing import List, Dict, Callable, Optional, Set from urllib.parse import urlparse, urlunsplit, ParseResult import logging @@ -82,6 +83,7 @@ class Connection: headers: dict, refer_from_origin: bool = True, raw_url: bool = False, + wait_on_403: bool = True, **kwargs ) -> Optional[requests.Response]: if try_count >= self.TRIES: @@ -122,12 +124,15 @@ class Connection: self.LOGGER.warning(f"{self.HOST.netloc} responded wit {r.status_code} " f"at {url}. ({try_count}-{self.TRIES})") self.LOGGER.debug(r.content) + if wait_on_403: + self.LOGGER.warning(f"Waiting for 5 seconds.") + time.sleep(5) self.rotate() return self._request( request=request, - try_count=try_count, + try_count=try_count+1, accepted_response_code=accepted_response_code, url=url, timeout=timeout, diff --git a/src/music_kraken/objects/metadata.py b/src/music_kraken/objects/metadata.py index 3d98d35..488af12 100644 --- a/src/music_kraken/objects/metadata.py +++ b/src/music_kraken/objects/metadata.py @@ -205,6 +205,19 @@ class ID3Timestamp: time_format = self.get_time_format() return time_format, self.date_obj.strftime(time_format) + @classmethod + def fromtimestamp(cls, utc_timestamp: int): + date_obj = datetime.datetime.fromtimestamp(utc_timestamp) + + return cls( + year=date_obj.year, + month=date_obj.month, + day=date_obj.day, + hour=date_obj.hour, + minute=date_obj.minute, + second=date_obj.second + ) + @classmethod def strptime(cls, time_stamp: str, format: str): """ diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index f166b4c..94d29ba 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -1,7 +1,5 @@ -from typing import List, Optional, Type +from typing import List, Optional, Type, Tuple from urllib.parse import urlparse, urlunparse, parse_qs -import logging -from dataclasses import dataclass from enum import Enum from ..objects import Source, DatabaseObject @@ -13,11 +11,13 @@ from ..objects import ( Song, Album, Label, - Target + Target, + FormattedText, + ID3Timestamp ) from ..connection import Connection from ..utils.support_classes import DownloadResult -from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE +from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE """ @@ -83,14 +83,14 @@ class YouTubeUrl: if "list" not in query_stuff: self.couldnt_find_id(url) else: - self.id = query_stuff["list"] + self.id = query_stuff["list"][0] elif self.url_type == YouTubeUrlType.VIDEO: query_stuff = parse_qs(parsed.query) if "v" not in query_stuff: self.couldnt_find_id(url) else: - self.id = query_stuff["v"] + self.id = query_stuff["v"][0] def couldnt_find_id(self, url: str): @@ -133,6 +133,11 @@ class YouTube(Page): logger=self.LOGGER ) + self.download_connection: Connection = Connection( + host="https://www.youtube.com/", + logger=self.LOGGER + ) + super().__init__(*args, **kwargs) def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: @@ -147,11 +152,7 @@ class YouTube(Page): return _url_type[parsed.url_type] def general_search(self, search_query: str) -> List[DatabaseObject]: - - return [Artist(name="works")] - - def label_search(self, label: Label) -> List[Label]: - return [] + return self.artist_search(Artist(name=search_query, dynamic=True)) def _json_to_artist(self, artist_json: dict) -> Artist:# return Artist( @@ -168,6 +169,9 @@ class YouTube(Page): artist_list = [] r = self.connection.get(endpoint) + if r is None: + return [] + for search_result in r.json(): if search_result["type"] != "channel": continue @@ -185,11 +189,90 @@ class YouTube(Page): def song_search(self, song: Song) -> List[Song]: return [] + def _fetch_song_from_id(self, youtube_id: str) -> Tuple[Song, Optional[int]]: + # https://yt.artemislena.eu/api/v1/videos/SULFl39UjgY + r = self.connection.get(get_invidious_url(path=f"/api/v1/videos/{youtube_id}")) + if r is None: + return Song(), None + + data = r.json() + if data["genre"] != "Music": + self.LOGGER.warning(f"Genre has to be music, trying anyways") + + title = data["title"] + license_str = None + for music_track in data.get("musicTracks", []): + title = music_track["song"] + license_str = music_track["license"] + + return Song( + title=title, + source_list=[Source( + self.SOURCE_TYPE, get_invidious_url(path="/watch", query=f"v={data['videoId']}") + )], + notes=FormattedText(html=data["descriptionHtml"] + f"\n

{license_str}" ) + ), int(data["published"]) + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: - return Song() + parsed = YouTubeUrl(source.url) + if parsed.url_type != YouTubeUrlType.VIDEO: + return Song() + + song, _ = self._fetch_song_from_id(parsed.id) + return song def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: - return Album() + parsed = YouTubeUrl(source.url) + if parsed.url_type != YouTubeUrlType.PLAYLIST: + return Album() + + title = None + source_list = [source] + notes = None + song_list = [] + + # https://yt.artemislena.eu/api/v1/playlists/OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM + r = self.connection.get(get_invidious_url(path=f"/api/v1/playlists/{parsed.id}")) + if r is None: + return Album() + + data = r.json() + if data["type"] != "playlist": + return Album() + + title = data["title"] + notes = FormattedText(html=data["descriptionHtml"]) + + timestamps: List[int] = [] + + """ + TODO + fetch the song and don't get it from there + """ + for video in data["videos"]: + other_song = Song( + source_list=[ + Source( + self.SOURCE_TYPE, get_invidious_url(path="/watch", query=f"v={video['videoId']}") + ) + ], + tracksort=video["index"]+1 + ) + + song, utc_timestamp = self._fetch_song_from_id(video["videoId"]) + song.merge(other_song) + + if utc_timestamp is not None: + timestamps.append(utc_timestamp) + song_list.append(song) + + return Album( + title=title, + source_list=source_list, + notes=notes, + song_list=song_list, + date=ID3Timestamp.fromtimestamp(round(sum(timestamps) / len(timestamps))) + ) def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: parsed = YouTubeUrl(source.url) @@ -202,6 +285,9 @@ class YouTube(Page): # playlist # https://yt.artemislena.eu/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA r = self.connection.get(get_invidious_url(f"/api/v1/channels/playlists/{parsed.id}")) + if r is None: + return Artist() + for playlist_json in r.json()["playlists"]: if playlist_json["type"] != "playlist": continue @@ -224,8 +310,51 @@ class YouTube(Page): return Artist(name=artist_name, main_album_list=album_list, source_list=[source]) - def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: - return Label() - def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: - return DownloadResult() + """ + 1. getting the optimal source + Only audio sources allowed + not a bitrate that is smaller than the selected bitrate, but not one that is wayyy huger + + 2. download it + + :param source: + :param target: + :param desc: + :return: + """ + r = self.connection.get(YouTubeUrl(source.url).api) + if r is None: + return DownloadResult(error_message="Api didn't even respond, maybe try another invidious Instance") + + audio_format = None + best_bitrate = 0 + + for possible_format in r.json()["adaptiveFormats"]: + format_type: str = possible_format["type"] + if not format_type.startswith("audio"): + continue + + bitrate = int(possible_format.get("bitrate", 0)) + + if bitrate >= BITRATE: + best_bitrate = bitrate + audio_format = possible_format + break + + if bitrate > best_bitrate: + best_bitrate = bitrate + audio_format = possible_format + + if audio_format is None: + return DownloadResult(error_message="Couldn't find the download link.") + + endpoint = audio_format["url"] + + r = self.download_connection.get(endpoint, stream=True, raw_url=True) + if r is None: + return DownloadResult(error_message=f"Couldn't connect to {endpoint}") + + if target.stream_into(r, desc=desc): + return DownloadResult(total=1) + return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}") From 01f882e0a126ddc1c21af3957ba4591493b8e98f Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 13 Jun 2023 22:12:35 +0200 Subject: [PATCH 52/81] completely implemented downloading from youtube --- src/music_kraken/pages/youtube.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 94d29ba..6f3a3cb 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -201,16 +201,33 @@ class YouTube(Page): title = data["title"] license_str = None - for music_track in data.get("musicTracks", []): - title = music_track["song"] - license_str = music_track["license"] + + artist_list: List[Artist] = [] + + _author: str = data["author"] + if _author.endswith(" - Topic"): + artist_list.append(Artist( + name=_author.replace(" - Topic", ""), + source_list=[Source( + self.SOURCE_TYPE, get_invidious_url(path=f"/channel/{data['authorId']}") + )] + )) + + else: + for music_track in data.get("musicTracks", []): + title = music_track["song"] + license_str = music_track["license"] + + for artist_name in music_track["artist"].split(" x "): + artist_list.append(Artist(name=artist_name)) return Song( title=title, source_list=[Source( self.SOURCE_TYPE, get_invidious_url(path="/watch", query=f"v={data['videoId']}") )], - notes=FormattedText(html=data["descriptionHtml"] + f"\n

{license_str}" ) + notes=FormattedText(html=data["descriptionHtml"] + f"\n

{license_str}" ), + main_artist_list=artist_list ), int(data["published"]) def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: From 1e49089de9cfb6ebd73b90c78d542fa2f9c8a9a0 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 14 Jun 2023 12:38:36 +0200 Subject: [PATCH 53/81] started implementing sponsorblock --- .VSCodeCounter/2023-02-28_10-10-37/details.md | 78 ----------- .../2023-02-28_10-10-37/diff-details.md | 19 --- .VSCodeCounter/2023-02-28_10-10-37/diff.csv | 6 - .VSCodeCounter/2023-02-28_10-10-37/diff.md | 24 ---- .VSCodeCounter/2023-02-28_10-10-37/diff.txt | 31 ---- .../2023-02-28_10-10-37/results.csv | 65 --------- .../2023-02-28_10-10-37/results.json | 1 - .VSCodeCounter/2023-02-28_10-10-37/results.md | 40 ------ .../2023-02-28_10-10-37/results.txt | 106 -------------- .VSCodeCounter/2023-06-14_07-47-51/details.md | 98 +++++++++++++ .../2023-06-14_07-47-51/diff-details.md | 59 ++++++++ .VSCodeCounter/2023-06-14_07-47-51/diff.csv | 46 ++++++ .VSCodeCounter/2023-06-14_07-47-51/diff.md | 45 ++++++ .VSCodeCounter/2023-06-14_07-47-51/diff.txt | 92 ++++++++++++ .../2023-06-14_07-47-51/results.csv | 85 +++++++++++ .../2023-06-14_07-47-51/results.json | 1 + .VSCodeCounter/2023-06-14_07-47-51/results.md | 46 ++++++ .../2023-06-14_07-47-51/results.txt | 132 ++++++++++++++++++ src/music_kraken/audio/codec.py | 20 +++ src/music_kraken/utils/config/connection.py | 9 +- 20 files changed, 632 insertions(+), 371 deletions(-) delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/details.md delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/diff-details.md delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/diff.csv delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/diff.md delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/diff.txt delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/results.csv delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/results.json delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/results.md delete mode 100644 .VSCodeCounter/2023-02-28_10-10-37/results.txt create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/details.md create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/diff-details.md create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/diff.csv create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/diff.md create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/diff.txt create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/results.csv create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/results.json create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/results.md create mode 100644 .VSCodeCounter/2023-06-14_07-47-51/results.txt diff --git a/.VSCodeCounter/2023-02-28_10-10-37/details.md b/.VSCodeCounter/2023-02-28_10-10-37/details.md deleted file mode 100644 index 5f9148f..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/details.md +++ /dev/null @@ -1,78 +0,0 @@ -# Details - -Date : 2023-02-28 10:10:37 - -Directory /home/lars/Projects/music-downloader/src - -Total : 63 files, 4150 codes, 1060 comments, 1228 blanks, all 6438 lines - -[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) - -## Files -| filename | language | code | comment | blank | total | -| :--- | :--- | ---: | ---: | ---: | ---: | -| [src/__init__.py](/src/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/create_custom_objects.py](/src/create_custom_objects.py) | Python | 80 | 3 | 18 | 101 | -| [src/metal_archives.py](/src/metal_archives.py) | Python | 48 | 4 | 15 | 67 | -| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | 57 | 8 | 25 | 90 | -| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 3 | 2 | 3 | 8 | -| [src/music_kraken/database/__init__.py](/src/music_kraken/database/__init__.py) | Python | 18 | 0 | 5 | 23 | -| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 123 | 23 | 52 | 198 | -| [src/music_kraken/database/database.py](/src/music_kraken/database/database.py) | Python | 84 | 48 | 26 | 158 | -| [src/music_kraken/database/object_cache.py](/src/music_kraken/database/object_cache.py) | Python | 35 | 56 | 16 | 107 | -| [src/music_kraken/database/old_database.py](/src/music_kraken/database/old_database.py) | Python | 432 | 154 | 115 | 701 | -| [src/music_kraken/database/read.py](/src/music_kraken/database/read.py) | Python | 0 | 0 | 1 | 1 | -| [src/music_kraken/database/temp_database.py](/src/music_kraken/database/temp_database.py) | Python | 12 | 0 | 8 | 20 | -| [src/music_kraken/database/write.py](/src/music_kraken/database/write.py) | Python | 210 | 62 | 63 | 335 | -| [src/music_kraken/not_used_anymore/__init__.py](/src/music_kraken/not_used_anymore/__init__.py) | Python | 0 | 0 | 3 | 3 | -| [src/music_kraken/not_used_anymore/fetch_audio.py](/src/music_kraken/not_used_anymore/fetch_audio.py) | Python | 75 | 12 | 20 | 107 | -| [src/music_kraken/not_used_anymore/fetch_source.py](/src/music_kraken/not_used_anymore/fetch_source.py) | Python | 54 | 1 | 16 | 71 | -| [src/music_kraken/not_used_anymore/metadata/__init__.py](/src/music_kraken/not_used_anymore/metadata/__init__.py) | Python | 6 | 0 | 2 | 8 | -| [src/music_kraken/not_used_anymore/metadata/metadata_fetch.py](/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py) | Python | 257 | 24 | 65 | 346 | -| [src/music_kraken/not_used_anymore/metadata/metadata_search.py](/src/music_kraken/not_used_anymore/metadata/metadata_search.py) | Python | 253 | 40 | 72 | 365 | -| [src/music_kraken/not_used_anymore/metadata/sources/__init__.py](/src/music_kraken/not_used_anymore/metadata/sources/__init__.py) | Python | 3 | 0 | 2 | 5 | -| [src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py](/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py) | Python | 42 | 6 | 12 | 60 | -| [src/music_kraken/not_used_anymore/sources/__init__.py](/src/music_kraken/not_used_anymore/sources/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/music_kraken/not_used_anymore/sources/genius.py](/src/music_kraken/not_used_anymore/sources/genius.py) | Python | 115 | 16 | 42 | 173 | -| [src/music_kraken/not_used_anymore/sources/local_files.py](/src/music_kraken/not_used_anymore/sources/local_files.py) | Python | 40 | 0 | 18 | 58 | -| [src/music_kraken/not_used_anymore/sources/musify.py](/src/music_kraken/not_used_anymore/sources/musify.py) | Python | 136 | 9 | 37 | 182 | -| [src/music_kraken/not_used_anymore/sources/source.py](/src/music_kraken/not_used_anymore/sources/source.py) | Python | 11 | 5 | 8 | 24 | -| [src/music_kraken/not_used_anymore/sources/youtube.py](/src/music_kraken/not_used_anymore/sources/youtube.py) | Python | 71 | 4 | 24 | 99 | -| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 24 | 0 | 7 | 31 | -| [src/music_kraken/objects/album.py](/src/music_kraken/objects/album.py) | Python | 15 | 6 | 5 | 26 | -| [src/music_kraken/objects/artist.py](/src/music_kraken/objects/artist.py) | Python | 18 | 0 | 5 | 23 | -| [src/music_kraken/objects/collection.py](/src/music_kraken/objects/collection.py) | Python | 52 | 15 | 24 | 91 | -| [src/music_kraken/objects/formatted_text.py](/src/music_kraken/objects/formatted_text.py) | Python | 53 | 57 | 20 | 130 | -| [src/music_kraken/objects/lyrics.py](/src/music_kraken/objects/lyrics.py) | Python | 21 | 0 | 7 | 28 | -| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 262 | 68 | 63 | 393 | -| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 22 | 15 | 12 | 49 | -| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 337 | 81 | 90 | 508 | -| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 116 | 38 | 41 | 195 | -| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 22 | 7 | 7 | 36 | -| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | 7 | 0 | 5 | 12 | -| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | 73 | 68 | 27 | 168 | -| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | 341 | 66 | 91 | 498 | -| [src/music_kraken/pages/youtube.py](/src/music_kraken/pages/youtube.py) | Python | 25 | 16 | 6 | 47 | -| [src/music_kraken/static_files/new_db.sql](/src/music_kraken/static_files/new_db.sql) | SQLite | 72 | 0 | 10 | 82 | -| [src/music_kraken/static_files/temp_database_structure.sql](/src/music_kraken/static_files/temp_database_structure.sql) | SQLite | 135 | 0 | 10 | 145 | -| [src/music_kraken/tagging/__init__.py](/src/music_kraken/tagging/__init__.py) | Python | 8 | 0 | 2 | 10 | -| [src/music_kraken/tagging/id3.py](/src/music_kraken/tagging/id3.py) | Python | 51 | 4 | 20 | 75 | -| [src/music_kraken/target/__init__.py](/src/music_kraken/target/__init__.py) | Python | 4 | 0 | 2 | 6 | -| [src/music_kraken/target/set_target.py](/src/music_kraken/target/set_target.py) | Python | 37 | 7 | 18 | 62 | -| [src/music_kraken/utils/__init__.py](/src/music_kraken/utils/__init__.py) | Python | 1 | 1 | 1 | 3 | -| [src/music_kraken/utils/functions.py](/src/music_kraken/utils/functions.py) | Python | 3 | 0 | 1 | 4 | -| [src/music_kraken/utils/object_handeling.py](/src/music_kraken/utils/object_handeling.py) | Python | 19 | 0 | 6 | 25 | -| [src/music_kraken/utils/phonetic_compares.py](/src/music_kraken/utils/phonetic_compares.py) | Python | 39 | 2 | 17 | 58 | -| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | 62 | 3 | 10 | 75 | -| [src/music_kraken/utils/string_processing.py](/src/music_kraken/utils/string_processing.py) | Python | 2 | 5 | 2 | 9 | -| [src/music_kraken_cli.py](/src/music_kraken_cli.py) | Python | 3 | 0 | 3 | 6 | -| [src/music_kraken_gtk.py](/src/music_kraken_gtk.py) | Python | 3 | 0 | 2 | 5 | -| [src/test.db](/src/test.db) | Database | 91 | 0 | 1 | 92 | -| [src/tests/__init__.py](/src/tests/__init__.py) | Python | 0 | 0 | 1 | 1 | -| [src/tests/conftest.py](/src/tests/conftest.py) | Python | 3 | 1 | 2 | 6 | -| [src/tests/example_data_objects.py](/src/tests/example_data_objects.py) | Python | 36 | 5 | 6 | 47 | -| [src/try-programming-interface.py](/src/try-programming-interface.py) | Python | 14 | 98 | 22 | 134 | -| [src/try.py](/src/try.py) | Python | 1 | 0 | 3 | 4 | -| [src/try_python.py](/src/try_python.py) | Python | 13 | 20 | 9 | 42 | - -[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/diff-details.md b/.VSCodeCounter/2023-02-28_10-10-37/diff-details.md deleted file mode 100644 index 57af95e..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/diff-details.md +++ /dev/null @@ -1,19 +0,0 @@ -# Diff Details - -Date : 2023-02-28 10:10:37 - -Directory /home/lars/Projects/music-downloader/src - -Total : 4 files, 55 codes, 0 comments, 12 blanks, all 67 lines - -[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details - -## Files -| filename | language | code | comment | blank | total | -| :--- | :--- | ---: | ---: | ---: | ---: | -| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 7 | 0 | 3 | 10 | -| [src/music_kraken/database/write.py](/src/music_kraken/database/write.py) | Python | 42 | 0 | 8 | 50 | -| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 1 | 0 | 0 | 1 | -| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 5 | 0 | 1 | 6 | - -[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/diff.csv b/.VSCodeCounter/2023-02-28_10-10-37/diff.csv deleted file mode 100644 index afd5729..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/diff.csv +++ /dev/null @@ -1,6 +0,0 @@ -"filename", "language", "Python", "comment", "blank", "total" -"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 7, 0, 3, 10 -"/home/lars/Projects/music-downloader/src/music_kraken/database/write.py", "Python", 42, 0, 8, 50 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 1, 0, 0, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 5, 0, 1, 6 -"Total", "-", 55, 0, 12, 67 \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/diff.md b/.VSCodeCounter/2023-02-28_10-10-37/diff.md deleted file mode 100644 index 8cba244..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/diff.md +++ /dev/null @@ -1,24 +0,0 @@ -# Diff Summary - -Date : 2023-02-28 10:10:37 - -Directory /home/lars/Projects/music-downloader/src - -Total : 4 files, 55 codes, 0 comments, 12 blanks, all 67 lines - -[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) - -## Languages -| language | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| Python | 4 | 55 | 0 | 12 | 67 | - -## Directories -| path | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| . | 4 | 55 | 0 | 12 | 67 | -| music_kraken | 4 | 55 | 0 | 12 | 67 | -| music_kraken/database | 2 | 49 | 0 | 11 | 60 | -| music_kraken/objects | 2 | 6 | 0 | 1 | 7 | - -[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/diff.txt b/.VSCodeCounter/2023-02-28_10-10-37/diff.txt deleted file mode 100644 index 9bce238..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/diff.txt +++ /dev/null @@ -1,31 +0,0 @@ -Date : 2023-02-28 10:10:37 -Directory : /home/lars/Projects/music-downloader/src -Total : 4 files, 55 codes, 0 comments, 12 blanks, all 67 lines - -Languages -+----------+------------+------------+------------+------------+------------+ -| language | files | code | comment | blank | total | -+----------+------------+------------+------------+------------+------------+ -| Python | 4 | 55 | 0 | 12 | 67 | -+----------+------------+------------+------------+------------+------------+ - -Directories -+-------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| path | files | code | comment | blank | total | -+-------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| . | 4 | 55 | 0 | 12 | 67 | -| music_kraken | 4 | 55 | 0 | 12 | 67 | -| music_kraken/database | 2 | 49 | 0 | 11 | 60 | -| music_kraken/objects | 2 | 6 | 0 | 1 | 7 | -+-------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ - -Files -+-------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| filename | language | code | comment | blank | total | -+-------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 7 | 0 | 3 | 10 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/write.py | Python | 42 | 0 | 8 | 50 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 1 | 0 | 0 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 5 | 0 | 1 | 6 | -| Total | | 55 | 0 | 12 | 67 | -+-------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/results.csv b/.VSCodeCounter/2023-02-28_10-10-37/results.csv deleted file mode 100644 index ee170db..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/results.csv +++ /dev/null @@ -1,65 +0,0 @@ -"filename", "language", "Python", "Database", "SQLite", "comment", "blank", "total" -"/home/lars/Projects/music-downloader/src/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/create_custom_objects.py", "Python", 80, 0, 0, 3, 18, 101 -"/home/lars/Projects/music-downloader/src/metal_archives.py", "Python", 48, 0, 0, 4, 15, 67 -"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", 57, 0, 0, 8, 25, 90 -"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 3, 0, 0, 2, 3, 8 -"/home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py", "Python", 18, 0, 0, 0, 5, 23 -"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 123, 0, 0, 23, 52, 198 -"/home/lars/Projects/music-downloader/src/music_kraken/database/database.py", "Python", 84, 0, 0, 48, 26, 158 -"/home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py", "Python", 35, 0, 0, 56, 16, 107 -"/home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py", "Python", 432, 0, 0, 154, 115, 701 -"/home/lars/Projects/music-downloader/src/music_kraken/database/read.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py", "Python", 12, 0, 0, 0, 8, 20 -"/home/lars/Projects/music-downloader/src/music_kraken/database/write.py", "Python", 210, 0, 0, 62, 63, 335 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py", "Python", 0, 0, 0, 0, 3, 3 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py", "Python", 75, 0, 0, 12, 20, 107 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py", "Python", 54, 0, 0, 1, 16, 71 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py", "Python", 6, 0, 0, 0, 2, 8 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py", "Python", 257, 0, 0, 24, 65, 346 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py", "Python", 253, 0, 0, 40, 72, 365 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py", "Python", 3, 0, 0, 0, 2, 5 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py", "Python", 42, 0, 0, 6, 12, 60 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py", "Python", 115, 0, 0, 16, 42, 173 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py", "Python", 40, 0, 0, 0, 18, 58 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py", "Python", 136, 0, 0, 9, 37, 182 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py", "Python", 11, 0, 0, 5, 8, 24 -"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py", "Python", 71, 0, 0, 4, 24, 99 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 24, 0, 0, 0, 7, 31 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/album.py", "Python", 15, 0, 0, 6, 5, 26 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py", "Python", 18, 0, 0, 0, 5, 23 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py", "Python", 52, 0, 0, 15, 24, 91 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py", "Python", 53, 0, 0, 57, 20, 130 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py", "Python", 21, 0, 0, 0, 7, 28 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 262, 0, 0, 68, 63, 393 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 22, 0, 0, 15, 12, 49 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 337, 0, 0, 81, 90, 508 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 116, 0, 0, 38, 41, 195 -"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 22, 0, 0, 7, 7, 36 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", 7, 0, 0, 0, 5, 12 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", 73, 0, 0, 68, 27, 168 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", 341, 0, 0, 66, 91, 498 -"/home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py", "Python", 25, 0, 0, 16, 6, 47 -"/home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql", "SQLite", 0, 0, 72, 0, 10, 82 -"/home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql", "SQLite", 0, 0, 135, 0, 10, 145 -"/home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py", "Python", 8, 0, 0, 0, 2, 10 -"/home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py", "Python", 51, 0, 0, 4, 20, 75 -"/home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py", "Python", 4, 0, 0, 0, 2, 6 -"/home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py", "Python", 37, 0, 0, 7, 18, 62 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py", "Python", 1, 0, 0, 1, 1, 3 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py", "Python", 3, 0, 0, 0, 1, 4 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py", "Python", 19, 0, 0, 0, 6, 25 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py", "Python", 39, 0, 0, 2, 17, 58 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", 62, 0, 0, 3, 10, 75 -"/home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py", "Python", 2, 0, 0, 5, 2, 9 -"/home/lars/Projects/music-downloader/src/music_kraken_cli.py", "Python", 3, 0, 0, 0, 3, 6 -"/home/lars/Projects/music-downloader/src/music_kraken_gtk.py", "Python", 3, 0, 0, 0, 2, 5 -"/home/lars/Projects/music-downloader/src/test.db", "Database", 0, 91, 0, 0, 1, 92 -"/home/lars/Projects/music-downloader/src/tests/__init__.py", "Python", 0, 0, 0, 0, 1, 1 -"/home/lars/Projects/music-downloader/src/tests/conftest.py", "Python", 3, 0, 0, 1, 2, 6 -"/home/lars/Projects/music-downloader/src/tests/example_data_objects.py", "Python", 36, 0, 0, 5, 6, 47 -"/home/lars/Projects/music-downloader/src/try-programming-interface.py", "Python", 14, 0, 0, 98, 22, 134 -"/home/lars/Projects/music-downloader/src/try.py", "Python", 1, 0, 0, 0, 3, 4 -"/home/lars/Projects/music-downloader/src/try_python.py", "Python", 13, 0, 0, 20, 9, 42 -"Total", "-", 3852, 91, 207, 1060, 1228, 6438 \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/results.json b/.VSCodeCounter/2023-02-28_10-10-37/results.json deleted file mode 100644 index aa50f6f..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/results.json +++ /dev/null @@ -1 +0,0 @@ -{"file:///home/lars/Projects/music-downloader/src/try.py":{"language":"Python","code":1,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/try_python.py":{"language":"Python","code":13,"comment":20,"blank":9},"file:///home/lars/Projects/music-downloader/src/tests/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/tests/conftest.py":{"language":"Python","code":3,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/__init__.py":{"language":"Python","code":57,"comment":8,"blank":25},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/database.py":{"language":"Python","code":84,"comment":48,"blank":26},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py":{"language":"Python","code":18,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py":{"language":"Python","code":123,"comment":23,"blank":52},"file:///home/lars/Projects/music-downloader/src/test.db":{"language":"Database","code":91,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/read.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py":{"language":"Python","code":35,"comment":56,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py":{"language":"Python","code":432,"comment":154,"blank":115},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py":{"language":"Python","code":12,"comment":0,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py":{"language":"Python","code":73,"comment":68,"blank":27},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py":{"language":"Python","code":7,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py":{"language":"Python","code":25,"comment":16,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py":{"language":"Python","code":1,"comment":1,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/write.py":{"language":"Python","code":210,"comment":62,"blank":63},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py":{"language":"Python","code":2,"comment":5,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py":{"language":"Python","code":19,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py":{"language":"Python","code":341,"comment":66,"blank":91},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py":{"language":"Python","code":39,"comment":2,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py":{"language":"Python","code":62,"comment":3,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql":{"language":"SQLite","code":135,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/target.py":{"language":"Python","code":22,"comment":7,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py":{"language":"Python","code":52,"comment":15,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/album.py":{"language":"Python","code":15,"comment":6,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py":{"language":"Python","code":24,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/song.py":{"language":"Python","code":337,"comment":81,"blank":90},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/source.py":{"language":"Python","code":116,"comment":38,"blank":41},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py":{"language":"Python","code":262,"comment":68,"blank":63},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py":{"language":"Python","code":18,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py":{"language":"Python","code":22,"comment":15,"blank":12},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py":{"language":"Python","code":21,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py":{"language":"Python","code":53,"comment":57,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py":{"language":"Python","code":8,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py":{"language":"Python","code":51,"comment":4,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/__main__.py":{"language":"Python","code":3,"comment":2,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py":{"language":"Python","code":4,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py":{"language":"Python","code":0,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql":{"language":"SQLite","code":72,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py":{"language":"Python","code":75,"comment":12,"blank":20},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py":{"language":"Python","code":115,"comment":16,"blank":42},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py":{"language":"Python","code":11,"comment":5,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py":{"language":"Python","code":71,"comment":4,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py":{"language":"Python","code":136,"comment":9,"blank":37},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py":{"language":"Python","code":40,"comment":0,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py":{"language":"Python","code":54,"comment":1,"blank":16},"file:///home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py":{"language":"Python","code":37,"comment":7,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py":{"language":"Python","code":6,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py":{"language":"Python","code":253,"comment":40,"blank":72},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py":{"language":"Python","code":42,"comment":6,"blank":12},"file:///home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py":{"language":"Python","code":257,"comment":24,"blank":65},"file:///home/lars/Projects/music-downloader/src/tests/example_data_objects.py":{"language":"Python","code":36,"comment":5,"blank":6},"file:///home/lars/Projects/music-downloader/src/metal_archives.py":{"language":"Python","code":48,"comment":4,"blank":15},"file:///home/lars/Projects/music-downloader/src/create_custom_objects.py":{"language":"Python","code":80,"comment":3,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken_gtk.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken_cli.py":{"language":"Python","code":3,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/try-programming-interface.py":{"language":"Python","code":14,"comment":98,"blank":22}} \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/results.md b/.VSCodeCounter/2023-02-28_10-10-37/results.md deleted file mode 100644 index 93664c3..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/results.md +++ /dev/null @@ -1,40 +0,0 @@ -# Summary - -Date : 2023-02-28 10:10:37 - -Directory /home/lars/Projects/music-downloader/src - -Total : 63 files, 4150 codes, 1060 comments, 1228 blanks, all 6438 lines - -Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) - -## Languages -| language | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| Python | 60 | 3,852 | 1,060 | 1,207 | 6,119 | -| SQLite | 2 | 207 | 0 | 20 | 227 | -| Database | 1 | 91 | 0 | 1 | 92 | - -## Directories -| path | files | code | comment | blank | total | -| :--- | ---: | ---: | ---: | ---: | ---: | -| . | 63 | 4,150 | 1,060 | 1,228 | 6,438 | -| . (Files) | 9 | 253 | 125 | 74 | 452 | -| music_kraken | 51 | 3,858 | 929 | 1,145 | 5,932 | -| music_kraken (Files) | 2 | 60 | 10 | 28 | 98 | -| music_kraken/database | 8 | 914 | 343 | 286 | 1,543 | -| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | -| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | -| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | -| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | -| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | -| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | -| music_kraken/objects | 11 | 942 | 287 | 281 | 1,510 | -| music_kraken/pages | 4 | 446 | 150 | 129 | 725 | -| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | -| music_kraken/tagging | 2 | 59 | 4 | 22 | 85 | -| music_kraken/target | 2 | 41 | 7 | 20 | 68 | -| music_kraken/utils | 6 | 126 | 11 | 37 | 174 | -| tests | 3 | 39 | 6 | 9 | 54 | - -Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-02-28_10-10-37/results.txt b/.VSCodeCounter/2023-02-28_10-10-37/results.txt deleted file mode 100644 index 417c15a..0000000 --- a/.VSCodeCounter/2023-02-28_10-10-37/results.txt +++ /dev/null @@ -1,106 +0,0 @@ -Date : 2023-02-28 10:10:37 -Directory : /home/lars/Projects/music-downloader/src -Total : 63 files, 4150 codes, 1060 comments, 1228 blanks, all 6438 lines - -Languages -+----------+------------+------------+------------+------------+------------+ -| language | files | code | comment | blank | total | -+----------+------------+------------+------------+------------+------------+ -| Python | 60 | 3,852 | 1,060 | 1,207 | 6,119 | -| SQLite | 2 | 207 | 0 | 20 | 227 | -| Database | 1 | 91 | 0 | 1 | 92 | -+----------+------------+------------+------------+------------+------------+ - -Directories -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| path | files | code | comment | blank | total | -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ -| . | 63 | 4,150 | 1,060 | 1,228 | 6,438 | -| . (Files) | 9 | 253 | 125 | 74 | 452 | -| music_kraken | 51 | 3,858 | 929 | 1,145 | 5,932 | -| music_kraken (Files) | 2 | 60 | 10 | 28 | 98 | -| music_kraken/database | 8 | 914 | 343 | 286 | 1,543 | -| music_kraken/not_used_anymore | 14 | 1,063 | 117 | 322 | 1,502 | -| music_kraken/not_used_anymore (Files) | 3 | 129 | 13 | 39 | 181 | -| music_kraken/not_used_anymore/metadata | 5 | 561 | 70 | 153 | 784 | -| music_kraken/not_used_anymore/metadata (Files) | 3 | 516 | 64 | 139 | 719 | -| music_kraken/not_used_anymore/metadata/sources | 2 | 45 | 6 | 14 | 65 | -| music_kraken/not_used_anymore/sources | 6 | 373 | 34 | 130 | 537 | -| music_kraken/objects | 11 | 942 | 287 | 281 | 1,510 | -| music_kraken/pages | 4 | 446 | 150 | 129 | 725 | -| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | -| music_kraken/tagging | 2 | 59 | 4 | 22 | 85 | -| music_kraken/target | 2 | 41 | 7 | 20 | 68 | -| music_kraken/utils | 6 | 126 | 11 | 37 | 174 | -| tests | 3 | 39 | 6 | 9 | 54 | -+--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ - -Files -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| filename | language | code | comment | blank | total | -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ -| /home/lars/Projects/music-downloader/src/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/create_custom_objects.py | Python | 80 | 3 | 18 | 101 | -| /home/lars/Projects/music-downloader/src/metal_archives.py | Python | 48 | 4 | 15 | 67 | -| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | 57 | 8 | 25 | 90 | -| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 3 | 2 | 3 | 8 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py | Python | 18 | 0 | 5 | 23 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 123 | 23 | 52 | 198 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/database.py | Python | 84 | 48 | 26 | 158 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/object_cache.py | Python | 35 | 56 | 16 | 107 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/old_database.py | Python | 432 | 154 | 115 | 701 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/read.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/temp_database.py | Python | 12 | 0 | 8 | 20 | -| /home/lars/Projects/music-downloader/src/music_kraken/database/write.py | Python | 210 | 62 | 63 | 335 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py | Python | 0 | 0 | 3 | 3 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py | Python | 75 | 12 | 20 | 107 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py | Python | 54 | 1 | 16 | 71 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py | Python | 6 | 0 | 2 | 8 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py | Python | 257 | 24 | 65 | 346 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py | Python | 253 | 40 | 72 | 365 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py | Python | 3 | 0 | 2 | 5 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py | Python | 42 | 6 | 12 | 60 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py | Python | 115 | 16 | 42 | 173 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py | Python | 40 | 0 | 18 | 58 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py | Python | 136 | 9 | 37 | 182 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py | Python | 11 | 5 | 8 | 24 | -| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py | Python | 71 | 4 | 24 | 99 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 24 | 0 | 7 | 31 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/album.py | Python | 15 | 6 | 5 | 26 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/artist.py | Python | 18 | 0 | 5 | 23 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py | Python | 52 | 15 | 24 | 91 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py | Python | 53 | 57 | 20 | 130 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py | Python | 21 | 0 | 7 | 28 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 262 | 68 | 63 | 393 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 22 | 15 | 12 | 49 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 337 | 81 | 90 | 508 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 116 | 38 | 41 | 195 | -| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 22 | 7 | 7 | 36 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | 7 | 0 | 5 | 12 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | 73 | 68 | 27 | 168 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | 341 | 66 | 91 | 498 | -| /home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py | Python | 25 | 16 | 6 | 47 | -| /home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql | SQLite | 72 | 0 | 10 | 82 | -| /home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql | SQLite | 135 | 0 | 10 | 145 | -| /home/lars/Projects/music-downloader/src/music_kraken/tagging/__init__.py | Python | 8 | 0 | 2 | 10 | -| /home/lars/Projects/music-downloader/src/music_kraken/tagging/id3.py | Python | 51 | 4 | 20 | 75 | -| /home/lars/Projects/music-downloader/src/music_kraken/target/__init__.py | Python | 4 | 0 | 2 | 6 | -| /home/lars/Projects/music-downloader/src/music_kraken/target/set_target.py | Python | 37 | 7 | 18 | 62 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py | Python | 1 | 1 | 1 | 3 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py | Python | 3 | 0 | 1 | 4 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py | Python | 19 | 0 | 6 | 25 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py | Python | 39 | 2 | 17 | 58 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | 62 | 3 | 10 | 75 | -| /home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py | Python | 2 | 5 | 2 | 9 | -| /home/lars/Projects/music-downloader/src/music_kraken_cli.py | Python | 3 | 0 | 3 | 6 | -| /home/lars/Projects/music-downloader/src/music_kraken_gtk.py | Python | 3 | 0 | 2 | 5 | -| /home/lars/Projects/music-downloader/src/test.db | Database | 91 | 0 | 1 | 92 | -| /home/lars/Projects/music-downloader/src/tests/__init__.py | Python | 0 | 0 | 1 | 1 | -| /home/lars/Projects/music-downloader/src/tests/conftest.py | Python | 3 | 1 | 2 | 6 | -| /home/lars/Projects/music-downloader/src/tests/example_data_objects.py | Python | 36 | 5 | 6 | 47 | -| /home/lars/Projects/music-downloader/src/try-programming-interface.py | Python | 14 | 98 | 22 | 134 | -| /home/lars/Projects/music-downloader/src/try.py | Python | 1 | 0 | 3 | 4 | -| /home/lars/Projects/music-downloader/src/try_python.py | Python | 13 | 20 | 9 | 42 | -| Total | | 4,150 | 1,060 | 1,228 | 6,438 | -+--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/details.md b/.VSCodeCounter/2023-06-14_07-47-51/details.md new file mode 100644 index 0000000..6e15e50 --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/details.md @@ -0,0 +1,98 @@ +# Details + +Date : 2023-06-14 07:47:51 + +Directory /home/lars/Projects/music-downloader/src + +Total : 83 files, 5827 codes, 1152 comments, 1825 blanks, all 8804 lines + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | +| [src/__init__.py](/src/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/actual_donwload.py](/src/actual_donwload.py) | Python | 16 | 0 | 6 | 22 | +| [src/create_custom_objects.py](/src/create_custom_objects.py) | Python | 58 | 0 | 6 | 64 | +| [src/metal_archives.py](/src/metal_archives.py) | Python | 30 | 0 | 12 | 42 | +| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | 112 | 6 | 33 | 151 | +| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 98 | 3 | 23 | 124 | +| [src/music_kraken/audio/__init__.py](/src/music_kraken/audio/__init__.py) | Python | 7 | 0 | 3 | 10 | +| [src/music_kraken/audio/codec.py](/src/music_kraken/audio/codec.py) | Python | 25 | 0 | 8 | 33 | +| [src/music_kraken/audio/metadata.py](/src/music_kraken/audio/metadata.py) | Python | 60 | 4 | 24 | 88 | +| [src/music_kraken/cli/__init__.py](/src/music_kraken/cli/__init__.py) | Python | 2 | 0 | 0 | 2 | +| [src/music_kraken/cli/download/__init__.py](/src/music_kraken/cli/download/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/cli/download/shell.py](/src/music_kraken/cli/download/shell.py) | Python | 199 | 86 | 78 | 363 | +| [src/music_kraken/cli/options/__init__.py](/src/music_kraken/cli/options/__init__.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/cli/options/invidious/__init__.py](/src/music_kraken/cli/options/invidious/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/cli/options/invidious/shell.py](/src/music_kraken/cli/options/invidious/shell.py) | Python | 66 | 7 | 28 | 101 | +| [src/music_kraken/connection/__init__.py](/src/music_kraken/connection/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/connection/connection.py](/src/music_kraken/connection/connection.py) | Python | 168 | 1 | 30 | 199 | +| [src/music_kraken/connection/rotating.py](/src/music_kraken/connection/rotating.py) | Python | 27 | 3 | 14 | 44 | +| [src/music_kraken/database/__init__.py](/src/music_kraken/database/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/database/data_models.py](/src/music_kraken/database/data_models.py) | Python | 122 | 24 | 52 | 198 | +| [src/music_kraken/database/database.py](/src/music_kraken/database/database.py) | Python | 104 | 47 | 38 | 189 | +| [src/music_kraken/download/__init__.py](/src/music_kraken/download/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/download/multiple_options.py](/src/music_kraken/download/multiple_options.py) | Python | 69 | 0 | 31 | 100 | +| [src/music_kraken/download/page_attributes.py](/src/music_kraken/download/page_attributes.py) | Python | 67 | 1 | 29 | 97 | +| [src/music_kraken/download/results.py](/src/music_kraken/download/results.py) | Python | 62 | 7 | 26 | 95 | +| [src/music_kraken/download/search.py](/src/music_kraken/download/search.py) | Python | 124 | 24 | 56 | 204 | +| [src/music_kraken/objects/__init__.py](/src/music_kraken/objects/__init__.py) | Python | 14 | 0 | 5 | 19 | +| [src/music_kraken/objects/cache.py](/src/music_kraken/objects/cache.py) | Python | 37 | 56 | 18 | 111 | +| [src/music_kraken/objects/collection.py](/src/music_kraken/objects/collection.py) | Python | 91 | 31 | 39 | 161 | +| [src/music_kraken/objects/formatted_text.py](/src/music_kraken/objects/formatted_text.py) | Python | 50 | 10 | 19 | 79 | +| [src/music_kraken/objects/lyrics.py](/src/music_kraken/objects/lyrics.py) | Python | 25 | 0 | 7 | 32 | +| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 272 | 62 | 63 | 397 | +| [src/music_kraken/objects/option.py](/src/music_kraken/objects/option.py) | Python | 28 | 0 | 13 | 41 | +| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 71 | 35 | 35 | 141 | +| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 480 | 113 | 120 | 713 | +| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 93 | 16 | 33 | 142 | +| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 65 | 15 | 24 | 104 | +| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | 4 | 0 | 2 | 6 | +| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | 234 | 46 | 93 | 373 | +| [src/music_kraken/pages/encyclopaedia_metallum.py](/src/music_kraken/pages/encyclopaedia_metallum.py) | Python | 432 | 90 | 127 | 649 | +| [src/music_kraken/pages/musify.py](/src/music_kraken/pages/musify.py) | Python | 576 | 275 | 166 | 1,017 | +| [src/music_kraken/pages/preset.py](/src/music_kraken/pages/preset.py) | Python | 47 | 1 | 17 | 65 | +| [src/music_kraken/pages/youtube.py](/src/music_kraken/pages/youtube.py) | Python | 254 | 45 | 79 | 378 | +| [src/music_kraken/static_files/new_db.sql](/src/music_kraken/static_files/new_db.sql) | SQLite | 72 | 0 | 10 | 82 | +| [src/music_kraken/static_files/temp_database_structure.sql](/src/music_kraken/static_files/temp_database_structure.sql) | SQLite | 135 | 0 | 10 | 145 | +| [src/music_kraken/utils/__init__.py](/src/music_kraken/utils/__init__.py) | Python | 2 | 1 | 2 | 5 | +| [src/music_kraken/utils/config/__init__.py](/src/music_kraken/utils/config/__init__.py) | Python | 7 | 0 | 4 | 11 | +| [src/music_kraken/utils/config/audio.py](/src/music_kraken/utils/config/audio.py) | Python | 152 | 15 | 28 | 195 | +| [src/music_kraken/utils/config/base_classes.py](/src/music_kraken/utils/config/base_classes.py) | Python | 136 | 35 | 61 | 232 | +| [src/music_kraken/utils/config/config.py](/src/music_kraken/utils/config/config.py) | Python | 92 | 16 | 30 | 138 | +| [src/music_kraken/utils/config/connection.py](/src/music_kraken/utils/config/connection.py) | Python | 80 | 2 | 15 | 97 | +| [src/music_kraken/utils/config/logging.py](/src/music_kraken/utils/config/logging.py) | Python | 104 | 4 | 17 | 125 | +| [src/music_kraken/utils/config/misc.py](/src/music_kraken/utils/config/misc.py) | Python | 40 | 0 | 9 | 49 | +| [src/music_kraken/utils/config/paths.py](/src/music_kraken/utils/config/paths.py) | Python | 40 | 0 | 13 | 53 | +| [src/music_kraken/utils/enums/__init__.py](/src/music_kraken/utils/enums/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/utils/enums/album.py](/src/music_kraken/utils/enums/album.py) | Python | 16 | 6 | 5 | 27 | +| [src/music_kraken/utils/enums/source.py](/src/music_kraken/utils/enums/source.py) | Python | 40 | 1 | 8 | 49 | +| [src/music_kraken/utils/exception/__init__.py](/src/music_kraken/utils/exception/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/utils/exception/config.py](/src/music_kraken/utils/exception/config.py) | Python | 14 | 8 | 7 | 29 | +| [src/music_kraken/utils/exception/download.py](/src/music_kraken/utils/exception/download.py) | Python | 8 | 0 | 4 | 12 | +| [src/music_kraken/utils/functions.py](/src/music_kraken/utils/functions.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/utils/object_handeling.py](/src/music_kraken/utils/object_handeling.py) | Python | 19 | 0 | 6 | 25 | +| [src/music_kraken/utils/path_manager/__init__.py](/src/music_kraken/utils/path_manager/__init__.py) | Python | 2 | 0 | 2 | 4 | +| [src/music_kraken/utils/path_manager/config_directory.py](/src/music_kraken/utils/path_manager/config_directory.py) | Python | 4 | 0 | 4 | 8 | +| [src/music_kraken/utils/path_manager/locations.py](/src/music_kraken/utils/path_manager/locations.py) | Python | 16 | 0 | 9 | 25 | +| [src/music_kraken/utils/path_manager/music_directory.py](/src/music_kraken/utils/path_manager/music_directory.py) | Python | 36 | 9 | 14 | 59 | +| [src/music_kraken/utils/phonetic_compares.py](/src/music_kraken/utils/phonetic_compares.py) | Python | 39 | 2 | 17 | 58 | +| [src/music_kraken/utils/regex.py](/src/music_kraken/utils/regex.py) | Python | 1 | 0 | 2 | 3 | +| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | 66 | 22 | 22 | 110 | +| [src/music_kraken/utils/string_processing.py](/src/music_kraken/utils/string_processing.py) | Python | 16 | 5 | 11 | 32 | +| [src/music_kraken/utils/support_classes/__init__.py](/src/music_kraken/utils/support_classes/__init__.py) | Python | 4 | 0 | 1 | 5 | +| [src/music_kraken/utils/support_classes/default_target.py](/src/music_kraken/utils/support_classes/default_target.py) | Python | 56 | 0 | 15 | 71 | +| [src/music_kraken/utils/support_classes/download_result.py](/src/music_kraken/utils/support_classes/download_result.py) | Python | 69 | 0 | 21 | 90 | +| [src/music_kraken/utils/support_classes/query.py](/src/music_kraken/utils/support_classes/query.py) | Python | 24 | 0 | 9 | 33 | +| [src/music_kraken/utils/support_classes/thread_classes.py](/src/music_kraken/utils/support_classes/thread_classes.py) | Python | 8 | 0 | 4 | 12 | +| [src/music_kraken_cli.py](/src/music_kraken_cli.py) | Python | 3 | 0 | 3 | 6 | +| [src/music_kraken_gtk.py](/src/music_kraken_gtk.py) | Python | 3 | 0 | 2 | 5 | +| [src/musify_search.py](/src/musify_search.py) | Python | 38 | 0 | 14 | 52 | +| [src/tests/__init__.py](/src/tests/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/tests/conftest.py](/src/tests/conftest.py) | Python | 3 | 1 | 2 | 6 | +| [src/tests/test_building_objects.py](/src/tests/test_building_objects.py) | Python | 81 | 1 | 13 | 95 | +| [src/tests/test_download.py](/src/tests/test_download.py) | Python | 30 | 1 | 12 | 43 | +| [src/tests/test_objects.py](/src/tests/test_objects.py) | Python | 173 | 15 | 51 | 239 | + +[Summary](results.md) / Details / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/diff-details.md b/.VSCodeCounter/2023-06-14_07-47-51/diff-details.md new file mode 100644 index 0000000..bf042cd --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/diff-details.md @@ -0,0 +1,59 @@ +# Diff Details + +Date : 2023-06-14 07:47:51 + +Directory /home/lars/Projects/music-downloader/src + +Total : 44 files, -709 codes, -5 comments, -155 blanks, all -869 lines + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details + +## Files +| filename | language | code | comment | blank | total | +| :--- | :--- | ---: | ---: | ---: | ---: | +| [src/actual_donwload.py](/src/actual_donwload.py) | Python | -27 | -2 | -12 | -41 | +| [src/music_kraken/__init__.py](/src/music_kraken/__init__.py) | Python | -77 | -15 | -27 | -119 | +| [src/music_kraken/__main__.py](/src/music_kraken/__main__.py) | Python | 10 | 0 | 2 | 12 | +| [src/music_kraken/cli/__init__.py](/src/music_kraken/cli/__init__.py) | Python | 2 | 0 | 0 | 2 | +| [src/music_kraken/cli/download/__init__.py](/src/music_kraken/cli/download/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/cli/download/shell.py](/src/music_kraken/cli/download/shell.py) | Python | 199 | 86 | 78 | 363 | +| [src/music_kraken/cli/options/__init__.py](/src/music_kraken/cli/options/__init__.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/cli/options/invidious/__init__.py](/src/music_kraken/cli/options/invidious/__init__.py) | Python | 0 | 0 | 1 | 1 | +| [src/music_kraken/cli/options/invidious/shell.py](/src/music_kraken/cli/options/invidious/shell.py) | Python | 66 | 7 | 28 | 101 | +| [src/music_kraken/connection/connection.py](/src/music_kraken/connection/connection.py) | Python | 6 | 0 | 0 | 6 | +| [src/music_kraken/download/__init__.py](/src/music_kraken/download/__init__.py) | Python | -1 | 0 | 0 | -1 | +| [src/music_kraken/download/download.py](/src/music_kraken/download/download.py) | Python | -35 | 0 | -14 | -49 | +| [src/music_kraken/download/page_attributes.py](/src/music_kraken/download/page_attributes.py) | Python | 46 | 0 | 19 | 65 | +| [src/music_kraken/download/results.py](/src/music_kraken/download/results.py) | Python | 62 | 7 | 26 | 95 | +| [src/music_kraken/download/search.py](/src/music_kraken/download/search.py) | Python | -6 | 0 | 0 | -6 | +| [src/music_kraken/not_used_anymore/__init__.py](/src/music_kraken/not_used_anymore/__init__.py) | Python | 0 | 0 | -3 | -3 | +| [src/music_kraken/not_used_anymore/fetch_audio.py](/src/music_kraken/not_used_anymore/fetch_audio.py) | Python | -75 | -12 | -20 | -107 | +| [src/music_kraken/not_used_anymore/fetch_source.py](/src/music_kraken/not_used_anymore/fetch_source.py) | Python | -54 | -1 | -16 | -71 | +| [src/music_kraken/not_used_anymore/metadata/__init__.py](/src/music_kraken/not_used_anymore/metadata/__init__.py) | Python | -6 | 0 | -2 | -8 | +| [src/music_kraken/not_used_anymore/metadata/metadata_fetch.py](/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py) | Python | -257 | -24 | -65 | -346 | +| [src/music_kraken/not_used_anymore/metadata/metadata_search.py](/src/music_kraken/not_used_anymore/metadata/metadata_search.py) | Python | -253 | -40 | -72 | -365 | +| [src/music_kraken/not_used_anymore/metadata/sources/__init__.py](/src/music_kraken/not_used_anymore/metadata/sources/__init__.py) | Python | -3 | 0 | -2 | -5 | +| [src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py](/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py) | Python | -42 | -6 | -12 | -60 | +| [src/music_kraken/not_used_anymore/sources/__init__.py](/src/music_kraken/not_used_anymore/sources/__init__.py) | Python | 0 | 0 | -1 | -1 | +| [src/music_kraken/not_used_anymore/sources/genius.py](/src/music_kraken/not_used_anymore/sources/genius.py) | Python | -115 | -16 | -42 | -173 | +| [src/music_kraken/not_used_anymore/sources/local_files.py](/src/music_kraken/not_used_anymore/sources/local_files.py) | Python | -40 | 0 | -18 | -58 | +| [src/music_kraken/not_used_anymore/sources/musify.py](/src/music_kraken/not_used_anymore/sources/musify.py) | Python | -136 | -9 | -37 | -182 | +| [src/music_kraken/not_used_anymore/sources/source.py](/src/music_kraken/not_used_anymore/sources/source.py) | Python | -11 | -5 | -8 | -24 | +| [src/music_kraken/not_used_anymore/sources/youtube.py](/src/music_kraken/not_used_anymore/sources/youtube.py) | Python | -71 | -4 | -24 | -99 | +| [src/music_kraken/objects/metadata.py](/src/music_kraken/objects/metadata.py) | Python | 11 | 0 | 2 | 13 | +| [src/music_kraken/objects/parents.py](/src/music_kraken/objects/parents.py) | Python | 6 | 2 | 2 | 10 | +| [src/music_kraken/objects/song.py](/src/music_kraken/objects/song.py) | Python | 7 | 0 | 5 | 12 | +| [src/music_kraken/objects/source.py](/src/music_kraken/objects/source.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/objects/target.py](/src/music_kraken/objects/target.py) | Python | 2 | 0 | 1 | 3 | +| [src/music_kraken/pages/__init__.py](/src/music_kraken/pages/__init__.py) | Python | 1 | 0 | 1 | 2 | +| [src/music_kraken/pages/abstract.py](/src/music_kraken/pages/abstract.py) | Python | -123 | 12 | -10 | -121 | +| [src/music_kraken/pages/musify.py](/src/music_kraken/pages/musify.py) | Python | -53 | -14 | -21 | -88 | +| [src/music_kraken/pages/preset.py](/src/music_kraken/pages/preset.py) | Python | 4 | 0 | 1 | 5 | +| [src/music_kraken/pages/youtube.py](/src/music_kraken/pages/youtube.py) | Python | 229 | 29 | 73 | 331 | +| [src/music_kraken/utils/config/connection.py](/src/music_kraken/utils/config/connection.py) | Python | -1 | 0 | 0 | -1 | +| [src/music_kraken/utils/exception/download.py](/src/music_kraken/utils/exception/download.py) | Python | 8 | 0 | 4 | 12 | +| [src/music_kraken/utils/shared.py](/src/music_kraken/utils/shared.py) | Python | 3 | 0 | 1 | 4 | +| [src/music_kraken/utils/support_classes/__init__.py](/src/music_kraken/utils/support_classes/__init__.py) | Python | 1 | 0 | 0 | 1 | +| [src/music_kraken/utils/support_classes/thread_classes.py](/src/music_kraken/utils/support_classes/thread_classes.py) | Python | 8 | 0 | 4 | 12 | + +[Summary](results.md) / [Details](details.md) / [Diff Summary](diff.md) / Diff Details \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/diff.csv b/.VSCodeCounter/2023-06-14_07-47-51/diff.csv new file mode 100644 index 0000000..d001392 --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/diff.csv @@ -0,0 +1,46 @@ +"filename", "language", "Python", "comment", "blank", "total" +"/home/lars/Projects/music-downloader/src/actual_donwload.py", "Python", -27, -2, -12, -41 +"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", -77, -15, -27, -119 +"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 10, 0, 2, 12 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/__init__.py", "Python", 2, 0, 0, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/download/__init__.py", "Python", 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/download/shell.py", "Python", 199, 86, 78, 363 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/__init__.py", "Python", 3, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/__init__.py", "Python", 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/shell.py", "Python", 66, 7, 28, 101 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py", "Python", 6, 0, 0, 6 +"/home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py", "Python", -1, 0, 0, -1 +"/home/lars/Projects/music-downloader/src/music_kraken/download/download.py", "Python", -35, 0, -14, -49 +"/home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py", "Python", 46, 0, 19, 65 +"/home/lars/Projects/music-downloader/src/music_kraken/download/results.py", "Python", 62, 7, 26, 95 +"/home/lars/Projects/music-downloader/src/music_kraken/download/search.py", "Python", -6, 0, 0, -6 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py", "Python", 0, 0, -3, -3 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py", "Python", -75, -12, -20, -107 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py", "Python", -54, -1, -16, -71 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py", "Python", -6, 0, -2, -8 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py", "Python", -257, -24, -65, -346 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py", "Python", -253, -40, -72, -365 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py", "Python", -3, 0, -2, -5 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py", "Python", -42, -6, -12, -60 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py", "Python", 0, 0, -1, -1 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py", "Python", -115, -16, -42, -173 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py", "Python", -40, 0, -18, -58 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py", "Python", -136, -9, -37, -182 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py", "Python", -11, -5, -8, -24 +"/home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py", "Python", -71, -4, -24, -99 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 11, 0, 2, 13 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 6, 2, 2, 10 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 7, 0, 5, 12 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 3, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 2, 0, 1, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", 1, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", -123, 12, -10, -121 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py", "Python", -53, -14, -21, -88 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py", "Python", 4, 0, 1, 5 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py", "Python", 229, 29, 73, 331 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py", "Python", -1, 0, 0, -1 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/download.py", "Python", 8, 0, 4, 12 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", 3, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py", "Python", 1, 0, 0, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/thread_classes.py", "Python", 8, 0, 4, 12 +"Total", "-", -709, -5, -155, -869 \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/diff.md b/.VSCodeCounter/2023-06-14_07-47-51/diff.md new file mode 100644 index 0000000..f9c1a87 --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/diff.md @@ -0,0 +1,45 @@ +# Diff Summary + +Date : 2023-06-14 07:47:51 + +Directory /home/lars/Projects/music-downloader/src + +Total : 44 files, -709 codes, -5 comments, -155 blanks, all -869 lines + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| Python | 44 | -709 | -5 | -155 | -869 | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| . | 44 | -709 | -5 | -155 | -869 | +| . (Files) | 1 | -27 | -2 | -12 | -41 | +| music_kraken | 43 | -682 | -3 | -143 | -828 | +| music_kraken (Files) | 2 | -67 | -15 | -25 | -107 | +| music_kraken/cli | 6 | 270 | 93 | 109 | 472 | +| music_kraken/cli (Files) | 1 | 2 | 0 | 0 | 2 | +| music_kraken/cli/download | 2 | 199 | 86 | 79 | 364 | +| music_kraken/cli/options | 3 | 69 | 7 | 30 | 106 | +| music_kraken/cli/options (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/cli/options/invidious | 2 | 66 | 7 | 29 | 102 | +| music_kraken/connection | 1 | 6 | 0 | 0 | 6 | +| music_kraken/download | 5 | 66 | 7 | 31 | 104 | +| music_kraken/not_used_anymore | 14 | -1,063 | -117 | -322 | -1,502 | +| music_kraken/not_used_anymore (Files) | 3 | -129 | -13 | -39 | -181 | +| music_kraken/not_used_anymore/metadata | 5 | -561 | -70 | -153 | -784 | +| music_kraken/not_used_anymore/metadata (Files) | 3 | -516 | -64 | -139 | -719 | +| music_kraken/not_used_anymore/metadata/sources | 2 | -45 | -6 | -14 | -65 | +| music_kraken/not_used_anymore/sources | 6 | -373 | -34 | -130 | -537 | +| music_kraken/objects | 5 | 29 | 2 | 11 | 42 | +| music_kraken/pages | 5 | 58 | 27 | 44 | 129 | +| music_kraken/utils | 5 | 19 | 0 | 9 | 28 | +| music_kraken/utils (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/utils/config | 1 | -1 | 0 | 0 | -1 | +| music_kraken/utils/exception | 1 | 8 | 0 | 4 | 12 | +| music_kraken/utils/support_classes | 2 | 9 | 0 | 4 | 13 | + +[Summary](results.md) / [Details](details.md) / Diff Summary / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/diff.txt b/.VSCodeCounter/2023-06-14_07-47-51/diff.txt new file mode 100644 index 0000000..f87b711 --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/diff.txt @@ -0,0 +1,92 @@ +Date : 2023-06-14 07:47:51 +Directory : /home/lars/Projects/music-downloader/src +Total : 44 files, -709 codes, -5 comments, -155 blanks, all -869 lines + +Languages ++----------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++----------+------------+------------+------------+------------+------------+ +| Python | 44 | -709 | -5 | -155 | -869 | ++----------+------------+------------+------------+------------+------------+ + +Directories ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| . | 44 | -709 | -5 | -155 | -869 | +| . (Files) | 1 | -27 | -2 | -12 | -41 | +| music_kraken | 43 | -682 | -3 | -143 | -828 | +| music_kraken (Files) | 2 | -67 | -15 | -25 | -107 | +| music_kraken/cli | 6 | 270 | 93 | 109 | 472 | +| music_kraken/cli (Files) | 1 | 2 | 0 | 0 | 2 | +| music_kraken/cli/download | 2 | 199 | 86 | 79 | 364 | +| music_kraken/cli/options | 3 | 69 | 7 | 30 | 106 | +| music_kraken/cli/options (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/cli/options/invidious | 2 | 66 | 7 | 29 | 102 | +| music_kraken/connection | 1 | 6 | 0 | 0 | 6 | +| music_kraken/download | 5 | 66 | 7 | 31 | 104 | +| music_kraken/not_used_anymore | 14 | -1,063 | -117 | -322 | -1,502 | +| music_kraken/not_used_anymore (Files) | 3 | -129 | -13 | -39 | -181 | +| music_kraken/not_used_anymore/metadata | 5 | -561 | -70 | -153 | -784 | +| music_kraken/not_used_anymore/metadata (Files) | 3 | -516 | -64 | -139 | -719 | +| music_kraken/not_used_anymore/metadata/sources | 2 | -45 | -6 | -14 | -65 | +| music_kraken/not_used_anymore/sources | 6 | -373 | -34 | -130 | -537 | +| music_kraken/objects | 5 | 29 | 2 | 11 | 42 | +| music_kraken/pages | 5 | 58 | 27 | 44 | 129 | +| music_kraken/utils | 5 | 19 | 0 | 9 | 28 | +| music_kraken/utils (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/utils/config | 1 | -1 | 0 | 0 | -1 | +| music_kraken/utils/exception | 1 | 8 | 0 | 4 | 12 | +| music_kraken/utils/support_classes | 2 | 9 | 0 | 4 | 13 | ++--------------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ + +Files ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| /home/lars/Projects/music-downloader/src/actual_donwload.py | Python | -27 | -2 | -12 | -41 | +| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | -77 | -15 | -27 | -119 | +| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 10 | 0 | 2 | 12 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/__init__.py | Python | 2 | 0 | 0 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/download/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/download/shell.py | Python | 199 | 86 | 78 | 363 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/__init__.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/shell.py | Python | 66 | 7 | 28 | 101 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py | Python | 6 | 0 | 0 | 6 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py | Python | -1 | 0 | 0 | -1 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/download.py | Python | -35 | 0 | -14 | -49 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py | Python | 46 | 0 | 19 | 65 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/results.py | Python | 62 | 7 | 26 | 95 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/search.py | Python | -6 | 0 | 0 | -6 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/__init__.py | Python | 0 | 0 | -3 | -3 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_audio.py | Python | -75 | -12 | -20 | -107 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/fetch_source.py | Python | -54 | -1 | -16 | -71 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/__init__.py | Python | -6 | 0 | -2 | -8 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_fetch.py | Python | -257 | -24 | -65 | -346 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/metadata_search.py | Python | -253 | -40 | -72 | -365 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/__init__.py | Python | -3 | 0 | -2 | -5 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/metadata/sources/musicbrainz.py | Python | -42 | -6 | -12 | -60 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/__init__.py | Python | 0 | 0 | -1 | -1 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/genius.py | Python | -115 | -16 | -42 | -173 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/local_files.py | Python | -40 | 0 | -18 | -58 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/musify.py | Python | -136 | -9 | -37 | -182 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/source.py | Python | -11 | -5 | -8 | -24 | +| /home/lars/Projects/music-downloader/src/music_kraken/not_used_anymore/sources/youtube.py | Python | -71 | -4 | -24 | -99 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 11 | 0 | 2 | 13 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 6 | 2 | 2 | 10 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 7 | 0 | 5 | 12 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 2 | 0 | 1 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | -123 | 12 | -10 | -121 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py | Python | -53 | -14 | -21 | -88 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py | Python | 4 | 0 | 1 | 5 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py | Python | 229 | 29 | 73 | 331 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py | Python | -1 | 0 | 0 | -1 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/download.py | Python | 8 | 0 | 4 | 12 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py | Python | 1 | 0 | 0 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/thread_classes.py | Python | 8 | 0 | 4 | 12 | +| Total | | -709 | -5 | -155 | -869 | ++--------------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/results.csv b/.VSCodeCounter/2023-06-14_07-47-51/results.csv new file mode 100644 index 0000000..0d100fb --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/results.csv @@ -0,0 +1,85 @@ +"filename", "language", "Python", "SQLite", "comment", "blank", "total" +"/home/lars/Projects/music-downloader/src/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/actual_donwload.py", "Python", 16, 0, 0, 6, 22 +"/home/lars/Projects/music-downloader/src/create_custom_objects.py", "Python", 58, 0, 0, 6, 64 +"/home/lars/Projects/music-downloader/src/metal_archives.py", "Python", 30, 0, 0, 12, 42 +"/home/lars/Projects/music-downloader/src/music_kraken/__init__.py", "Python", 112, 0, 6, 33, 151 +"/home/lars/Projects/music-downloader/src/music_kraken/__main__.py", "Python", 98, 0, 3, 23, 124 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py", "Python", 7, 0, 0, 3, 10 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py", "Python", 25, 0, 0, 8, 33 +"/home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py", "Python", 60, 0, 4, 24, 88 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/__init__.py", "Python", 2, 0, 0, 0, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/download/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/download/shell.py", "Python", 199, 0, 86, 78, 363 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/__init__.py", "Python", 3, 0, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/shell.py", "Python", 66, 0, 7, 28, 101 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py", "Python", 1, 0, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py", "Python", 168, 0, 1, 30, 199 +"/home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py", "Python", 27, 0, 3, 14, 44 +"/home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py", "Python", 122, 0, 24, 52, 198 +"/home/lars/Projects/music-downloader/src/music_kraken/database/database.py", "Python", 104, 0, 47, 38, 189 +"/home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py", "Python", 1, 0, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py", "Python", 69, 0, 0, 31, 100 +"/home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py", "Python", 67, 0, 1, 29, 97 +"/home/lars/Projects/music-downloader/src/music_kraken/download/results.py", "Python", 62, 0, 7, 26, 95 +"/home/lars/Projects/music-downloader/src/music_kraken/download/search.py", "Python", 124, 0, 24, 56, 204 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py", "Python", 14, 0, 0, 5, 19 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py", "Python", 37, 0, 56, 18, 111 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py", "Python", 91, 0, 31, 39, 161 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py", "Python", 50, 0, 10, 19, 79 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py", "Python", 25, 0, 0, 7, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py", "Python", 272, 0, 62, 63, 397 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/option.py", "Python", 28, 0, 0, 13, 41 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py", "Python", 71, 0, 35, 35, 141 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/song.py", "Python", 480, 0, 113, 120, 713 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/source.py", "Python", 93, 0, 16, 33, 142 +"/home/lars/Projects/music-downloader/src/music_kraken/objects/target.py", "Python", 65, 0, 15, 24, 104 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py", "Python", 4, 0, 0, 2, 6 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py", "Python", 234, 0, 46, 93, 373 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py", "Python", 432, 0, 90, 127, 649 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py", "Python", 576, 0, 275, 166, 1017 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py", "Python", 47, 0, 1, 17, 65 +"/home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py", "Python", 254, 0, 45, 79, 378 +"/home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql", "SQLite", 0, 72, 0, 10, 82 +"/home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql", "SQLite", 0, 135, 0, 10, 145 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py", "Python", 2, 0, 1, 2, 5 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py", "Python", 7, 0, 0, 4, 11 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py", "Python", 152, 0, 15, 28, 195 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py", "Python", 136, 0, 35, 61, 232 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py", "Python", 92, 0, 16, 30, 138 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py", "Python", 80, 0, 2, 15, 97 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py", "Python", 104, 0, 4, 17, 125 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py", "Python", 40, 0, 0, 9, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py", "Python", 40, 0, 0, 13, 53 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py", "Python", 16, 0, 6, 5, 27 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py", "Python", 40, 0, 1, 8, 49 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py", "Python", 1, 0, 0, 1, 2 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py", "Python", 14, 0, 8, 7, 29 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/exception/download.py", "Python", 8, 0, 0, 4, 12 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py", "Python", 3, 0, 0, 1, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py", "Python", 19, 0, 0, 6, 25 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py", "Python", 2, 0, 0, 2, 4 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py", "Python", 4, 0, 0, 4, 8 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py", "Python", 16, 0, 0, 9, 25 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py", "Python", 36, 0, 9, 14, 59 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py", "Python", 39, 0, 2, 17, 58 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py", "Python", 1, 0, 0, 2, 3 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py", "Python", 66, 0, 22, 22, 110 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py", "Python", 16, 0, 5, 11, 32 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py", "Python", 4, 0, 0, 1, 5 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py", "Python", 56, 0, 0, 15, 71 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py", "Python", 69, 0, 0, 21, 90 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py", "Python", 24, 0, 0, 9, 33 +"/home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/thread_classes.py", "Python", 8, 0, 0, 4, 12 +"/home/lars/Projects/music-downloader/src/music_kraken_cli.py", "Python", 3, 0, 0, 3, 6 +"/home/lars/Projects/music-downloader/src/music_kraken_gtk.py", "Python", 3, 0, 0, 2, 5 +"/home/lars/Projects/music-downloader/src/musify_search.py", "Python", 38, 0, 0, 14, 52 +"/home/lars/Projects/music-downloader/src/tests/__init__.py", "Python", 0, 0, 0, 1, 1 +"/home/lars/Projects/music-downloader/src/tests/conftest.py", "Python", 3, 0, 1, 2, 6 +"/home/lars/Projects/music-downloader/src/tests/test_building_objects.py", "Python", 81, 0, 1, 13, 95 +"/home/lars/Projects/music-downloader/src/tests/test_download.py", "Python", 30, 0, 1, 12, 43 +"/home/lars/Projects/music-downloader/src/tests/test_objects.py", "Python", 173, 0, 15, 51, 239 +"Total", "-", 5620, 207, 1152, 1825, 8804 \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/results.json b/.VSCodeCounter/2023-06-14_07-47-51/results.json new file mode 100644 index 0000000..283137c --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/results.json @@ -0,0 +1 @@ +{"file:///home/lars/Projects/music-downloader/src/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/__init__.py":{"language":"Python","code":112,"comment":6,"blank":33},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/search.py":{"language":"Python","code":124,"comment":24,"blank":56},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py":{"language":"Python","code":67,"comment":1,"blank":29},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py":{"language":"Python","code":1,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/results.py":{"language":"Python","code":62,"comment":7,"blank":26},"file:///home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py":{"language":"Python","code":69,"comment":0,"blank":31},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py":{"language":"Python","code":7,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py":{"language":"Python","code":25,"comment":0,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py":{"language":"Python","code":60,"comment":4,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/database.py":{"language":"Python","code":104,"comment":47,"blank":38},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py":{"language":"Python","code":2,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py":{"language":"Python","code":122,"comment":24,"blank":52},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py":{"language":"Python","code":1,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py":{"language":"Python","code":14,"comment":8,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/exception/download.py":{"language":"Python","code":8,"comment":0,"blank":4},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py":{"language":"Python","code":16,"comment":5,"blank":11},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py":{"language":"Python","code":1,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py":{"language":"Python","code":19,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py":{"language":"Python","code":16,"comment":6,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py":{"language":"Python","code":40,"comment":1,"blank":8},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py":{"language":"Python","code":39,"comment":2,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py":{"language":"Python","code":4,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/thread_classes.py":{"language":"Python","code":8,"comment":0,"blank":4},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py":{"language":"Python","code":56,"comment":0,"blank":15},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py":{"language":"Python","code":24,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py":{"language":"Python","code":69,"comment":0,"blank":21},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py":{"language":"Python","code":40,"comment":0,"blank":13},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py":{"language":"Python","code":7,"comment":0,"blank":4},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py":{"language":"Python","code":80,"comment":2,"blank":15},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py":{"language":"Python","code":104,"comment":4,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py":{"language":"Python","code":92,"comment":16,"blank":30},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py":{"language":"Python","code":40,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py":{"language":"Python","code":136,"comment":35,"blank":61},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py":{"language":"Python","code":152,"comment":15,"blank":28},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py":{"language":"Python","code":66,"comment":22,"blank":22},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py":{"language":"Python","code":2,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py":{"language":"Python","code":36,"comment":9,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py":{"language":"Python","code":16,"comment":0,"blank":9},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py":{"language":"Python","code":254,"comment":45,"blank":79},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py":{"language":"Python","code":4,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py":{"language":"Python","code":47,"comment":1,"blank":17},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py":{"language":"Python","code":234,"comment":46,"blank":93},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py":{"language":"Python","code":432,"comment":90,"blank":127},"file:///home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py":{"language":"Python","code":576,"comment":275,"blank":166},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql":{"language":"SQLite","code":135,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py":{"language":"Python","code":168,"comment":1,"blank":30},"file:///home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql":{"language":"SQLite","code":72,"comment":0,"blank":10},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py":{"language":"Python","code":27,"comment":3,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py":{"language":"Python","code":14,"comment":0,"blank":5},"file:///home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py":{"language":"Python","code":1,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py":{"language":"Python","code":37,"comment":56,"blank":18},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/song.py":{"language":"Python","code":480,"comment":113,"blank":120},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py":{"language":"Python","code":272,"comment":62,"blank":63},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/source.py":{"language":"Python","code":93,"comment":16,"blank":33},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py":{"language":"Python","code":71,"comment":35,"blank":35},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/target.py":{"language":"Python","code":65,"comment":15,"blank":24},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/option.py":{"language":"Python","code":28,"comment":0,"blank":13},"file:///home/lars/Projects/music-downloader/src/music_kraken/__main__.py":{"language":"Python","code":98,"comment":3,"blank":23},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py":{"language":"Python","code":91,"comment":31,"blank":39},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/download/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py":{"language":"Python","code":25,"comment":0,"blank":7},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/options/__init__.py":{"language":"Python","code":3,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py":{"language":"Python","code":50,"comment":10,"blank":19},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/__init__.py":{"language":"Python","code":2,"comment":0,"blank":0},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/download/shell.py":{"language":"Python","code":199,"comment":86,"blank":78},"file:///home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/shell.py":{"language":"Python","code":66,"comment":7,"blank":28},"file:///home/lars/Projects/music-downloader/src/tests/__init__.py":{"language":"Python","code":0,"comment":0,"blank":1},"file:///home/lars/Projects/music-downloader/src/metal_archives.py":{"language":"Python","code":30,"comment":0,"blank":12},"file:///home/lars/Projects/music-downloader/src/tests/test_objects.py":{"language":"Python","code":173,"comment":15,"blank":51},"file:///home/lars/Projects/music-downloader/src/create_custom_objects.py":{"language":"Python","code":58,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/tests/conftest.py":{"language":"Python","code":3,"comment":1,"blank":2},"file:///home/lars/Projects/music-downloader/src/tests/test_download.py":{"language":"Python","code":30,"comment":1,"blank":12},"file:///home/lars/Projects/music-downloader/src/music_kraken_cli.py":{"language":"Python","code":3,"comment":0,"blank":3},"file:///home/lars/Projects/music-downloader/src/actual_donwload.py":{"language":"Python","code":16,"comment":0,"blank":6},"file:///home/lars/Projects/music-downloader/src/music_kraken_gtk.py":{"language":"Python","code":3,"comment":0,"blank":2},"file:///home/lars/Projects/music-downloader/src/tests/test_building_objects.py":{"language":"Python","code":81,"comment":1,"blank":13},"file:///home/lars/Projects/music-downloader/src/musify_search.py":{"language":"Python","code":38,"comment":0,"blank":14},"file:///home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py":{"language":"Python","code":4,"comment":0,"blank":4}} \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/results.md b/.VSCodeCounter/2023-06-14_07-47-51/results.md new file mode 100644 index 0000000..2bbd90d --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/results.md @@ -0,0 +1,46 @@ +# Summary + +Date : 2023-06-14 07:47:51 + +Directory /home/lars/Projects/music-downloader/src + +Total : 83 files, 5827 codes, 1152 comments, 1825 blanks, all 8804 lines + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) + +## Languages +| language | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| Python | 81 | 5,620 | 1,152 | 1,805 | 8,577 | +| SQLite | 2 | 207 | 0 | 20 | 227 | + +## Directories +| path | files | code | comment | blank | total | +| :--- | ---: | ---: | ---: | ---: | ---: | +| . | 83 | 5,827 | 1,152 | 1,825 | 8,804 | +| . (Files) | 7 | 148 | 0 | 44 | 192 | +| music_kraken | 71 | 5,392 | 1,134 | 1,702 | 8,228 | +| music_kraken (Files) | 2 | 210 | 9 | 56 | 275 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/cli | 6 | 270 | 93 | 109 | 472 | +| music_kraken/cli (Files) | 1 | 2 | 0 | 0 | 2 | +| music_kraken/cli/download | 2 | 199 | 86 | 79 | 364 | +| music_kraken/cli/options | 3 | 69 | 7 | 30 | 106 | +| music_kraken/cli/options (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/cli/options/invidious | 2 | 66 | 7 | 29 | 102 | +| music_kraken/connection | 3 | 196 | 4 | 45 | 245 | +| music_kraken/database | 3 | 226 | 71 | 91 | 388 | +| music_kraken/download | 5 | 323 | 32 | 143 | 498 | +| music_kraken/objects | 11 | 1,226 | 338 | 376 | 1,940 | +| music_kraken/pages | 6 | 1,547 | 457 | 484 | 2,488 | +| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | +| music_kraken/utils | 30 | 1,095 | 126 | 343 | 1,564 | +| music_kraken/utils (Files) | 7 | 146 | 30 | 61 | 237 | +| music_kraken/utils/config | 8 | 651 | 72 | 177 | 900 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 3 | 23 | 8 | 12 | 43 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 5 | 161 | 0 | 50 | 211 | +| tests | 5 | 287 | 18 | 79 | 384 | + +Summary / [Details](details.md) / [Diff Summary](diff.md) / [Diff Details](diff-details.md) \ No newline at end of file diff --git a/.VSCodeCounter/2023-06-14_07-47-51/results.txt b/.VSCodeCounter/2023-06-14_07-47-51/results.txt new file mode 100644 index 0000000..530258b --- /dev/null +++ b/.VSCodeCounter/2023-06-14_07-47-51/results.txt @@ -0,0 +1,132 @@ +Date : 2023-06-14 07:47:51 +Directory : /home/lars/Projects/music-downloader/src +Total : 83 files, 5827 codes, 1152 comments, 1825 blanks, all 8804 lines + +Languages ++----------+------------+------------+------------+------------+------------+ +| language | files | code | comment | blank | total | ++----------+------------+------------+------------+------------+------------+ +| Python | 81 | 5,620 | 1,152 | 1,805 | 8,577 | +| SQLite | 2 | 207 | 0 | 20 | 227 | ++----------+------------+------------+------------+------------+------------+ + +Directories ++------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| path | files | code | comment | blank | total | ++------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ +| . | 83 | 5,827 | 1,152 | 1,825 | 8,804 | +| . (Files) | 7 | 148 | 0 | 44 | 192 | +| music_kraken | 71 | 5,392 | 1,134 | 1,702 | 8,228 | +| music_kraken (Files) | 2 | 210 | 9 | 56 | 275 | +| music_kraken/audio | 3 | 92 | 4 | 35 | 131 | +| music_kraken/cli | 6 | 270 | 93 | 109 | 472 | +| music_kraken/cli (Files) | 1 | 2 | 0 | 0 | 2 | +| music_kraken/cli/download | 2 | 199 | 86 | 79 | 364 | +| music_kraken/cli/options | 3 | 69 | 7 | 30 | 106 | +| music_kraken/cli/options (Files) | 1 | 3 | 0 | 1 | 4 | +| music_kraken/cli/options/invidious | 2 | 66 | 7 | 29 | 102 | +| music_kraken/connection | 3 | 196 | 4 | 45 | 245 | +| music_kraken/database | 3 | 226 | 71 | 91 | 388 | +| music_kraken/download | 5 | 323 | 32 | 143 | 498 | +| music_kraken/objects | 11 | 1,226 | 338 | 376 | 1,940 | +| music_kraken/pages | 6 | 1,547 | 457 | 484 | 2,488 | +| music_kraken/static_files | 2 | 207 | 0 | 20 | 227 | +| music_kraken/utils | 30 | 1,095 | 126 | 343 | 1,564 | +| music_kraken/utils (Files) | 7 | 146 | 30 | 61 | 237 | +| music_kraken/utils/config | 8 | 651 | 72 | 177 | 900 | +| music_kraken/utils/enums | 3 | 56 | 7 | 14 | 77 | +| music_kraken/utils/exception | 3 | 23 | 8 | 12 | 43 | +| music_kraken/utils/path_manager | 4 | 58 | 9 | 29 | 96 | +| music_kraken/utils/support_classes | 5 | 161 | 0 | 50 | 211 | +| tests | 5 | 287 | 18 | 79 | 384 | ++------------------------------------------------------------------------------------------------+------------+------------+------------+------------+------------+ + +Files ++------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| filename | language | code | comment | blank | total | ++------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ +| /home/lars/Projects/music-downloader/src/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/actual_donwload.py | Python | 16 | 0 | 6 | 22 | +| /home/lars/Projects/music-downloader/src/create_custom_objects.py | Python | 58 | 0 | 6 | 64 | +| /home/lars/Projects/music-downloader/src/metal_archives.py | Python | 30 | 0 | 12 | 42 | +| /home/lars/Projects/music-downloader/src/music_kraken/__init__.py | Python | 112 | 6 | 33 | 151 | +| /home/lars/Projects/music-downloader/src/music_kraken/__main__.py | Python | 98 | 3 | 23 | 124 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/__init__.py | Python | 7 | 0 | 3 | 10 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/codec.py | Python | 25 | 0 | 8 | 33 | +| /home/lars/Projects/music-downloader/src/music_kraken/audio/metadata.py | Python | 60 | 4 | 24 | 88 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/__init__.py | Python | 2 | 0 | 0 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/download/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/download/shell.py | Python | 199 | 86 | 78 | 363 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/__init__.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/cli/options/invidious/shell.py | Python | 66 | 7 | 28 | 101 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/connection.py | Python | 168 | 1 | 30 | 199 | +| /home/lars/Projects/music-downloader/src/music_kraken/connection/rotating.py | Python | 27 | 3 | 14 | 44 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/data_models.py | Python | 122 | 24 | 52 | 198 | +| /home/lars/Projects/music-downloader/src/music_kraken/database/database.py | Python | 104 | 47 | 38 | 189 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/multiple_options.py | Python | 69 | 0 | 31 | 100 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/page_attributes.py | Python | 67 | 1 | 29 | 97 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/results.py | Python | 62 | 7 | 26 | 95 | +| /home/lars/Projects/music-downloader/src/music_kraken/download/search.py | Python | 124 | 24 | 56 | 204 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/__init__.py | Python | 14 | 0 | 5 | 19 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/cache.py | Python | 37 | 56 | 18 | 111 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/collection.py | Python | 91 | 31 | 39 | 161 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/formatted_text.py | Python | 50 | 10 | 19 | 79 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/lyrics.py | Python | 25 | 0 | 7 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/metadata.py | Python | 272 | 62 | 63 | 397 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/option.py | Python | 28 | 0 | 13 | 41 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/parents.py | Python | 71 | 35 | 35 | 141 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/song.py | Python | 480 | 113 | 120 | 713 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/source.py | Python | 93 | 16 | 33 | 142 | +| /home/lars/Projects/music-downloader/src/music_kraken/objects/target.py | Python | 65 | 15 | 24 | 104 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/__init__.py | Python | 4 | 0 | 2 | 6 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/abstract.py | Python | 234 | 46 | 93 | 373 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/encyclopaedia_metallum.py | Python | 432 | 90 | 127 | 649 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/musify.py | Python | 576 | 275 | 166 | 1,017 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/preset.py | Python | 47 | 1 | 17 | 65 | +| /home/lars/Projects/music-downloader/src/music_kraken/pages/youtube.py | Python | 254 | 45 | 79 | 378 | +| /home/lars/Projects/music-downloader/src/music_kraken/static_files/new_db.sql | SQLite | 72 | 0 | 10 | 82 | +| /home/lars/Projects/music-downloader/src/music_kraken/static_files/temp_database_structure.sql | SQLite | 135 | 0 | 10 | 145 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/__init__.py | Python | 2 | 1 | 2 | 5 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/__init__.py | Python | 7 | 0 | 4 | 11 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/audio.py | Python | 152 | 15 | 28 | 195 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/base_classes.py | Python | 136 | 35 | 61 | 232 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/config.py | Python | 92 | 16 | 30 | 138 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/connection.py | Python | 80 | 2 | 15 | 97 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/logging.py | Python | 104 | 4 | 17 | 125 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/misc.py | Python | 40 | 0 | 9 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/config/paths.py | Python | 40 | 0 | 13 | 53 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/album.py | Python | 16 | 6 | 5 | 27 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/enums/source.py | Python | 40 | 1 | 8 | 49 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/__init__.py | Python | 1 | 0 | 1 | 2 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/config.py | Python | 14 | 8 | 7 | 29 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/exception/download.py | Python | 8 | 0 | 4 | 12 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/functions.py | Python | 3 | 0 | 1 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/object_handeling.py | Python | 19 | 0 | 6 | 25 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/__init__.py | Python | 2 | 0 | 2 | 4 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/config_directory.py | Python | 4 | 0 | 4 | 8 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/locations.py | Python | 16 | 0 | 9 | 25 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/path_manager/music_directory.py | Python | 36 | 9 | 14 | 59 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/phonetic_compares.py | Python | 39 | 2 | 17 | 58 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/regex.py | Python | 1 | 0 | 2 | 3 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/shared.py | Python | 66 | 22 | 22 | 110 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/string_processing.py | Python | 16 | 5 | 11 | 32 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/__init__.py | Python | 4 | 0 | 1 | 5 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/default_target.py | Python | 56 | 0 | 15 | 71 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/download_result.py | Python | 69 | 0 | 21 | 90 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/query.py | Python | 24 | 0 | 9 | 33 | +| /home/lars/Projects/music-downloader/src/music_kraken/utils/support_classes/thread_classes.py | Python | 8 | 0 | 4 | 12 | +| /home/lars/Projects/music-downloader/src/music_kraken_cli.py | Python | 3 | 0 | 3 | 6 | +| /home/lars/Projects/music-downloader/src/music_kraken_gtk.py | Python | 3 | 0 | 2 | 5 | +| /home/lars/Projects/music-downloader/src/musify_search.py | Python | 38 | 0 | 14 | 52 | +| /home/lars/Projects/music-downloader/src/tests/__init__.py | Python | 0 | 0 | 1 | 1 | +| /home/lars/Projects/music-downloader/src/tests/conftest.py | Python | 3 | 1 | 2 | 6 | +| /home/lars/Projects/music-downloader/src/tests/test_building_objects.py | Python | 81 | 1 | 13 | 95 | +| /home/lars/Projects/music-downloader/src/tests/test_download.py | Python | 30 | 1 | 12 | 43 | +| /home/lars/Projects/music-downloader/src/tests/test_objects.py | Python | 173 | 15 | 51 | 239 | +| Total | | 5,827 | 1,152 | 1,825 | 8,804 | ++------------------------------------------------------------------------------------------------+----------+------------+------------+------------+------------+ \ No newline at end of file diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index 66a9ce6..4dd8a73 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -1,9 +1,29 @@ +from typing import List, Tuple import ffmpeg from ..utils.shared import BITRATE, AUDIO_FORMAT, CODEX_LOGGER as LOGGER from ..objects import Target +def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): + if not target.exists: + LOGGER.warning(f"Target doesn't exist: {target.file_path}") + return + + # https://stackoverflow.com/questions/50594412/cut-multiple-parts-of-a-video-with-ffmpeg + aselect_list: List[str] = [] + + start = 0 + for end, next_start in interval_list: + aselect_list.append(f"between(t,{start},{end})") + + start = next_start + + aselect_list.append(f"gte(t,{next_start})") + + select = f"aselect='{'+'.join(aselect_list)}',asetpts=N/SR/TB" + + def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = AUDIO_FORMAT): if not target.exists: LOGGER.warning(f"Target doesn't exist: {target.file_path}") diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index b02326b..d6e8225 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -80,6 +80,12 @@ class ConnectionSection(Section): value="false", description="Downloads the videos using the given instances." ) + + self.SPONSOR_BLOCK = BoolAttribute( + name="use_sponsor_block", + value="true", + description="Use sponsor block to remove adds or simmilar from the youtube videos." + ) self.attribute_list = [ self.USE_TOR, @@ -87,7 +93,8 @@ class ConnectionSection(Section): self.CHUNK_SIZE, self.SHOW_DOWNLOAD_ERRORS_THRESHOLD, self.INVIDIOUS_INSTANCE, - self.INVIDIOUS_PROXY_VIDEOS + self.INVIDIOUS_PROXY_VIDEOS, + self.SPONSOR_BLOCK ] super().__init__() From cc5c7809fb7ded3cf6c33deb302aab8d4b6a670f Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 14 Jun 2023 15:25:28 +0200 Subject: [PATCH 54/81] roughly finished implementing sponsorblock --- src/music_kraken/audio/codec.py | 16 ++++++++++++++- src/music_kraken/pages/abstract.py | 6 ++++++ src/music_kraken/pages/youtube.py | 32 +++++++++++++++++++++++++++++- src/music_kraken/utils/shared.py | 1 + 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index 4dd8a73..8e37b01 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -1,5 +1,6 @@ from typing import List, Tuple import ffmpeg +import subprocess from ..utils.shared import BITRATE, AUDIO_FORMAT, CODEX_LOGGER as LOGGER from ..objects import Target @@ -10,6 +11,12 @@ def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): LOGGER.warning(f"Target doesn't exist: {target.file_path}") return + temp_target = Target( + path=target._path, + file=str(target._file) + ".temp" + ) + target.copy_content(temp_target) + # https://stackoverflow.com/questions/50594412/cut-multiple-parts-of-a-video-with-ffmpeg aselect_list: List[str] = [] @@ -22,6 +29,13 @@ def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): aselect_list.append(f"gte(t,{next_start})") select = f"aselect='{'+'.join(aselect_list)}',asetpts=N/SR/TB" + + command = f"ffmpeg -i {str(temp_target.file_path)} -af \"{select}\" {str(target.file_path)}" + r = subprocess.run(command) + if r.stderr != "": + LOGGER.debug(r.stderr) + + temp_target.delete() def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = AUDIO_FORMAT): @@ -49,4 +63,4 @@ def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = LOGGER.debug(err) output_target.copy_content(target) - output_target.file_path.unlink() + output_target.delete() diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index cbeb46a..641020a 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -354,6 +354,9 @@ class Page: def _post_process_targets(self, song: Song, temp_target: Target) -> DownloadResult: correct_codec(temp_target) + + self.post_process_hook(song, temp_target) + write_metadata_to_target(song.metadata, temp_target) r = DownloadResult() @@ -368,5 +371,8 @@ class Page: return r + def post_process_hook(self, song: Song, temp_target: Target, **kwargs): + pass + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: return DownloadResult() diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 6f3a3cb..12f6b7b 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -2,6 +2,11 @@ from typing import List, Optional, Type, Tuple from urllib.parse import urlparse, urlunparse, parse_qs from enum import Enum +import sponsorblock + +from music_kraken.objects import Song, Target +from music_kraken.utils.support_classes import DownloadResult + from ..objects import Source, DatabaseObject from .abstract import Page from ..objects import ( @@ -15,9 +20,10 @@ from ..objects import ( FormattedText, ID3Timestamp ) +from ..audio.codec import remove_intervals from ..connection import Connection from ..utils.support_classes import DownloadResult -from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE +from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK """ @@ -137,6 +143,10 @@ class YouTube(Page): host="https://www.youtube.com/", logger=self.LOGGER ) + + # the stuff with the connection is, to ensure sponsorblock uses the proxies, my programm does + _sponsorblock_connection: Connection = Connection() + self.sponsorblock_client = sponsorblock.Client(session=_sponsorblock_connection.session) super().__init__(*args, **kwargs) @@ -375,3 +385,23 @@ class YouTube(Page): if target.stream_into(r, desc=desc): return DownloadResult(total=1) return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}") + + def post_process_hook(self, song: Song, temp_target: Target, **kwargs): + if not ENABLE_SPONSOR_BLOCK: + return + + # find the youtube id + source_list = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) + if len(source_list) <= 0: + self.LOGGER.warning(f"Couldn't find a youtube source in the post_processing_hook for {song.option_string}") + return + + source = source_list[0] + parsed = YouTubeUrl(source.url) + if parsed.url_type != YouTubeUrlType.VIDEO: + self.LOGGER.warning(f"{source.url} is no video url.") + return + + # call the sponsorblock api, and remove the segments from the audio + segments = self.sponsorblock_client.get_skip_segments(video_id=parsed.id) + remove_intervals(temp_target, [(segment.start, segment.end) for segment in segments]) diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 713f8ff..155599f 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -92,6 +92,7 @@ if TOR: 'https': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}' } INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.object_from_value +ENABLE_SPONSOR_BLOCK: bool = CONNECTION_SECTION.SPONSOR_BLOCK.object_from_value # size of the chunks that are streamed CHUNK_SIZE = CONNECTION_SECTION.CHUNK_SIZE.object_from_value From 5c31d6f7a8c3291b2dd6d6e8a5ff29e23fc026bc Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Wed, 14 Jun 2023 17:43:20 +0200 Subject: [PATCH 55/81] fixed and finish sponsor block --- .idea/modules.xml | 1 + .idea/music-downloader.iml | 1 + .idea/vcs.xml | 1 + requirements.txt | 4 +++- src/music_kraken/audio/codec.py | 18 +++++++++--------- src/music_kraken/pages/youtube.py | 22 ++++++++++++++-------- 6 files changed, 29 insertions(+), 18 deletions(-) diff --git a/.idea/modules.xml b/.idea/modules.xml index 7b56bcd..56fe61a 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -4,6 +4,7 @@ + \ No newline at end of file diff --git a/.idea/music-downloader.iml b/.idea/music-downloader.iml index 95b8e80..e7dba2c 100644 --- a/.idea/music-downloader.iml +++ b/.idea/music-downloader.iml @@ -7,5 +7,6 @@ + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index 35eb1dd..1a15e6d 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -2,5 +2,6 @@ + \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index a67d5a0..1771286 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,6 @@ tqdm~=4.65.0 peewee~=3.15.4 ffmpeg-python~=0.2.0 platformdirs~=3.2.0 -transliterate~=1.10.2 \ No newline at end of file +transliterate~=1.10.2 +sponsorblock~=0.1.3 +regex~=2022.9.13 \ No newline at end of file diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index 8e37b01..d2b9fd1 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -10,17 +10,17 @@ def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): if not target.exists: LOGGER.warning(f"Target doesn't exist: {target.file_path}") return - - temp_target = Target( + + output_target = Target( path=target._path, - file=str(target._file) + ".temp" + file=str(target._file) + "." + AUDIO_FORMAT ) - target.copy_content(temp_target) # https://stackoverflow.com/questions/50594412/cut-multiple-parts-of-a-video-with-ffmpeg aselect_list: List[str] = [] start = 0 + next_start = 0 for end, next_start in interval_list: aselect_list.append(f"between(t,{start},{end})") @@ -29,13 +29,13 @@ def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): aselect_list.append(f"gte(t,{next_start})") select = f"aselect='{'+'.join(aselect_list)}',asetpts=N/SR/TB" - - command = f"ffmpeg -i {str(temp_target.file_path)} -af \"{select}\" {str(target.file_path)}" - r = subprocess.run(command) + + r = subprocess.run(["ffmpeg", "-i", str(target.file_path), "-af", select, str(output_target.file_path)]) if r.stderr != "": LOGGER.debug(r.stderr) - - temp_target.delete() + + output_target.copy_content(target) + output_target.delete() def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = AUDIO_FORMAT): diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 12f6b7b..b3ce15b 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -3,11 +3,9 @@ from urllib.parse import urlparse, urlunparse, parse_qs from enum import Enum import sponsorblock +from sponsorblock.errors import HTTPException, NotFoundException -from music_kraken.objects import Song, Target -from music_kraken.utils.support_classes import DownloadResult - -from ..objects import Source, DatabaseObject +from ..objects import Source, DatabaseObject, Song, Target from .abstract import Page from ..objects import ( Artist, @@ -145,7 +143,7 @@ class YouTube(Page): ) # the stuff with the connection is, to ensure sponsorblock uses the proxies, my programm does - _sponsorblock_connection: Connection = Connection() + _sponsorblock_connection: Connection = Connection(host="https://sponsor.ajay.app/") self.sponsorblock_client = sponsorblock.Client(session=_sponsorblock_connection.session) super().__init__(*args, **kwargs) @@ -390,7 +388,7 @@ class YouTube(Page): if not ENABLE_SPONSOR_BLOCK: return - # find the youtube id + # find the YouTube id source_list = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) if len(source_list) <= 0: self.LOGGER.warning(f"Couldn't find a youtube source in the post_processing_hook for {song.option_string}") @@ -403,5 +401,13 @@ class YouTube(Page): return # call the sponsorblock api, and remove the segments from the audio - segments = self.sponsorblock_client.get_skip_segments(video_id=parsed.id) - remove_intervals(temp_target, [(segment.start, segment.end) for segment in segments]) + segments = [] + try: + segments = self.sponsorblock_client.get_skip_segments(parsed.id) + except NotFoundException: + self.LOGGER.debug(f"No sponsor found for the video {parsed.id}.") + except HTTPException as e: + self.LOGGER.warning(f"{e}") + + if len(segments) <= 0: + remove_intervals(temp_target, [(segment.start, segment.end) for segment in segments]) From 805017fea53a476cd55c7cb8da5925adf3b7388f Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 15 Jun 2023 09:58:48 +0200 Subject: [PATCH 56/81] added ffmpeg progressbar --- src/music_kraken/audio/codec.py | 77 +++++++++++++----------------- src/music_kraken/pages/abstract.py | 14 ++++-- src/music_kraken/pages/youtube.py | 22 +++------ 3 files changed, 49 insertions(+), 64 deletions(-) diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index d2b9fd1..43eed9f 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -1,47 +1,18 @@ from typing import List, Tuple -import ffmpeg +from tqdm import tqdm +from ffmpeg_progress_yield import FfmpegProgress import subprocess from ..utils.shared import BITRATE, AUDIO_FORMAT, CODEX_LOGGER as LOGGER from ..objects import Target -def remove_intervals(target: Target, interval_list: List[Tuple[float, float]]): +def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = AUDIO_FORMAT, interval_list: List[Tuple[float, float]] = None): if not target.exists: LOGGER.warning(f"Target doesn't exist: {target.file_path}") return - - output_target = Target( - path=target._path, - file=str(target._file) + "." + AUDIO_FORMAT - ) - # https://stackoverflow.com/questions/50594412/cut-multiple-parts-of-a-video-with-ffmpeg - aselect_list: List[str] = [] - - start = 0 - next_start = 0 - for end, next_start in interval_list: - aselect_list.append(f"between(t,{start},{end})") - - start = next_start - - aselect_list.append(f"gte(t,{next_start})") - - select = f"aselect='{'+'.join(aselect_list)}',asetpts=N/SR/TB" - - r = subprocess.run(["ffmpeg", "-i", str(target.file_path), "-af", select, str(output_target.file_path)]) - if r.stderr != "": - LOGGER.debug(r.stderr) - - output_target.copy_content(target) - output_target.delete() - - -def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = AUDIO_FORMAT): - if not target.exists: - LOGGER.warning(f"Target doesn't exist: {target.file_path}") - return + interval_list = interval_list or [] bitrate_b = int(bitrate_kb / 1024) @@ -49,18 +20,36 @@ def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = path=target._path, file=str(target._file) + "." + audio_format ) + + # get the select thingie + # https://stackoverflow.com/questions/50594412/cut-multiple-parts-of-a-video-with-ffmpeg + aselect_list: List[str] = [] + + start = 0 + next_start = 0 + for end, next_start in interval_list: + aselect_list.append(f"between(t,{start},{end})") + start = next_start + aselect_list.append(f"gte(t,{next_start})") + + select = f"aselect='{'+'.join(aselect_list)}',asetpts=N/SR/TB" + + # build the ffmpeg command + ffmpeg_command = [ + "ffmpeg", + "-i", str(target.file_path), + "-af", select, + "-b", str(bitrate_b), + str(output_target.file_path) + ] - stream = ffmpeg.input(target.file_path) - stream = stream.audio - stream = ffmpeg.output( - stream, - str(output_target.file_path), - audio_bitrate=bitrate_b, - format=audio_format - ) - out, err = ffmpeg.run(stream, quiet=True, overwrite_output=True) - if err != "": - LOGGER.debug(err) + # run the ffmpeg command with a progressbar + ff = FfmpegProgress(ffmpeg_command) + with tqdm(total=100, position=1, desc="ffmpeg") as pbar: + for progress in ff.run_command_with_progress(): + pbar.update(progress - pbar.n) + + LOGGER.debug(ff.stderr) output_target.copy_content(target) output_target.delete() diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 641020a..8f8d8dd 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -1,7 +1,7 @@ import logging import random from copy import copy -from typing import Optional, Union, Type, Dict, Set, List +from typing import Optional, Union, Type, Dict, Set, List, Tuple import threading from queue import Queue @@ -345,15 +345,16 @@ class Page: file=str(random.randint(0, 999999)) ) - r = self.download_song_to_target(source=sources[0], target=temp_target, desc=song.title) + source = sources[0] + r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) if not r.is_fatal_error: - r.merge(self._post_process_targets(song, temp_target)) + r.merge(self._post_process_targets(song, temp_target, source)) return r - def _post_process_targets(self, song: Song, temp_target: Target) -> DownloadResult: - correct_codec(temp_target) + def _post_process_targets(self, song: Song, temp_target: Target, source: Source) -> DownloadResult: + correct_codec(temp_target, interval_list=self.get_skip_intervals(song, source)) self.post_process_hook(song, temp_target) @@ -371,6 +372,9 @@ class Page: return r + def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]: + return [] + def post_process_hook(self, song: Song, temp_target: Target, **kwargs): pass diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index b3ce15b..de5cba6 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -5,6 +5,8 @@ from enum import Enum import sponsorblock from sponsorblock.errors import HTTPException, NotFoundException +from music_kraken.objects import Song, Source + from ..objects import Source, DatabaseObject, Song, Target from .abstract import Page from ..objects import ( @@ -18,7 +20,6 @@ from ..objects import ( FormattedText, ID3Timestamp ) -from ..audio.codec import remove_intervals from ..connection import Connection from ..utils.support_classes import DownloadResult from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK @@ -384,23 +385,15 @@ class YouTube(Page): return DownloadResult(total=1) return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}") - def post_process_hook(self, song: Song, temp_target: Target, **kwargs): + def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]: if not ENABLE_SPONSOR_BLOCK: - return + return [] - # find the YouTube id - source_list = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) - if len(source_list) <= 0: - self.LOGGER.warning(f"Couldn't find a youtube source in the post_processing_hook for {song.option_string}") - return - - source = source_list[0] parsed = YouTubeUrl(source.url) if parsed.url_type != YouTubeUrlType.VIDEO: self.LOGGER.warning(f"{source.url} is no video url.") - return + return [] - # call the sponsorblock api, and remove the segments from the audio segments = [] try: segments = self.sponsorblock_client.get_skip_segments(parsed.id) @@ -408,6 +401,5 @@ class YouTube(Page): self.LOGGER.debug(f"No sponsor found for the video {parsed.id}.") except HTTPException as e: self.LOGGER.warning(f"{e}") - - if len(segments) <= 0: - remove_intervals(temp_target, [(segment.start, segment.end) for segment in segments]) + + return [(segment.start, segment.end) for segment in segments] From 1d0e10a02215faeeed2eb8e97f56f0fe21eb447e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 15 Jun 2023 11:28:35 +0200 Subject: [PATCH 57/81] fixed the path naming --- src/actual_donwload.py | 2 +- src/music_kraken/objects/song.py | 4 ++ src/music_kraken/pages/abstract.py | 112 +++++++++++++++++++---------- src/music_kraken/pages/youtube.py | 6 -- 4 files changed, 81 insertions(+), 43 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 2c24a28..360fbea 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -18,4 +18,4 @@ if __name__ == "__main__": "8" ] - music_kraken.cli(genre="test", command_list=youtube_search) + music_kraken.cli(genre="test", command_list=direct_download) diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index e984376..6df2001 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -403,6 +403,10 @@ class Album(MainObject): :return: """ return len(self.artist_collection) > 1 + + @property + def album_type_string(self) -> str: + return self.album_type.value """ diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 8f8d8dd..5e96be8 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -2,8 +2,7 @@ import logging import random from copy import copy from typing import Optional, Union, Type, Dict, Set, List, Tuple -import threading -from queue import Queue +from string import Formatter import requests from bs4 import BeautifulSoup @@ -24,7 +23,7 @@ from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType from ..audio import write_metadata_to_target, correct_codec from ..utils import shared -from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE, THREADED +from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE, THREADED, AUDIO_FORMAT from ..utils.support_classes import Query, DownloadResult, DefaultTarget, EndThread, FinishedSearch @@ -32,6 +31,66 @@ INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] +class NamingDict(dict): + CUSTOM_KEYS: Dict[str, str] = { + "label": "label.name", + "artist": "artist.name", + "song": "song.title", + "isrc": "song.isrc", + "album": "album.title", + "album_type": "album.album_type_string" + } + + def __init__(self, values: dict, object_mappings: Dict[str, DatabaseObject] = None): + self.object_mappings: Dict[str, DatabaseObject] = object_mappings or dict() + + super().__init__(values) + self["audio_format"] = AUDIO_FORMAT + + def add_object(self, music_object: DatabaseObject): + self.object_mappings[type(music_object).__name__.lower()] = music_object + + def copy(self) -> dict: + return type(self)(super().copy(), self.object_mappings.copy()) + + def __getitem__(self, key: str) -> str: + return super().__getitem__(key) + + def default_value_for_name(self, name: str) -> str: + return f'Various {name.replace("_", " ").title()}' + + def __missing__(self, key: str) -> str: + """ + TODO + add proper logging + """ + + if "." not in key: + if key not in self.CUSTOM_KEYS: + return self.default_value_for_name(key) + + key = self.CUSTOM_KEYS[key] + + frag_list = key.split(".") + + object_name = frag_list[0].strip().lower() + attribute_name = frag_list[-1].strip().lower() + + if object_name not in self.object_mappings: + return self.default_value_for_name(attribute_name) + + music_object = self.object_mappings[object_name] + try: + value = getattr(music_object, attribute_name) + if value is None: + return self.default_value_for_name(attribute_name) + + return str(value) + + except AttributeError: + return self.default_value_for_name(attribute_name) + + def _clean_music_object(music_object: INDEPENDENT_DB_OBJECTS, collections: Dict[INDEPENDENT_DB_TYPES, Collection]): if type(music_object) == Label: return _clean_label(label=music_object, collections=collections) @@ -262,38 +321,37 @@ class Page: return Label() def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: - naming_objects = {"genre": genre} - + naming_dict: NamingDict = NamingDict({"genre": genre}) + def fill_naming_objects(naming_music_object: DatabaseObject): - nonlocal naming_objects + nonlocal naming_dict for collection_name in naming_music_object.UPWARDS_COLLECTION_ATTRIBUTES: collection: Collection = getattr(naming_music_object, collection_name) if collection.empty: continue - if collection.element_type in naming_objects: - continue dom_ordered_music_object: DatabaseObject = collection[0] + naming_dict.add_object(dom_ordered_music_object) return fill_naming_objects(dom_ordered_music_object) fill_naming_objects(music_object) - return self._download(music_object, naming_objects, download_all) + return self._download(music_object, naming_dict, download_all) - def _download(self, music_object: DatabaseObject, naming_objects: Dict[Union[Type[DatabaseObject], str], DatabaseObject], download_all: bool = False) -> DownloadResult: + def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False) -> DownloadResult: # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False if isinstance(music_object, Album): if not download_all and music_object.album_type in shared.ALBUM_TYPE_BLACKLIST: return DownloadResult() self.fetch_details(music_object=music_object, stop_at_level=2) - naming_objects[type(music_object)] = music_object + naming_dict.add_object(music_object) if isinstance(music_object, Song): - return self._download_song(music_object, naming_objects) + return self._download_song(music_object, naming_dict) download_result: DownloadResult = DownloadResult() @@ -302,35 +360,17 @@ class Page: sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: - download_result.merge(self._download(sub_ordered_music_object, naming_objects.copy(), download_all)) + download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all)) return download_result - def _download_song(self, song: Song, naming_objects: Dict[Type[DatabaseObject], Union[DatabaseObject, str]]): - name_attribute = DEFAULT_VALUES.copy() - - # song - if "genre" in naming_objects: - name_attribute["genre"] = naming_objects["genre"] - name_attribute["song"] = song.title - - if Album in naming_objects: - album: Album = naming_objects[Album] - name_attribute["album"] = album.title - name_attribute["album_type"] = album.album_type.value - - if Artist in naming_objects: - artist: Artist = naming_objects[Artist] - naming_objects["artist"] = artist.name - - if Label in naming_objects: - label: Label = naming_objects[Label] - naming_objects["label"] = label.name - + def _download_song(self, song: Song, naming_dict: NamingDict): + path_parts = Formatter().parse(DOWNLOAD_PATH) + file_parts = Formatter().parse(DOWNLOAD_FILE) new_target = Target( relative_to_music_dir=True, - path=DOWNLOAD_PATH.format(**name_attribute), - file=DOWNLOAD_FILE.format(**name_attribute) + path=DOWNLOAD_PATH.format(**{part[1]: naming_dict[part[1]] for part in path_parts}), + file=DOWNLOAD_FILE.format(**{part[1]: naming_dict[part[1]] for part in file_parts}) ) if song.target_collection.empty: diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index de5cba6..b343282 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -192,12 +192,6 @@ class YouTube(Page): return artist_list - def album_search(self, album: Album) -> List[Album]: - return [] - - def song_search(self, song: Song) -> List[Song]: - return [] - def _fetch_song_from_id(self, youtube_id: str) -> Tuple[Song, Optional[int]]: # https://yt.artemislena.eu/api/v1/videos/SULFl39UjgY r = self.connection.get(get_invidious_url(path=f"/api/v1/videos/{youtube_id}")) From fb3a1e22b01362fce0eb56628f48b6598be37c00 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 15 Jun 2023 11:48:04 +0200 Subject: [PATCH 58/81] removed redundand requests to the invidious api --- src/music_kraken/pages/abstract.py | 16 +++++++++++++--- src/music_kraken/pages/youtube.py | 2 ++ 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 5e96be8..d5e9b99 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -172,6 +172,9 @@ class Page: SOURCE_TYPE: SourcePages LOGGER = logging.getLogger("this shouldn't be used") + # set this to true, if all song details can also be fetched by fetching album details + NO_ADDITIONAL_DATA_FROM_SONG = False + def __init__(self): super().__init__() @@ -341,13 +344,20 @@ class Page: return self._download(music_object, naming_dict, download_all) - def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False) -> DownloadResult: + def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False, skip_details: bool = False) -> DownloadResult: + skip_next_details = skip_details + # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False if isinstance(music_object, Album): + if self.NO_ADDITIONAL_DATA_FROM_SONG: + skip_next_details = True + if not download_all and music_object.album_type in shared.ALBUM_TYPE_BLACKLIST: return DownloadResult() - self.fetch_details(music_object=music_object, stop_at_level=2) + if not isinstance(music_object, Song) or not self.NO_ADDITIONAL_DATA_FROM_SONG: + self.fetch_details(music_object=music_object, stop_at_level=2) + naming_dict.add_object(music_object) if isinstance(music_object, Song): @@ -360,7 +370,7 @@ class Page: sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: - download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all)) + download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all, skip_details=skip_next_details)) return download_result diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index b343282..4e5b9fe 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -132,6 +132,8 @@ class YouTube(Page): SOURCE_TYPE = SourcePages.YOUTUBE LOGGER = YOUTUBE_LOGGER + NO_ADDITIONAL_DATA_FROM_SONG = True + def __init__(self, *args, **kwargs): self.connection: Connection = Connection( host=get_invidious_url(), From 1324802db7610563957dd1c368741f6bb75e8347 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Thu, 15 Jun 2023 11:53:55 +0200 Subject: [PATCH 59/81] removed unused settings --- src/music_kraken/pages/abstract.py | 5 +- src/music_kraken/utils/config/audio.py | 40 ----------- src/music_kraken/utils/shared.py | 9 --- .../utils/support_classes/__init__.py | 1 - .../utils/support_classes/default_target.py | 70 ------------------- 5 files changed, 2 insertions(+), 123 deletions(-) delete mode 100644 src/music_kraken/utils/support_classes/default_target.py diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index d5e9b99..01e6ec3 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -23,9 +23,8 @@ from ..utils.enums.source import SourcePages from ..utils.enums.album import AlbumType from ..audio import write_metadata_to_target, correct_codec from ..utils import shared -from ..utils.shared import DEFAULT_VALUES, DOWNLOAD_PATH, DOWNLOAD_FILE, THREADED, AUDIO_FORMAT -from ..utils.support_classes import Query, DownloadResult, DefaultTarget, EndThread, FinishedSearch - +from ..utils.shared import DOWNLOAD_PATH, DOWNLOAD_FILE, AUDIO_FORMAT +from ..utils.support_classes import Query, DownloadResult INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] diff --git a/src/music_kraken/utils/config/audio.py b/src/music_kraken/utils/config/audio.py index 1d253ef..969bb67 100644 --- a/src/music_kraken/utils/config/audio.py +++ b/src/music_kraken/utils/config/audio.py @@ -116,42 +116,7 @@ ID3.1: {', '.join(_sorted_id3_1_formats)} description="The filename of the audio file." ) - self.DEFAULT_GENRE = StringAttribute( - name="default_genre", - value="Various Genre", - description="The default value for the genre field." - ) - self.DEFAULT_LABEL = StringAttribute( - name="default_label", - value="Various Labels", - description="The Label refers to a lable that signs artists." - ) - - self.DEFAULT_ARTIST = StringAttribute( - name="default_artist", - value="Various Artists", - description="You know Various Artist." - ) - - self.DEFAULT_ALBUM = StringAttribute( - name="default_album", - value="Various Album", - description="This value will hopefully not be used." - ) - - self.DEFAULT_SONG = StringAttribute( - name="default_song", - value="Various Song", - description="If it has to fall back to this value, something did go really wrong." - ) - - self.DEFAULT_ALBUM_TYPE = StringAttribute( - name="default_album_type", - value="Other", - description="Weirdly enough I barely see this used in file systems." - ) - self.ALBUM_TYPE_BLACKLIST = AlbumTypeListAttribute( name="album_type_blacklist", description="Music Kraken ignores all albums of those types.\n" @@ -181,11 +146,6 @@ There are multiple fields, you can use for the path and file name: """.strip()), self.DOWNLOAD_PATH, self.DOWNLOAD_FILE, - self.DEFAULT_ALBUM_TYPE, - self.DEFAULT_ARTIST, - self.DEFAULT_GENRE, - self.DEFAULT_LABEL, - self.DEFAULT_SONG, self.ALBUM_TYPE_BLACKLIST, ] super().__init__() diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 155599f..9d1574f 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -67,15 +67,6 @@ AUDIO_FORMAT = AUDIO_SECTION.AUDIO_FORMAT.object_from_value DOWNLOAD_PATH = AUDIO_SECTION.DOWNLOAD_PATH.object_from_value DOWNLOAD_FILE = AUDIO_SECTION.DOWNLOAD_FILE.object_from_value -DEFAULT_VALUES = { - "genre": AUDIO_SECTION.DEFAULT_GENRE.object_from_value, - "label": AUDIO_SECTION.DEFAULT_LABEL.object_from_value, - "artist": AUDIO_SECTION.DEFAULT_ARTIST.object_from_value, - "album": AUDIO_SECTION.DEFAULT_ALBUM.object_from_value, - "song": AUDIO_SECTION.DEFAULT_SONG.object_from_value, - "album_type": AUDIO_SECTION.DEFAULT_ALBUM_TYPE.object_from_value, - "audio_format": AUDIO_FORMAT -} TOR: bool = CONNECTION_SECTION.USE_TOR.object_from_value PROXIES_LIST: List[Dict[str, str]] = CONNECTION_SECTION.PROXIES.object_from_value diff --git a/src/music_kraken/utils/support_classes/__init__.py b/src/music_kraken/utils/support_classes/__init__.py index 0789e87..4a04f30 100644 --- a/src/music_kraken/utils/support_classes/__init__.py +++ b/src/music_kraken/utils/support_classes/__init__.py @@ -1,4 +1,3 @@ -from .default_target import DefaultTarget from .download_result import DownloadResult from .query import Query from .thread_classes import EndThread, FinishedSearch diff --git a/src/music_kraken/utils/support_classes/default_target.py b/src/music_kraken/utils/support_classes/default_target.py deleted file mode 100644 index f5417df..0000000 --- a/src/music_kraken/utils/support_classes/default_target.py +++ /dev/null @@ -1,70 +0,0 @@ -from dataclasses import dataclass - -from ...utils.shared import DOWNLOAD_PATH, DOWNLOAD_FILE, DEFAULT_VALUES -from ...utils.string_processing import fit_to_file_system -from ...objects import ( - Song, - Album, - Artist, - Target, - Label -) - - -@dataclass -class DefaultTarget: - genre: str = DEFAULT_VALUES["genre"] - label: str = DEFAULT_VALUES["label"] - artist: str = DEFAULT_VALUES["artist"] - album: str = DEFAULT_VALUES["album"] - album_type: str = DEFAULT_VALUES["album_type"] - song: str = DEFAULT_VALUES["song"] - audio_format: str = DEFAULT_VALUES["audio_format"] - - def __setattr__(self, __name: str, __value: str) -> None: - if __name in DEFAULT_VALUES: - if type(__value) != str: - return - - if self.__getattribute__(__name) == DEFAULT_VALUES[__name]: - super().__setattr__(__name, fit_to_file_system(__value)) - return - - super().__setattr__(__name, __value) - - @property - def target(self) -> Target: - return Target( - relative_to_music_dir=True, - path=DOWNLOAD_PATH.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album, - song=self.song, album_type=self.album_type, audio_format=self.audio_format), - file=DOWNLOAD_FILE.format(genre=self.genre, label=self.label, artist=self.artist, album=self.album, - song=self.song, album_type=self.album_type, audio_format=self.audio_format) - ) - - def song_object(self, song: Song): - self.song = song.title - self.genre = song.genre - - if not song.album_collection.empty: - self.album_object(song.album_collection[0]) - if not song.main_artist_collection.empty: - self.artist_object(song.main_artist_collection[0]) - - def album_object(self, album: Album): - self.album = album.title - self.album_type = album.album_type.value - - if not album.artist_collection.empty: - self.artist_object(album.artist_collection[0]) - if not album.label_collection.empty: - self.label_object(album.label_collection[0]) - - def artist_object(self, artist: Artist): - self.artist = artist.name - - if not artist.label_collection.empty: - self.label_object(artist.label_collection[0]) - - def label_object(self, label: Label): - self.label = label.name From 61973450239012698c29fa8db5f462e9049c6293 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:46:01 +0200 Subject: [PATCH 60/81] fixed some minor stuff --- src/music_kraken/audio/codec.py | 4 ++-- src/music_kraken/utils/config/config.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index 43eed9f..f8ba952 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -45,9 +45,9 @@ def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = # run the ffmpeg command with a progressbar ff = FfmpegProgress(ffmpeg_command) - with tqdm(total=100, position=1, desc="ffmpeg") as pbar: + with tqdm(total=100, desc="ffmpeg") as pbar: for progress in ff.run_command_with_progress(): - pbar.update(progress - pbar.n) + pbar.update(int(progress)-pbar.n) LOGGER.debug(ff.stderr) diff --git a/src/music_kraken/utils/config/config.py b/src/music_kraken/utils/config/config.py index f4e7282..1ecfe28 100644 --- a/src/music_kraken/utils/config/config.py +++ b/src/music_kraken/utils/config/config.py @@ -58,7 +58,7 @@ class Config: self._name_section_map[name] = element self._length += 1 - def set_name_to_value(self, name: str, value: str): + def set_name_to_value(self, name: str, value: str, silent: bool = True): """ :raises SettingValueError, SettingNotFound: :param name: @@ -66,6 +66,9 @@ class Config: :return: """ if name not in self._name_section_map: + if silent: + LOGGER.warning(f"The setting \"{name}\" is either deprecated, or doesn't exist.") + return raise SettingNotFound(setting_name=name) LOGGER.debug(f"setting: {name} value: {value}") From 9bba470bd1c4684d40440d9b5ca7d97944e63d70 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Thu, 15 Jun 2023 18:22:00 +0200 Subject: [PATCH 61/81] connection --- src/music_kraken/connection/connection.py | 52 ++++++++++++++++++++++- src/music_kraken/objects/target.py | 5 ++- src/music_kraken/pages/youtube.py | 13 +++--- 3 files changed, 61 insertions(+), 9 deletions(-) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 46c22de..d243bb4 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -4,9 +4,11 @@ from urllib.parse import urlparse, urlunsplit, ParseResult import logging import requests +from tqdm import tqdm from .rotating import RotatingProxy -from ..utils.shared import PROXIES_LIST +from ..utils.shared import PROXIES_LIST, CHUNK_SIZE +from ..objects import Target class Connection: @@ -196,3 +198,51 @@ class Connection: self.LOGGER.warning(f"Max attempts ({self.TRIES}) exceeded for: GET:{url}") self.LOGGER.warning(f"payload: {json}") return r + + def stream_into( + self, + url: str, + target: Target, + description: str = "download", + refer_from_origin: bool = True, + accepted_response_codes: set = None, + timeout: float = None, + headers: dict = None, + raw_url: bool = False, + **kwargs + ): + r = self._request( + request=self.session.get, + try_count=0, + accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + url=url, + timeout=timeout, + headers=headers, + raw_url=raw_url, + refer_from_origin=refer_from_origin, + stream=True, + **kwargs + ) + + if r is None: + return False + + target.create_path() + total_size = int(r.headers.get('content-length')) + + with target.open("wb") as f: + try: + """ + https://en.wikipedia.org/wiki/Kilobyte + > The internationally recommended unit symbol for the kilobyte is kB. + """ + with tqdm(total=total_size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: + + for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + size = f.write(chunk) + t.update(size) + return True + + except requests.exceptions.ConnectionError: + self.LOGGER.error("Stream timed out.") + return False diff --git a/src/music_kraken/objects/target.py b/src/music_kraken/objects/target.py index 8c70755..b979951 100644 --- a/src/music_kraken/objects/target.py +++ b/src/music_kraken/objects/target.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import List, Tuple +from typing import List, Tuple, TextIO import requests from tqdm import tqdm @@ -98,6 +98,9 @@ class Target(DatabaseObject): except requests.exceptions.Timeout: shared.DOWNLOAD_LOGGER.error("Stream timed out.") return False + + def open(self, file_mode: str, **kwargs) -> TextIO: + return self.file_path.open(file_mode, **kwargs) def delete(self): self.file_path.unlink(missing_ok=True) diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 4e5b9fe..797480e 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -5,8 +5,6 @@ from enum import Enum import sponsorblock from sponsorblock.errors import HTTPException, NotFoundException -from music_kraken.objects import Song, Source - from ..objects import Source, DatabaseObject, Song, Target from .abstract import Page from ..objects import ( @@ -373,11 +371,9 @@ class YouTube(Page): endpoint = audio_format["url"] - r = self.download_connection.get(endpoint, stream=True, raw_url=True) - if r is None: - return DownloadResult(error_message=f"Couldn't connect to {endpoint}") + self.download_connection.stream_into(endpoint, target, description=desc, raw_url=True) - if target.stream_into(r, desc=desc): + if self.download_connection.get(endpoint, stream=True, raw_url=True): return DownloadResult(total=1) return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}") @@ -397,5 +393,8 @@ class YouTube(Page): self.LOGGER.debug(f"No sponsor found for the video {parsed.id}.") except HTTPException as e: self.LOGGER.warning(f"{e}") - + + if len(segments) > 0: + print(f"Removing {len(segments)} interruptions in the audio...") + return [(segment.start, segment.end) for segment in segments] From bf9269161dad289774ebd920a3eb08964d9886f9 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Thu, 15 Jun 2023 18:30:02 +0200 Subject: [PATCH 62/81] implement skipping of file --- src/music_kraken/pages/abstract.py | 30 ++++++++++++++++++++++-------- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 01e6ec3..3ebf040 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -375,16 +375,17 @@ class Page: def _download_song(self, song: Song, naming_dict: NamingDict): path_parts = Formatter().parse(DOWNLOAD_PATH) - file_parts = Formatter().parse(DOWNLOAD_FILE) + file_parts = Formatter().parse(DOWNLOAD_FILE) new_target = Target( relative_to_music_dir=True, path=DOWNLOAD_PATH.format(**{part[1]: naming_dict[part[1]] for part in path_parts}), file=DOWNLOAD_FILE.format(**{part[1]: naming_dict[part[1]] for part in file_parts}) ) - + + if song.target_collection.empty: song.target_collection.append(new_target) - + sources = song.source_collection.get_sources_from_page(self.SOURCE_TYPE) if len(sources) == 0: return DownloadResult(error_message=f"No source found for {song.title} as {self.__class__.__name__}.") @@ -393,17 +394,30 @@ class Page: path=shared.TEMP_DIR, file=str(random.randint(0, 999999)) ) - + + found_on_disc = False + target: Target + for target in song.target_collection: + if target.exists: + target.copy_content(temp_target) + found_on_disc = True + break + + r = DownloadResult(1) + source = sources[0] - r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) + if not found_on_disc: + r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) + else: + self.LOGGER.info(f"{song.option_string} already exists, thus not downloading again.") if not r.is_fatal_error: - r.merge(self._post_process_targets(song, temp_target, source)) + r.merge(self._post_process_targets(song, temp_target, [] if found_on_disc else self.get_skip_intervals(song, source))) return r - def _post_process_targets(self, song: Song, temp_target: Target, source: Source) -> DownloadResult: - correct_codec(temp_target, interval_list=self.get_skip_intervals(song, source)) + def _post_process_targets(self, song: Song, temp_target: Target, interval_list: List) -> DownloadResult: + correct_codec(temp_target, interval_list=interval_list) self.post_process_hook(song, temp_target) From c2e070a89932c99a7aa4e58685bf65b9335c3870 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 16 Jun 2023 08:39:30 +0200 Subject: [PATCH 63/81] some stuff --- src/music_kraken/__init__.py | 9 --------- src/music_kraken/cli/informations/__init__.py | 1 + src/music_kraken/cli/informations/paths.py | 17 +++++++++++++++++ 3 files changed, 18 insertions(+), 9 deletions(-) create mode 100644 src/music_kraken/cli/informations/__init__.py create mode 100644 src/music_kraken/cli/informations/paths.py diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index 147be6a..28d505a 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -61,15 +61,6 @@ def exit_message(): print("See you soon! :3") -def paths(): - print(f"Temp dir:\t{shared.TEMP_DIR}\n" - f"Music dir:\t{shared.MUSIC_DIR}\n" - f"Log file:\t{shared.LOG_PATH}") - print() - print_cute_message() - print() - - def settings( name: str = None, value: str = None, diff --git a/src/music_kraken/cli/informations/__init__.py b/src/music_kraken/cli/informations/__init__.py new file mode 100644 index 0000000..be110bf --- /dev/null +++ b/src/music_kraken/cli/informations/__init__.py @@ -0,0 +1 @@ +from .paths import print_paths \ No newline at end of file diff --git a/src/music_kraken/cli/informations/paths.py b/src/music_kraken/cli/informations/paths.py new file mode 100644 index 0000000..d6bff30 --- /dev/null +++ b/src/music_kraken/cli/informations/paths.py @@ -0,0 +1,17 @@ +from ...utils.path_manager import LOCATIONS +from ...utils import shared + + +def all_paths(): + return { + "Temp dir": LOCATIONS.TEMP_DIRECTORY, + "Music dir": LOCATIONS.MUSIC_DIRECTORY, + "Log file": shared.LOG_PATH, + "Conf dir": LOCATIONS.CONFIG_DIRECTORY, + "Conf file": LOCATIONS.CONFIG_FILE + } + + +def print_paths(): + for name, path in all_paths().items(): + print(f"{name}:\t{path}") \ No newline at end of file From b20d7fcc76ee5bd95f995daf98e94f91f1fa84bd Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 16 Jun 2023 10:31:57 +0200 Subject: [PATCH 64/81] added result history, with the appropriate setting. --- src/music_kraken/cli/download/shell.py | 27 ++- src/music_kraken/download/__init__.py | 1 - src/music_kraken/download/multiple_options.py | 100 --------- src/music_kraken/download/search.py | 203 ------------------ src/music_kraken/utils/config/misc.py | 17 ++ src/music_kraken/utils/shared.py | 3 + 6 files changed, 46 insertions(+), 305 deletions(-) delete mode 100644 src/music_kraken/download/multiple_options.py delete mode 100644 src/music_kraken/download/search.py diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/download/shell.py index 1db4e6d..91f546f 100644 --- a/src/music_kraken/cli/download/shell.py +++ b/src/music_kraken/cli/download/shell.py @@ -2,7 +2,7 @@ from typing import Set, Type, Dict, List from pathlib import Path import re -from ...utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX +from ...utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX, ENABLE_RESULT_HISTORY, HISTORY_LENGTH from ...utils.regex import URL_PATTERN from ...utils.string_processing import fit_to_file_system from ...utils.support_classes import Query, DownloadResult @@ -168,6 +168,7 @@ class Shell: self.option_digits: int = option_digits self.current_results: Results = SearchResults + self._result_history: List[Results] = [] self.genre = genre or get_genre() @@ -197,7 +198,26 @@ class Shell: print() def set_current_options(self, current_options: Results): + if ENABLE_RESULT_HISTORY: + self._result_history.append(current_options) + + if HISTORY_LENGTH != -1: + if len(self._result_history) > HISTORY_LENGTH: + self._result_history.pop(0) + self.current_results = current_options + + def previous_option(self) -> bool: + if not ENABLE_RESULT_HISTORY: + print("History is turned of.\nGo to settings, and change the value at 'result_history' to 'true'.") + return False + + if len(self._result_history) <= 1: + print(f"No results in history.") + return False + self._result_history.pop() + self.current_results = self._result_history[-1] + return True def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) @@ -340,6 +360,11 @@ class Shell: self.print_current_options() return False + if processed_input == "..": + if self.previous_option(): + self.print_current_options() + return False + if processed_input.startswith("s: "): self.search(input_str[3:]) return False diff --git a/src/music_kraken/download/__init__.py b/src/music_kraken/download/__init__.py index 327a571..e69de29 100644 --- a/src/music_kraken/download/__init__.py +++ b/src/music_kraken/download/__init__.py @@ -1 +0,0 @@ -from .search import Search diff --git a/src/music_kraken/download/multiple_options.py b/src/music_kraken/download/multiple_options.py deleted file mode 100644 index 4585952..0000000 --- a/src/music_kraken/download/multiple_options.py +++ /dev/null @@ -1,100 +0,0 @@ -from collections import defaultdict -from typing import Tuple, List, Dict, Type - -from . import page_attributes -from ..pages import Page -from ..objects import Options, DatabaseObject, Source - - -class MultiPageOptions: - def __init__( - self, - max_displayed_options: int = 10, - option_digits: int = 3, - derived_from: DatabaseObject = None - ) -> None: - self.max_displayed_options = max_displayed_options - self.option_digits: int = option_digits - - self._length = 0 - self._current_option_dict: Dict[Type[Page], Options] = defaultdict(lambda: Options()) - - self._derive_from = derived_from - - def __getitem__(self, key: Type[Page]): - return self._current_option_dict[key] - - def __setitem__(self, key: Type[Page], value: Options): - self._current_option_dict[key] = value - - self._length = 0 - for key in self._current_option_dict: - self._length += 1 - - def __len__(self) -> int: - return self._length - - def get_page_str(self, page: Page) -> str: - page_name_fill = "-" - max_page_len = 21 - - return f"({page_attributes.PAGE_NAME_MAP[page]}) ------------------------{type(page).__name__:{page_name_fill}<{max_page_len}}------------" - - def string_from_all_pages(self) -> str: - if self._length == 1: - for key in self._current_option_dict: - return self.string_from_single_page(key) - - lines: List[str] = [] - - j = 0 - for page, options in self._current_option_dict.items(): - lines.append(self.get_page_str(page)) - - i = -1 - - option_obj: DatabaseObject - for i, option_obj in enumerate(options): - if i >= self.max_displayed_options: - lines.append("...") - break - - lines.append(f"{j + i:0{self.option_digits}} {option_obj.option_string}") - - j += i + 1 - - return "\n".join(lines) - - def choose_from_all_pages(self, index: int) -> Tuple[DatabaseObject, Type[Page]]: - if self._length == 1: - for key in self._current_option_dict: - return self.choose_from_single_page(key, index), key - - sum_of_length = 0 - for page, options in self._current_option_dict.items(): - option_len = min((len(options), self.max_displayed_options)) - - index_of_list = index - sum_of_length - - if index_of_list < option_len: - return options[index_of_list], page - - sum_of_length += option_len - - raise IndexError("index is out of range") - - def string_from_single_page(self, page: Type[Page]) -> str: - lines: List[str] = [self.get_page_str(page)] - - option_obj: DatabaseObject - for i, option_obj in enumerate(self._current_option_dict[page]): - lines.append(f"{i:0{self.option_digits}} {option_obj.option_string}") - - return "\n".join(lines) - - def choose_from_single_page(self, page: Type[Page], index: int) -> DatabaseObject: - return self._current_option_dict[page][index] - - def __repr__(self) -> str: - return self.string_from_all_pages() - \ No newline at end of file diff --git a/src/music_kraken/download/search.py b/src/music_kraken/download/search.py deleted file mode 100644 index ce667a2..0000000 --- a/src/music_kraken/download/search.py +++ /dev/null @@ -1,203 +0,0 @@ -from typing import Tuple, List, Set, Type, Optional, Dict - -from .page_attributes import Pages -from .multiple_options import MultiPageOptions -from ..pages.abstract import Page -from ..utils.support_classes import DownloadResult, Query -from ..objects import DatabaseObject, Source, Artist, Song, Album -from ..utils.enums.source import SourcePages - - -class Search: - def __init__( - self, - exclude_pages: Set[Type[Page]] = None, - exclude_shady: bool = False, - max_displayed_options: int = 10, - option_digits: int = 3, - ) -> None: - self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) - - self.max_displayed_options = max_displayed_options - self.option_digits: int = option_digits - - self._option_history: List[MultiPageOptions] = [] - - self._current_option: MultiPageOptions = self.next_options() - - - def __repr__(self): - return self._current_option.__repr__() - - def next_options(self, derive_from: DatabaseObject = None) -> MultiPageOptions: - mpo = MultiPageOptions( - max_displayed_options=self.max_displayed_options, - option_digits=self.option_digits, - derived_from=derive_from - ) - - self._option_history.append(mpo) - self._current_option = mpo - - return mpo - - def _previous_options(self) -> MultiPageOptions: - self._option_history.pop() - self._current_option = self._option_history[-1] - - return self._option_history[-1] - - def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: - song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) - album = None if not "r" in key_text else Album(title=key_text["r"], dynamic=True) - artist = None if not "a" in key_text else Artist(name=key_text["a"], dynamic=True) - - if song is not None: - song.album_collection.append(album) - song.main_artist_collection.append(artist) - return Query(raw_query=query, music_object=song) - - if album is not None: - album.artist_collection.append(artist) - return Query(raw_query=query, music_object=album) - - if artist is not None: - return Query(raw_query=query, music_object=artist) - - - def search(self, query: str): - """ - # The Query - - You can define a new parameter with "#", - the letter behind it defines the *type* of parameter, - followed by a space "#a Psychonaut 4 #r Tired, Numb and #t Drop by Drop" - if no # is in the query it gets treated as "unspecified query" - - doesn't set derived_from thus, - can't download right after - """ - - special_characters = "#\\" - query = query + " " - - key_text = {} - - skip_next = False - escape_next = False - new_text = "" - latest_key: str = None - for i in range(len(query) - 1): - current_char = query[i] - next_char = query[i+1] - - if skip_next: - skip_next = False - continue - - if escape_next: - new_text += current_char - escape_next = False - - # escaping - if current_char == "\\": - if next_char in special_characters: - escape_next = True - continue - - if current_char == "#": - if latest_key is not None: - key_text[latest_key] = new_text - new_text = "" - - latest_key = next_char - skip_next = True - continue - - new_text += current_char - - if latest_key is not None: - key_text[latest_key] = new_text - - - parsed_query: Query = self._process_parsed(key_text, query) - - - for page in self.pages: - self._current_option[page].extend(page.search(parsed_query)) - - def choose_page(self, page: Type[Page]): - """ - doesn't set derived_from thus, - can't download right after - """ - - if page not in page_attributes.ALL_PAGES: - raise ValueError(f"Page \"{page.__name__}\" does not exist in page_attributes.ALL_PAGES") - - prev_mpo = self._current_option - mpo = self.next_options() - - mpo[page] = prev_mpo[page] - - def get_page_from_query(self, query: str) -> Optional[Type[Page]]: - """ - query can be for example: - "a" or "EncyclopaediaMetallum" to choose a page - """ - - page = page_attributes.NAME_PAGE_MAP.get(query.lower().strip()) - - if page in self.pages: - return page - - def _get_page_from_source(self, source: Source) -> Optional[Type[Page]]: - return page_attributes.SOURCE_PAGE_MAP.get(source.page_enum) - - def choose_index(self, index: int): - db_object, page = self._current_option.choose_from_all_pages(index=index) - - music_object = self.fetch_details(db_object) - mpo = self.next_options(derive_from=music_object) - - mpo[page] = music_object.options - - def goto_previous(self): - try: - self._previous_options() - except IndexError: - pass - - def search_url(self, url: str) -> bool: - """ - sets derived_from, thus - can download directly after - """ - - source = Source.match_url(url=url, referer_page=SourcePages.MANUAL) - if source is None: - return False - - new_object = self.fetch_source(source) - if new_object is None: - return False - - page = page_attributes.SOURCE_PAGE_MAP[source.page_enum] - mpo = self.next_options(derive_from=new_object) - mpo[page] = new_object.options - - return True - - def download_chosen(self, genre: str = None, download_all: bool = False, **kwargs) -> DownloadResult: - if self._current_option._derive_from is None: - return DownloadResult(error_message="No option has been chosen yet.") - - source: Source - for source in self._current_option._derive_from.source_collection: - page = self._get_page_from_source(source=source) - - if page in self.audio_pages: - return page.download(music_object=self._current_option._derive_from, genre=genre, download_all=download_all) - - return DownloadResult(error_message=f"Didn't find a source for {self._current_option._derive_from.option_string}.") - diff --git a/src/music_kraken/utils/config/misc.py b/src/music_kraken/utils/config/misc.py index 021f154..3f5f24d 100644 --- a/src/music_kraken/utils/config/misc.py +++ b/src/music_kraken/utils/config/misc.py @@ -3,6 +3,21 @@ from .base_classes import Section, IntAttribute, ListAttribute, BoolAttribute class MiscSection(Section): def __init__(self): + self.ENABLE_RESULT_HISTORY = BoolAttribute( + name="result_history", + description="If enabled, you can go back to the previous results.\n" + "The consequence is a higher meory consumption, because every result is saved.", + value="false" + ) + + self.HISTORY_LENGTH = IntAttribute( + name="history_length", + description="You can choose how far back you can go in the result history.\n" + "The further you choose to be able to go back, the higher the memory usage.\n" + "'-1' removes the Limit entirely.", + value="8" + ) + self.HAPPY_MESSAGES = ListAttribute( name="happy_messages", description="Just some nice and wholesome messages.\n" @@ -37,6 +52,8 @@ class MiscSection(Section): ) self.attribute_list = [ + self.ENABLE_RESULT_HISTORY, + self.HISTORY_LENGTH, self.HAPPY_MESSAGES, self.MODIFY_GC, self.ID_BITS diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 9d1574f..addcca7 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -99,3 +99,6 @@ SORT_BY_ALBUM_TYPE = AUDIO_SECTION.SORT_BY_ALBUM_TYPE.object_from_value ALBUM_TYPE_BLACKLIST: Set[AlbumType] = set(AUDIO_SECTION.ALBUM_TYPE_BLACKLIST.object_from_value) THREADED = False + +ENABLE_RESULT_HISTORY: bool = MISC_SECTION.ENABLE_RESULT_HISTORY.object_from_value +HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value \ No newline at end of file From f471c6a72be032973bac35040384f9f9d1934a6e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 16 Jun 2023 10:43:35 +0200 Subject: [PATCH 65/81] implemented the deletion of unnecesarry details in not chosen objects, for the sake of memory efficiency in the history --- src/music_kraken/cli/download/shell.py | 5 ++++- src/music_kraken/download/results.py | 7 +++++++ src/music_kraken/objects/collection.py | 3 +++ src/music_kraken/objects/parents.py | 4 ++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/download/shell.py index 91f546f..03e804f 100644 --- a/src/music_kraken/cli/download/shell.py +++ b/src/music_kraken/cli/download/shell.py @@ -167,7 +167,7 @@ class Shell: self.max_displayed_options = max_displayed_options self.option_digits: int = option_digits - self.current_results: Results = SearchResults + self.current_results: Results = None self._result_history: List[Results] = [] self.genre = genre or get_genre() @@ -296,6 +296,9 @@ class Shell: page: Type[Page] music_object: DatabaseObject + if self.current_results is not None: + self.current_results.delete_details(index) + try: page, music_object = self.current_results.get_music_object_by_index(index) except KeyError: diff --git a/src/music_kraken/download/results.py b/src/music_kraken/download/results.py index fa41369..631ad48 100644 --- a/src/music_kraken/download/results.py +++ b/src/music_kraken/download/results.py @@ -29,6 +29,13 @@ class Results: def get_music_object_by_index(self, index: int) -> Tuple[Type[Page], DatabaseObject]: # if this throws a key error, either the formated generator needs to be iterated, or the option doesn't exist. return self._page_by_index[index], self._by_index[index] + + def delete_details(self, exclude_index: int): + for index, music_object in self._by_index.items(): + if index == exclude_index: + continue + + music_object.strip_details() class SearchResults(Results): diff --git a/src/music_kraken/objects/collection.py b/src/music_kraken/objects/collection.py index 2daf0fa..675fd7e 100644 --- a/src/music_kraken/objects/collection.py +++ b/src/music_kraken/objects/collection.py @@ -158,3 +158,6 @@ class Collection: @property def empty(self) -> bool: return len(self._data) == 0 + + def clear(self): + self.__init__(element_type=self.element_type) diff --git a/src/music_kraken/objects/parents.py b/src/music_kraken/objects/parents.py index 9185a82..3121d4f 100644 --- a/src/music_kraken/objects/parents.py +++ b/src/music_kraken/objects/parents.py @@ -95,6 +95,10 @@ class DatabaseObject: if override or getattr(self, simple_attribute) == default_value: setattr(self, simple_attribute, getattr(other, simple_attribute)) + def strip_details(self): + for collection in type(self).DOWNWARDS_COLLECTION_ATTRIBUTES: + getattr(self, collection).clear() + @property def metadata(self) -> Metadata: return Metadata() From 7277b7e512ffa85d7f3bdfc52ca97d06ed2f3738 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Fri, 16 Jun 2023 12:26:02 +0200 Subject: [PATCH 66/81] updated conecting behaviour --- src/music_kraken/connection/connection.py | 44 ++++++++++++++++--- src/music_kraken/pages/abstract.py | 21 +++++---- .../utils/support_classes/download_result.py | 8 +++- 3 files changed, 59 insertions(+), 14 deletions(-) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index d243bb4..81dc21a 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -8,6 +8,7 @@ from tqdm import tqdm from .rotating import RotatingProxy from ..utils.shared import PROXIES_LIST, CHUNK_SIZE +from ..utils.support_classes import DownloadResult from ..objects import Target @@ -209,8 +210,12 @@ class Connection: timeout: float = None, headers: dict = None, raw_url: bool = False, + chunk_size: int = CHUNK_SIZE, + try_count: int = 0, + progress: int = 0, **kwargs - ): + ) -> DownloadResult: + r = self._request( request=self.session.get, try_count=0, @@ -225,10 +230,13 @@ class Connection: ) if r is None: - return False + return DownloadResult(error_message=f"Could not establish connection to: {url}") target.create_path() total_size = int(r.headers.get('content-length')) + progress = 0 + + retry = False with target.open("wb") as f: try: @@ -236,13 +244,39 @@ class Connection: https://en.wikipedia.org/wiki/Kilobyte > The internationally recommended unit symbol for the kilobyte is kB. """ + with tqdm(total=total_size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: - for chunk in r.iter_content(chunk_size=CHUNK_SIZE): + for chunk in r.iter_content(chunk_size=chunk_size): size = f.write(chunk) + progress += size t.update(size) return True except requests.exceptions.ConnectionError: - self.LOGGER.error("Stream timed out.") - return False + if try_count >= self.TRIES: + self.LOGGER.warning(f"Stream timed out at \"{url}\": to many retries, aborting.") + return DownloadResult(error_message=f"Stream timed out from {url}, reducing the chunksize might help.") + + self.LOGGER.warning(f"Stream timed out at \"{url}\": ({try_count}-{self.TRIES})") + retry = True + + + finally: + if total_size > progress or retry: + return self.stream_into( + url = url, + target = target, + description = description, + try_count=try_count+1, + progress=progress, + accepted_response_code=accepted_response_codes, + timeout=timeout, + headers=headers, + raw_url=raw_url, + refer_from_origin=refer_from_origin, + chunk_size=chunk_size, + **kwargs + ) + + return DownloadResult() diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 3ebf040..65ad85f 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -394,22 +394,26 @@ class Page: path=shared.TEMP_DIR, file=str(random.randint(0, 999999)) ) + + r = DownloadResult(1) found_on_disc = False target: Target for target in song.target_collection: if target.exists: - target.copy_content(temp_target) found_on_disc = True - break - - r = DownloadResult(1) + + r.found_on_disk += 1 + r.add_target(target) + + if found_on_disc: + self.LOGGER.info(f"{song.option_string} already exists, thus not downloading again.") + return r source = sources[0] - if not found_on_disc: - r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) - else: - self.LOGGER.info(f"{song.option_string} already exists, thus not downloading again.") + + r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) + if not r.is_fatal_error: r.merge(self._post_process_targets(song, temp_target, [] if found_on_disc else self.get_skip_intervals(song, source))) @@ -432,6 +436,7 @@ class Page: r.add_target(target) temp_target.delete() + r.sponsor_segments += len(interval_list) return r diff --git a/src/music_kraken/utils/support_classes/download_result.py b/src/music_kraken/utils/support_classes/download_result.py index 8ba7e57..a47a054 100644 --- a/src/music_kraken/utils/support_classes/download_result.py +++ b/src/music_kraken/utils/support_classes/download_result.py @@ -12,8 +12,10 @@ UNIT_DIVISOR = 1024 class DownloadResult: total: int = 0 fail: int = 0 + sponsor_segments: int = 0 error_message: str = None total_size = 0 + found_on_disk: int = 0 _error_message_list: List[str] = field(default_factory=list) @@ -71,7 +73,9 @@ class DownloadResult: self.fail += other.fail self._error_message_list.extend(other._error_message_list) + self.sponsor_segments += other.sponsor_segments self.total_size += other.total_size + self.found_on_disk += other.found_on_disk def __str__(self): if self.is_fatal_error: @@ -79,7 +83,9 @@ class DownloadResult: head = f"{self.fail} from {self.total} downloads failed:\n" \ f"successrate:\t{int(self.success_percentage * 100)}%\n" \ f"failrate:\t{int(self.failure_percentage * 100)}%\n" \ - f"total size:\t{self.formated_size}" + f"total size:\t{self.formated_size}\n" \ + f"skipped segments:\t{self.sponsor_segments}" \ + f"found on disc:\t{self.found_on_disk}" if not self.is_mild_failure: return head From 3cba909c05dad1bb04b06d72d469fb71071bec9f Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Fri, 16 Jun 2023 23:27:08 +0200 Subject: [PATCH 67/81] made streaming more stable --- documentation/html/youtube/channel_api.md | 471 ++++++++++++++++++ .../html/youtube/channel_api_request.md | 204 ++++++++ src/actual_donwload.py | 4 +- src/music_kraken/audio/codec.py | 4 +- src/music_kraken/connection/connection.py | 93 ++-- src/music_kraken/pages/youtube.py | 3 - src/music_kraken/utils/config/audio.py | 2 +- .../utils/support_classes/download_result.py | 10 +- 8 files changed, 738 insertions(+), 53 deletions(-) create mode 100644 documentation/html/youtube/channel_api.md create mode 100644 documentation/html/youtube/channel_api_request.md diff --git a/documentation/html/youtube/channel_api.md b/documentation/html/youtube/channel_api.md new file mode 100644 index 0000000..a51f1b8 --- /dev/null +++ b/documentation/html/youtube/channel_api.md @@ -0,0 +1,471 @@ +# Zombiez + + +https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false + +```json +{ + "responseContext": { + ... + }, + "contents": { + "twoColumnBrowseResultsRenderer": { + "tabs": [ + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CBAQ8JMBGAUiEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/featured", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCV0Ntl3lVR7xDXKoCU6uUXA", + "params": "EghmZWF0dXJlZPIGBAoCMgA%3D", + "canonicalBaseUrl": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + }, + "title": "Home", + "selected": true, + "content": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + { + "shelfRenderer": { + "title": { + "runs": [ + { + "text": "Albums & Singles", + "navigationEndpoint": { + "clickTrackingParams": "CBMQ3BwYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/playlists?view=50&sort=dd&shelf_id=17666223384013636040", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCVQK_XpXPuE8xcIpWp_JtlA", + "params": "EglwbGF5bGlzdHMYAyAycMiD1PaWksKV9QHyBgkKB0IAogECCAE%3D", + "canonicalBaseUrl": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + } + } + ] + }, + "endpoint": { + ... + }, + "content": { + "horizontalListRenderer": { + "items": [ + + /* first playlist */ + { + "gridPlaylistRenderer": { + "playlistId": "OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/-OPkZtrBlVM/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLASssT7fLkqB10cML8k_Tv5qoF4hg", + "width": 480, + "height": 270 + } + ], + "sampledThumbnailColor": { + "red": 62, + "green": 21, + "blue": 114 + } + }, + "title": { + "runs": [ + { + "text": "KATHARZIZ", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiDIGZy1oaWdoWhhVQ1YwTnRsM2xWUjd4RFhLb0NVNnVVWEGaAQYQ8jgYlwE=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=-OPkZtrBlVM&list=OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832 + } + }, + "watchEndpoint": { + "videoId": "-OPkZtrBlVM", + "playlistId": "OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "params": "OAI%3D", + "loggingContext": { + "vssLoggingContext": { + "serializedContextData": "GilPTEFLNXV5X25idlFlc2tyOG5iSXV6ZUx4b2NlTkx1Q0xfS2pBbXpWdw%3D%3D" + } + }, + "watchEndpointSupportedOnesieConfig": { + "html5PlaybackOnesieConfig": { + "commonConfig": { + "url": "https://rr6---sn-8xgn5uxa-4g5l.googlevideo.com/initplayback?source=youtube&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&beids=24350018&msp=1&odepv=1&id=f8e3e466dac19553&ip=87.123.241.91&initcwndbps=2165000&mt=1686834329&oweuc=" + } + } + } + } + } + } + ] + }, + "shortBylineText": { + ... + }, + "videoCountText": { + "runs": [ + { + "text": "16" + }, + { + "text": " videos" + } + ] + }, + "navigationEndpoint": { + ... + }, + "publishedTimeText": { + "simpleText": "Updated today" + }, + "videoCountShortText": { + "simpleText": "16" + }, + "trackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "sidebarThumbnails": [ + ... + ], + "thumbnailText": { + "runs": [ + { + "text": "16", + "bold": true + }, + { + "text": " videos" + } + ] + }, + "longBylineText": { + "runs": [ + { + "text": "Album" + }, + { + "text": " · " + }, + { + "text": "ZOMBIEZ", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCVQK_XpXPuE8xcIpWp_JtlA", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCVQK_XpXPuE8xcIpWp_JtlA", + "canonicalBaseUrl": "/channel/UCVQK_XpXPuE8xcIpWp_JtlA" + } + } + } + ] + }, + "thumbnailOverlays": [ + ... + ], + "viewPlaylistText": { + "runs": [ + { + "text": "View full playlist", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiDIGZy1oaWdo", + "commandMetadata": { + "webCommandMetadata": { + "url": "/playlist?list=OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "webPageType": "WEB_PAGE_TYPE_PLAYLIST", + "rootVe": 5754, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "VLOLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw" + } + } + } + ] + } + } + }, + + /* next playlist (same format as first one) */ + { + ... + }, + + /* many more playlists */ + ... + + + ], + ... + } + }, + "trackingParams": "CBMQ3BwYACITCP_c2oWtxf8CFULHEQgdXmwFiA==" + } + } + ], + "trackingParams": "CBIQuy8YACITCP_c2oWtxf8CFULHEQgdXmwFiA==" + } + } + ], + "trackingParams": "CBEQui8iEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "targetId": "browse-feedUCV0Ntl3lVR7xDXKoCU6uUXA", + "disablePullToRefresh": true + } + }, + "trackingParams": "CBAQ8JMBGAUiEwj_3NqFrcX_AhVCxxEIHV5sBYg=" + } + }, + ... + ] + } + }, + + /* Some probably irrelevant stuff */ + + "header": { + ... + }, + "metadata": { + "channelMetadataRenderer": { + "title": "ZOMBIEZ - Topic", + "description": "", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?channel_id=UCV0Ntl3lVR7xDXKoCU6uUXA", + "externalId": "UCV0Ntl3lVR7xDXKoCU6uUXA", + "keywords": "", + "ownerUrls": [ + "http://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + ], + "avatar": { + "thumbnails": [ + { + "url": "https://yt3.googleusercontent.com/nGpgytnZ-gQ8bcmUvqQ4UQruCRqWiQiujnAaHD6jSlLMMPGbl2J7V-j8iD1ZGj2iUEylvsFkNQ=s900-c-k-c0x00ffffff-no-rj", + "width": 900, + "height": 900 + } + ] + }, + "channelUrl": "https://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "isFamilySafe": true, + "availableCountryCodes": [ + ], + "musicArtistName": "ZOMBIEZ", + "androidDeepLink": "android-app://com.google.android.youtube/http/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "androidAppindexingLink": "android-app://com.google.android.youtube/http/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "iosAppindexingLink": "ios-app://544007664/vnd.youtube/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "vanityChannelUrl": "http://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + }, + "trackingParams": "CAAQhGciEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "topbar": { + ... + }, + "microformat": { + .... + } +} +``` + +# Eminem + +https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false + +```json +{ + "responseContext": { + ... + }, + "contents": { + "twoColumnBrowseResultsRenderer": { + "tabs": [ + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CDEQ8JMBGAUiEwiSlomurMX_AhWhiTgKHeKqA0o=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCedvOgsKFzcK3hA5taf3KoQ/featured", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCedvOgsKFzcK3hA5taf3KoQ", + "params": "EghmZWF0dXJlZPIGBAoCMgA%3D", + "canonicalBaseUrl": "/channel/UCedvOgsKFzcK3hA5taf3KoQ" + } + }, + "title": "Home", + "trackingParams": "CDEQ8JMBGAUiEwiSlomurMX_AhWhiTgKHeKqA0o=" + } + }, + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CA8Q8JMBGAYiEwiSlomurMX_AhWhiTgKHeKqA0o=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCedvOgsKFzcK3hA5taf3KoQ/playlists", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCedvOgsKFzcK3hA5taf3KoQ", + "params": "EglwbGF5bGlzdHPyBgQKAkIA", + "canonicalBaseUrl": "/channel/UCedvOgsKFzcK3hA5taf3KoQ" + } + }, + "title": "Playlists", + "selected": true, + "content": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + { + "gridRenderer": { + "items": [ + { + "gridPlaylistRenderer": { + "playlistId": "OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/r_0JjYUe5jo/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBDE5zUUu7bOlXU1uOhaNxJl5zIbQ", + "width": 480, + "height": 270 + } + ], + "sampledThumbnailColor": { + "red": 74, + "green": 89, + "blue": 88 + } + }, + "title": { + "runs": [ + { + "text": "Curtain Call 2", + "navigationEndpoint": { + "clickTrackingParams": "CDAQljUYACITCJKWia6sxf8CFaGJOAod4qoDSjIGZy1oaWdoWhhVQ2Vkdk9nc0tGemNLM2hBNXRhZjNLb1GaAQYQ8jgYlwE=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=r_0JjYUe5jo&list=OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832 + } + }, + "watchEndpoint": { + "videoId": "r_0JjYUe5jo", + "playlistId": "OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "params": "OAI%3D", + "loggingContext": { + "vssLoggingContext": { + "serializedContextData": "GilPTEFLNXV5X21IbVF0QW9qZE80UFJNeGh5R185RlIyY1NjcTUyWl93dw%3D%3D" + } + }, + "watchEndpointSupportedOnesieConfig": { + "html5PlaybackOnesieConfig": { + "commonConfig": { + "url": "https://rr2---sn-8xgn5uxa-4g5s.googlevideo.com/initplayback?source=youtube&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&beids=24350018&msp=1&odepv=1&id=affd098d851ee63a&ip=87.123.241.91&initcwndbps=1772500&mt=1686834084&oweuc=" + } + } + } + } + } + } + ] + }, + "shortBylineText": { + "runs": [ + + ] + }, + "videoCountText": { + "runs": [ + { + "text": "35" + }, + { + "text": " videos" + } + ] + }, + "navigationEndpoint": { + ... + } + }, + "publishedTimeText": { + "simpleText": "Updated today" + }, + "videoCountShortText": { + "simpleText": "35" + }, + ... + ], + "thumbnailText": { + "runs": [ + { + "text": "35", + "bold": true + }, + { + "text": " videos" + } + ] + }, + ... + "viewPlaylistText": { + "runs": [ + { + "text": "View full playlist", + "navigationEndpoint": { + "clickTrackingParams": "CDAQljUYACITCJKWia6sxf8CFaGJOAod4qoDSjIGZy1oaWdo", + "commandMetadata": { + "webCommandMetadata": { + "url": "/playlist?list=OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "webPageType": "WEB_PAGE_TYPE_PLAYLIST", + "rootVe": 5754, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "VLOLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww" + } + } + } + ] + } + } + }, +``` diff --git a/documentation/html/youtube/channel_api_request.md b/documentation/html/youtube/channel_api_request.md new file mode 100644 index 0000000..a2f3ffa --- /dev/null +++ b/documentation/html/youtube/channel_api_request.md @@ -0,0 +1,204 @@ +# Results + +## Request + +```curl +curl 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false' \ + -H 'authority: www.youtube.com' \ + -H 'accept: */*' \ + -H 'accept-language: en-US,en;q=0.7' \ + -H 'authorization: SAPISIDHASH 1686916263_e46803ac72f6036ba3b55e6edb0e45218c2d9222' \ + -H 'cache-control: no-cache' \ + -H 'content-type: application/json' \ + -H 'cookie: CONSENT=PENDING+359; SOCS=CAESEwgDEgk1MDQzNzQ2NDAaAmVuIAEaBgiAhceeBg; DEVICE_INFO=ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ+2kJ8GGJ+2kJ8G; HSID=ACaGQ6WFrIz4Ip9MQ; SSID=AkEQqNjJYkdquRRu6; APISID=c9DHm8ianHgc4LdY/AQMhE_0ECWj0r_orU; SAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-1PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-3PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; LOGIN_INFO=AFmmF2swRQIhANli-SWeYk6WvtIgBzsX4XMeuzlvdXMVtbfJTdhUJPRAAiAlG7PiNiEuUynQLUz5y-t7IvHPtpbmp0sU7_1B8ebQBw:QUQ3MjNmejhuWW5EZV9OVklIOTNxYXpEMXYwQWVvM1ptYTc2OWtObE9TcllyOEc4VndOR0tXVVRUOUhmT2hRbFRkNVZjeVhDRUJ0bmwxcTJTU3ZKWWhvX1hKMFdFbDVBYTdOS2tNMU41ZDI1Y3pIQ2NTVjNrdzJNVGFENDROT2IwdDBTTlV6cmh2RGFWaXpCWndGTHUxMzZBM1NGNk5FbVZn; VISITOR_INFO1_LIVE=swm2xaU6T4s; SID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6N69ULSrjryLwj4ZSvzfy6qA.; __Secure-1PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NcKwuUDKgPPeuOmwXW3uZUg.; __Secure-3PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NIh3P5codR8w_2O6jDK4PVw.; PREF=tz=Europe.Berlin&f6=40000000&f7=100; YSC=nVbKyjjk8dw; __Secure-YEC=CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D; SIDCC=AP8dLty32FdokPf_5Qg2eA8TWwebRgwg1n55TG2yQUd-mJkLvroZJj9QGMOIbE33Q2ovqK10H_uA; __Secure-1PSIDCC=AP8dLtz5c8aDk1Gw5VFhcM6EuUcQiZP50kH2XnUIx0PF4bDPWTPwLEq1LhVvadHnHwagqV8DXJI; __Secure-3PSIDCC=AP8dLtzfSXJb4bJhF8czo12Tuc90ONVeuOVXtinx8-_tXE7D4HW1XwHwDfspIBZItbz0aYCjgYDO; ST-txiubb=itct=CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw%3D&csn=MC45MTAyMTY0NjM3ODQ2MTUz&endpoint=%7B%22clickTrackingParams%22%3A%22CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw%3D%22%2C%22commandMetadata%22%3A%7B%22webCommandMetadata%22%3A%7B%22url%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%2Fplaylists%22%2C%22webPageType%22%3A%22WEB_PAGE_TYPE_CHANNEL%22%2C%22rootVe%22%3A3611%2C%22apiUrl%22%3A%22%2Fyoutubei%2Fv1%2Fbrowse%22%7D%7D%2C%22browseEndpoint%22%3A%7B%22browseId%22%3A%22UC8JxlMZ9VPddCuQGwB49fYg%22%2C%22params%22%3A%22EglwbGF5bGlzdHPyBgQKAkIA%22%2C%22canonicalBaseUrl%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%22%7D%7D; ST-plcuz1=' \ + -H 'origin: https://www.youtube.com' \ + -H 'pragma: no-cache' \ + -H 'referer: https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists' \ + -H 'sec-fetch-dest: empty' \ + -H 'sec-fetch-mode: same-origin' \ + -H 'sec-fetch-site: same-origin' \ + -H 'sec-gpc: 1' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' \ + -H 'x-goog-authuser: 0' \ + -H 'x-goog-visitor-id: CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D' \ + -H 'x-origin: https://www.youtube.com' \ + -H 'x-youtube-bootstrap-logged-in: true' \ + -H 'x-youtube-client-name: 1' \ + -H 'x-youtube-client-version: 2.20230613.01.00' \ + --data-raw '{ + "context":{ + "client":{ + "hl":"de", + "gl":"DE", + "remoteHost":"87.123.241.79", + "deviceMake":"", + "deviceModel":"", + "visitorData":"CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D", + "userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36,gzip(gfe)", + "clientName":"WEB", + "clientVersion":"2.20230613.01.00", + "osName":"X11", + "osVersion":"", + "originalUrl":"https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists", + "platform":"DESKTOP", + "clientFormFactor":"UNKNOWN_FORM_FACTOR", + "configInfo":{ + "appInstallData":"CJ2ZsaQGEKXC_hIQzLf-EhDftq8FELq0rwUQvbauBRDbr68FEPOorwUQ4tSuBRDj0f4SEMzfrgUQ7NH-EhDrk64FEI_DrwUQorSvBRDUoa8FEKqy_hIQpZmvBRC4i64FEMyu_hIQscavBRDnuq8FEN62rwUQouyuBRCCna8FEMO3_hIQ1bavBRCMt68FEJCjrwUQ7qKvBRDpw68FEInorgUQl9L-EhD4ta8FEOf3rgUQrLevBRD-ta8FEOSz_hIQ_u6uBQ%3D%3D" + }, + "userInterfaceTheme":"USER_INTERFACE_THEME_DARK", + "timeZone":"Europe/Berlin", + "browserName":"Chrome", + "browserVersion":"114.0.0.0", + "acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "deviceExperimentId":"ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ2ZsaQGGJ-2kJ8G", + "screenWidthPoints":1090, + "screenHeightPoints":980, + "screenPixelDensity":1, + "screenDensityFloat":1, + "utcOffsetMinutes":120, + "memoryTotalKbytes":"500000", + "mainAppWebInfo":{ + "graftUrl":"/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists", + "pwaInstallabilityStatus":"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED", + "webDisplayMode":"WEB_DISPLAY_MODE_BROWSER", + "isWebNativeShareAvailable":false + } + }, + "user":{ + "lockedSafetyMode":false + }, + "request":{ + "useSsl":true, + "internalExperimentFlags":[ + + ], + "consistencyTokenJars":[ + + ] + }, + "clickTracking":{ + "clickTrackingParams":"CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw=" + }, + "adSignalsInfo":{ + "params":[{"key":"dt","value":"1686916254647"},{"key":"flash","value":"0"},{"key":"frm","value":"0"},{"key":"u_tz","value":"120"},{"key":"u_his","value":"14"},{"key":"u_h","value":"1080"},{"key":"u_w","value":"1920"},{"key":"u_ah","value":"1049"},{"key":"u_aw","value":"1866"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"980"},{"key":"biw","value":"1075"},{"key":"brdim","value":"1280,31,1280,31,1866,31,1866,1049,1090,980"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}] + } + }, + "browseId":"UC8JxlMZ9VPddCuQGwB49fYg", + "params":"EglwbGF5bGlzdHPyBgQKAkIA" +}' \ + --compressed +``` + + +# No Results + +## Request + +```curl +curl 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false' \ + -H 'authority: www.youtube.com' \ + -H 'accept: */*' \ + -H 'accept-language: en-US,en;q=0.7' \ + -H 'authorization: SAPISIDHASH 1686916270_9e68abaf319d6994e84c3b64f46a2f6b91258f26' \ + -H 'cache-control: no-cache' \ + -H 'content-type: application/json' \ + -H 'cookie: CONSENT=PENDING+359; SOCS=CAESEwgDEgk1MDQzNzQ2NDAaAmVuIAEaBgiAhceeBg; DEVICE_INFO=ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ+2kJ8GGJ+2kJ8G; HSID=ACaGQ6WFrIz4Ip9MQ; SSID=AkEQqNjJYkdquRRu6; APISID=c9DHm8ianHgc4LdY/AQMhE_0ECWj0r_orU; SAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-1PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-3PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; LOGIN_INFO=AFmmF2swRQIhANli-SWeYk6WvtIgBzsX4XMeuzlvdXMVtbfJTdhUJPRAAiAlG7PiNiEuUynQLUz5y-t7IvHPtpbmp0sU7_1B8ebQBw:QUQ3MjNmejhuWW5EZV9OVklIOTNxYXpEMXYwQWVvM1ptYTc2OWtObE9TcllyOEc4VndOR0tXVVRUOUhmT2hRbFRkNVZjeVhDRUJ0bmwxcTJTU3ZKWWhvX1hKMFdFbDVBYTdOS2tNMU41ZDI1Y3pIQ2NTVjNrdzJNVGFENDROT2IwdDBTTlV6cmh2RGFWaXpCWndGTHUxMzZBM1NGNk5FbVZn; VISITOR_INFO1_LIVE=swm2xaU6T4s; SID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6N69ULSrjryLwj4ZSvzfy6qA.; __Secure-1PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NcKwuUDKgPPeuOmwXW3uZUg.; __Secure-3PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NIh3P5codR8w_2O6jDK4PVw.; PREF=tz=Europe.Berlin&f6=40000000&f7=100; YSC=nVbKyjjk8dw; __Secure-YEC=CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D; SIDCC=AP8dLtxamg1oLrM-gw2tCtJfMUighwYfCtCfwGT6G39nrxXXhtRoBcacfSPMOQwAOIN0xyOUSLyb; __Secure-1PSIDCC=AP8dLtxZ3YHCdW5cYDDaNKO8wvAGqqy4Kp7_xZtEYQW5bryvlEuU2iuJArAJskwLMwqiCznn038; __Secure-3PSIDCC=AP8dLtywHnuXOD7p59fWeUsQXp6MpPsJgZiUgW5ydTzajVvM6w64k4u7aE5WIuB5InMlTNJzKGGK; ST-plcuz1=; ST-ximm1t=csn=MC43NjY5NTkyNzczNjg1NjUy&itct=CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o%3D&endpoint=%7B%22clickTrackingParams%22%3A%22CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o%3D%22%2C%22commandMetadata%22%3A%7B%22webCommandMetadata%22%3A%7B%22url%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%2Fplaylists%3Fview%3D1%22%2C%22webPageType%22%3A%22WEB_PAGE_TYPE_CHANNEL%22%2C%22rootVe%22%3A3611%2C%22apiUrl%22%3A%22%2Fyoutubei%2Fv1%2Fbrowse%22%7D%7D%2C%22browseEndpoint%22%3A%7B%22browseId%22%3A%22UC8JxlMZ9VPddCuQGwB49fYg%22%2C%22params%22%3A%22EglwbGF5bGlzdHMgAQ%253D%253D%22%2C%22canonicalBaseUrl%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%22%7D%7D' \ + -H 'origin: https://www.youtube.com' \ + -H 'pragma: no-cache' \ + -H 'referer: https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1' \ + -H 'sec-fetch-dest: empty' \ + -H 'sec-fetch-mode: same-origin' \ + -H 'sec-fetch-site: same-origin' \ + -H 'sec-gpc: 1' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' \ + -H 'x-goog-authuser: 0' \ + -H 'x-goog-visitor-id: CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D' \ + -H 'x-origin: https://www.youtube.com' \ + -H 'x-youtube-bootstrap-logged-in: true' \ + -H 'x-youtube-client-name: 1' \ + -H 'x-youtube-client-version: 2.20230613.01.00' \ + --data-raw '{ + "context":{ + "client":{ + "hl":"de", + "gl":"DE", + "remoteHost":"87.123.241.79", + "deviceMake":"", + "deviceModel":"", + "visitorData":"CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D", + "userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36,gzip(gfe)", + "clientName":"WEB", + "clientVersion":"2.20230613.01.00", + "osName":"X11", + "osVersion":"", + "originalUrl":"https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1", + "platform":"DESKTOP", + "clientFormFactor":"UNKNOWN_FORM_FACTOR", + "configInfo":{ + "appInstallData":"CJ2ZsaQGEKXC_hIQzLf-EhDftq8FELq0rwUQvbauBRDbr68FEPOorwUQ4tSuBRDj0f4SEMzfrgUQ7NH-EhDrk64FEI_DrwUQorSvBRDUoa8FEKqy_hIQpZmvBRC4i64FEMyu_hIQscavBRDnuq8FEN62rwUQouyuBRCCna8FEMO3_hIQ1bavBRCMt68FEJCjrwUQ7qKvBRDpw68FEInorgUQl9L-EhD4ta8FEOf3rgUQrLevBRD-ta8FEOSz_hIQ_u6uBQ%3D%3D" + }, + "userInterfaceTheme":"USER_INTERFACE_THEME_DARK", + "timeZone":"Europe/Berlin", + "browserName":"Chrome", + "browserVersion":"114.0.0.0", + "acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "deviceExperimentId":"ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ2ZsaQGGJ-2kJ8G", + "screenWidthPoints":1090, + "screenHeightPoints":980, + "screenPixelDensity":1, + "screenDensityFloat":1, + "utcOffsetMinutes":120, + "memoryTotalKbytes":"500000", + "mainAppWebInfo":{ + "graftUrl":"/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1", + "pwaInstallabilityStatus":"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED", + "webDisplayMode":"WEB_DISPLAY_MODE_BROWSER", + "isWebNativeShareAvailable":false + } + }, + "user":{ + "lockedSafetyMode":false + }, + "request":{ + "useSsl":true, + "internalExperimentFlags":[ + + ], + "consistencyTokenJars":[ + + ] + }, + "clickTracking":{ + "clickTrackingParams":"CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o=" + }, + "adSignalsInfo":{ + "params":[{"key":"dt","value":"1686916254647"},{"key":"flash","value":"0"},{"key":"frm","value":"0"},{"key":"u_tz","value":"120"},{"key":"u_his","value":"15"},{"key":"u_h","value":"1080"},{"key":"u_w","value":"1920"},{"key":"u_ah","value":"1049"},{"key":"u_aw","value":"1866"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"980"},{"key":"biw","value":"1075"},{"key":"brdim","value":"1280,31,1280,31,1866,31,1866,1049,1090,980"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}] + } + }, + "browseId":"UC8JxlMZ9VPddCuQGwB49fYg", + "params":"EglwbGF5bGlzdHMgAQ%3D%3D" +}' \ + --compressed + ``` + +# Potentially relevant difference + +difference | Result | No Result +referer | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1 +`"originalURL"` | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1 + +## What I think the problem is + +I think the problem arises, because the request is made in the wrong manner. Refering to the by the endpoint returned data from earlier, the data is given in pretty much the same format. + +Now the thing is, that there are some Topic Channels, where you can't switch the playlist type. + +- https://www.youtube.com/channel/UCPOUrPpYMpxQU_gKtBQZroQ/playlists +- https://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/playlists + +But then there are those Topic Channels, where you actually can switch the playlist type. These are usually the bigger ones. I haven't found why that difference exists though. + +- https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists +- https://www.youtube.com/channel/UCedvOgsKFzcK3hA5taf3KoQ/playlists + +So modifying the requests from one of those channels, that currently do yield results, to have the `?view=1` parameter in the url, they still yield results. So my bet right now would be, that the reasons some channels do yield results and some don't is, because invidious makes the requests with `?view=1`, or does something simmilar, which causes the same behaviour. diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 360fbea..9e8bd89 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -15,7 +15,7 @@ if __name__ == "__main__": youtube_search = [ "s: #a Zombiez", "10", - "8" + "d: 5" ] - music_kraken.cli(genre="test", command_list=direct_download) + music_kraken.cli(genre="test", command_list=youtube_search) diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index f8ba952..42518a9 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -45,9 +45,9 @@ def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = # run the ffmpeg command with a progressbar ff = FfmpegProgress(ffmpeg_command) - with tqdm(total=100, desc="ffmpeg") as pbar: + with tqdm(total=100, desc=f"removing {len(interval_list)} segments") as pbar: for progress in ff.run_command_with_progress(): - pbar.update(int(progress)-pbar.n) + pbar.update(progress-pbar.n) LOGGER.debug(ff.stderr) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 81dc21a..cd739c4 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -17,7 +17,7 @@ class Connection: self, host: str, proxies: List[dict] = None, - tries: int = (len(PROXIES_LIST) + 1) * 2, + tries: int = (len(PROXIES_LIST) + 1) * 4, timeout: int = 7, logger: logging.Logger = logging.getLogger("connection"), header_values: Dict[str, str] = None, @@ -80,7 +80,7 @@ class Connection: self, request: Callable, try_count: int, - accepted_response_code: set, + accepted_response_codes: set, url: str, timeout: float, headers: dict, @@ -109,7 +109,7 @@ class Connection: try: r: requests.Response = request(request_url, timeout=timeout, headers=headers, **kwargs) - if r.status_code in accepted_response_code: + if r.status_code in accepted_response_codes: return r if self.SEMANTIC_NOT_FOUND and r.status_code == 404: @@ -136,7 +136,7 @@ class Connection: return self._request( request=request, try_count=try_count+1, - accepted_response_code=accepted_response_code, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -154,10 +154,13 @@ class Connection: raw_url: bool = False, **kwargs ) -> Optional[requests.Response]: + if accepted_response_codes is None: + accepted_response_codes = self.ACCEPTED_RESPONSE_CODES + r = self._request( request=self.session.get, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -185,7 +188,7 @@ class Connection: r = self._request( request=self.session.post, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, timeout=timeout, headers=headers, @@ -215,11 +218,19 @@ class Connection: progress: int = 0, **kwargs ) -> DownloadResult: + + if progress > 0: + if headers is None: + headers = dict() + headers["Range"] = f"bytes={target.size}-" + + if accepted_response_codes is None: + accepted_response_codes = self.ACCEPTED_RESPONSE_CODES r = self._request( request=self.session.get, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -238,45 +249,47 @@ class Connection: retry = False - with target.open("wb") as f: - try: - """ - https://en.wikipedia.org/wiki/Kilobyte - > The internationally recommended unit symbol for the kilobyte is kB. - """ + with target.open("ab") as f: + """ + https://en.wikipedia.org/wiki/Kilobyte + > The internationally recommended unit symbol for the kilobyte is kB. + """ - with tqdm(total=total_size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: - + with tqdm(total=total_size-target.size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: + try: for chunk in r.iter_content(chunk_size=chunk_size): size = f.write(chunk) progress += size t.update(size) - return True - except requests.exceptions.ConnectionError: - if try_count >= self.TRIES: - self.LOGGER.warning(f"Stream timed out at \"{url}\": to many retries, aborting.") - return DownloadResult(error_message=f"Stream timed out from {url}, reducing the chunksize might help.") - - self.LOGGER.warning(f"Stream timed out at \"{url}\": ({try_count}-{self.TRIES})") + except requests.exceptions.ConnectionError: + if try_count >= self.TRIES: + self.LOGGER.warning(f"Stream timed out at \"{url}\": to many retries, aborting.") + return DownloadResult(error_message=f"Stream timed out from {url}, reducing the chunksize might help.") + + self.LOGGER.warning(f"Stream timed out at \"{url}\": ({try_count}-{self.TRIES})") + retry = True + + if total_size > progress: retry = True - finally: - if total_size > progress or retry: - return self.stream_into( - url = url, - target = target, - description = description, - try_count=try_count+1, - progress=progress, - accepted_response_code=accepted_response_codes, - timeout=timeout, - headers=headers, - raw_url=raw_url, - refer_from_origin=refer_from_origin, - chunk_size=chunk_size, - **kwargs - ) - - return DownloadResult() + if retry: + self.LOGGER.warning(f"Retrying stream...") + accepted_response_codes.add(206) + return self.stream_into( + url = url, + target = target, + description = description, + try_count=try_count+1, + progress=progress, + accepted_response_codes=accepted_response_codes, + timeout=timeout, + headers=headers, + raw_url=raw_url, + refer_from_origin=refer_from_origin, + chunk_size=chunk_size, + **kwargs + ) + + return DownloadResult() diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 797480e..40c62b5 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -394,7 +394,4 @@ class YouTube(Page): except HTTPException as e: self.LOGGER.warning(f"{e}") - if len(segments) > 0: - print(f"Removing {len(segments)} interruptions in the audio...") - return [(segment.start, segment.end) for segment in segments] diff --git a/src/music_kraken/utils/config/audio.py b/src/music_kraken/utils/config/audio.py index 969bb67..ea85d84 100644 --- a/src/music_kraken/utils/config/audio.py +++ b/src/music_kraken/utils/config/audio.py @@ -106,7 +106,7 @@ ID3.1: {', '.join(_sorted_id3_1_formats)} self.DOWNLOAD_PATH = StringAttribute( name="download_path", - value="{genre}/{artist}/{album_type}/{album}", + value="{genre}/{artist}/{album}", description="The folder music kraken should put the songs into." ) diff --git a/src/music_kraken/utils/support_classes/download_result.py b/src/music_kraken/utils/support_classes/download_result.py index a47a054..2f54111 100644 --- a/src/music_kraken/utils/support_classes/download_result.py +++ b/src/music_kraken/utils/support_classes/download_result.py @@ -80,11 +80,11 @@ class DownloadResult: def __str__(self): if self.is_fatal_error: return self.error_message - head = f"{self.fail} from {self.total} downloads failed:\n" \ - f"successrate:\t{int(self.success_percentage * 100)}%\n" \ - f"failrate:\t{int(self.failure_percentage * 100)}%\n" \ - f"total size:\t{self.formated_size}\n" \ - f"skipped segments:\t{self.sponsor_segments}" \ + head = f"{self.fail} from {self.total} downloads failed:\n" \ + f"successrate:\t{int(self.success_percentage * 100)}%\n" \ + f"failrate:\t{int(self.failure_percentage * 100)}%\n" \ + f"total size:\t{self.formated_size}\n" \ + f"skipped segments:\t{self.sponsor_segments}\n" \ f"found on disc:\t{self.found_on_disk}" if not self.is_mild_failure: From 0fb8a250944817cfe1d251b909c4e0e9f2f9fff9 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 19 Jun 2023 14:32:32 +0200 Subject: [PATCH 68/81] added config for piped --- src/music_kraken/__main__.py | 12 +- src/music_kraken/cli/__init__.py | 3 +- src/music_kraken/cli/options/__init__.py | 4 - src/music_kraken/cli/options/frontend.py | 181 ++++++++++++++++++ .../cli/options/invidious/shell.py | 101 ---------- src/music_kraken/objects/__init__.py | 2 + src/music_kraken/objects/country.py | 66 +++++++ src/music_kraken/pages/__init__.py | 2 +- .../invidious => pages/youtube}/__init__.py | 0 src/music_kraken/pages/youtube/piped.py | 0 .../pages/{ => youtube}/youtube.py | 12 +- src/music_kraken/utils/config/connection.py | 18 +- 12 files changed, 274 insertions(+), 127 deletions(-) create mode 100644 src/music_kraken/cli/options/frontend.py delete mode 100644 src/music_kraken/cli/options/invidious/shell.py create mode 100644 src/music_kraken/objects/country.py rename src/music_kraken/{cli/options/invidious => pages/youtube}/__init__.py (100%) create mode 100644 src/music_kraken/pages/youtube/piped.py rename src/music_kraken/pages/{ => youtube}/youtube.py (97%) diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 4aa30ef..776622b 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -67,9 +67,9 @@ if __name__ == "__main__": ) parser.add_argument( - "--invidious", - "-i", - help="Set a good and fast invidious instance from your homecountry, to reduce the latency.", + "--frontend", + "-f", + help="Set a good and fast invidious/piped instance from your homecountry, to reduce the latency.", action="store_true" ) @@ -102,9 +102,9 @@ if __name__ == "__main__": os.remove(music_kraken.shared.CONFIG_FILE) music_kraken.read() - if arguments.invidious: - from .cli.options import invidious - invidious() + if arguments.frontend: + from .cli.options.frontend import set_frontend + set_frontend() exit() # getting the genre diff --git a/src/music_kraken/cli/__init__.py b/src/music_kraken/cli/__init__.py index b4e57ef..c21af81 100644 --- a/src/music_kraken/cli/__init__.py +++ b/src/music_kraken/cli/__init__.py @@ -1,2 +1 @@ -from .download.shell import Shell -from .options import invidious \ No newline at end of file +from .download.shell import Shell \ No newline at end of file diff --git a/src/music_kraken/cli/options/__init__.py b/src/music_kraken/cli/options/__init__.py index 0ab4d7f..e69de29 100644 --- a/src/music_kraken/cli/options/__init__.py +++ b/src/music_kraken/cli/options/__init__.py @@ -1,4 +0,0 @@ -from .invidious.shell import InvidiousShell - -def invidious(): - shell = InvidiousShell() \ No newline at end of file diff --git a/src/music_kraken/cli/options/frontend.py b/src/music_kraken/cli/options/frontend.py new file mode 100644 index 0000000..34395f4 --- /dev/null +++ b/src/music_kraken/cli/options/frontend.py @@ -0,0 +1,181 @@ +from typing import Dict, List +import requests +from dataclasses import dataclass +from collections import defaultdict + +from ...objects import Country +from ...utils import config, write +from ...connection import Connection + + +@dataclass +class Instance: + """ + Attributes which influence the quality of an instance: + + - users + """ + name: str + uri: str + regions: List[Country] + users: int = 0 + + def __str__(self) -> str: + return f"{self.name} with {self.users} users." + + +class FrontendInstance: + SETTING_NAME = "placeholder" + + def __init__(self) -> None: + self.region_instances: Dict[Country, List[Instance]] = defaultdict(list) + self.all_instances: List[Instance] = [] + + def add_instance(self, instance: Instance): + self.all_instances.append(instance) + + for region in instance.regions: + self.region_instances[region].append(instance) + + def fetch(self, silent: bool = False): + if not silent: + print(f"Downloading {type(self).__name__} instances...") + + def set_instance(self, instance: Instance): + config.set_name_to_value(self.SETTING_NAME, instance.uri) + write() + + def _choose_country(self) -> List[Instance]: + print("Input the country code, an example would be \"US\"") + print('\n'.join(f'{region.name} ({region.alpha_2})' for region in self.region_instances)) + print() + + + available_instances = set(i.alpha_2 for i in self.region_instances) + + chosen_region = "" + + while chosen_region not in available_instances: + chosen_region = input("nearest country: ").strip().upper() + + return self.region_instances[Country.by_alpha_2(chosen_region)] + + def choose(self, silent: bool = False): + instances = self.all_instances if silent else self._choose_country() + instances.sort(key=lambda x: x.users, reverse=True) + + if silent: + self.set_instance(instances[0]) + return + + # output the options + print("Choose your instance (input needs to be a digit):") + for i, instance in enumerate(instances): + print(f"{i}) {instance}") + + print() + + # ask for index + index = "" + while not index.isdigit() or int(index) >= len(instances): + index = input("> ").strip() + + instance = instances[int(index)] + print() + print(f"Setting the instance to {instance}") + + self.set_instance(instance) + + +class Invidious(FrontendInstance): + SETTING_NAME = "invidious_instance" + + def __init__(self) -> None: + self.connection = Connection(host="https://api.invidious.io/") + self.endpoint = "https://api.invidious.io/instances.json" + + super().__init__() + + + def _process_instance(self, all_instance_data: dict): + instance_data = all_instance_data[1] + stats = instance_data["stats"] + + if not instance_data["api"]: + return + if instance_data["type"] != "https": + return + + region = instance_data["region"] + + instance = Instance( + name=all_instance_data[0], + uri=instance_data["uri"], + regions=[Country.by_alpha_2(region)], + users=stats["usage"]["users"]["total"] + ) + + self.add_instance(instance) + + def fetch(self, silent: bool): + print("jhdflashfö") + r = self.connection.get(self.endpoint) + if r is None: + return + + for instance in r.json(): + self._process_instance(all_instance_data=instance) + + +class Piped(FrontendInstance): + SETTING_NAME = "piped_instance" + + def __init__(self) -> None: + self.connection = Connection(host="https://raw.githubusercontent.com") + + super().__init__() + + def process_instance(self, instance_data: str): + cells = instance_data.split(" | ") + + instance = Instance( + name=cells[0].strip(), + uri=cells[1].strip(), + regions=[Country.by_emoji(flag) for flag in cells[2].split(", ")] + ) + + self.add_instance(instance) + + def fetch(self, silent: bool = False): + r = self.connection.get("https://raw.githubusercontent.com/wiki/TeamPiped/Piped-Frontend/Instances.md") + if r is None: + return + + process = False + + for line in r.content.decode("utf-8").split("\n"): + line = line.strip() + + if line != "" and process: + self.process_instance(line) + + if line.startswith("---"): + process = True + + +class FrontendSelection: + def __init__(self): + self.invidious = Invidious() + self.piped = Piped() + + def choose(self, silent: bool = False): + self.invidious.fetch(silent) + self.invidious.choose(silent) + + self.piped.fetch(silent) + self.piped.choose(silent) + + +def set_frontend(silent: bool = False): + shell = FrontendSelection() + shell.choose(silent=silent) diff --git a/src/music_kraken/cli/options/invidious/shell.py b/src/music_kraken/cli/options/invidious/shell.py deleted file mode 100644 index 131a9d8..0000000 --- a/src/music_kraken/cli/options/invidious/shell.py +++ /dev/null @@ -1,101 +0,0 @@ -from typing import Dict, List -import requests -from dataclasses import dataclass -from collections import defaultdict - -from ....utils import config, write - - -INSTANCES_ENDPOINT = "https://api.invidious.io/instances.json" - -@dataclass -class Instance: - """ - Attributes which influence the quality of an instance: - - - users - """ - name: str - uri: str - region: str - users: int - - def __str__(self) -> str: - return f"{self.name} with {self.users} users." - - -class InvidiousShell: - def __init__(self): - self.region_flags = dict() - self.region_instances: Dict[str, List[Instance]] = defaultdict(list) - - self.download_instances() - self.region = self.get_country() - print() - self.choose_instance() - - - def process_instance(self, all_instance_data: dict): - instance_data = all_instance_data[1] - stats = instance_data["stats"] - - if not instance_data["api"]: - return - if instance_data["type"] != "https": - return - - region = instance_data["region"] - flag = instance_data["flag"] - - self.region_flags[region] = flag - - instance = Instance( - name=all_instance_data[0], - uri=instance_data["uri"], - region=region, - users=stats["usage"]["users"]["total"] - ) - - self.region_instances[region].append(instance) - - def download_instances(self): - print("Download idonvidious instances...") - r = requests.get(INSTANCES_ENDPOINT) - - for instance in r.json(): - self.process_instance(all_instance_data=instance) - - def get_country(self): - print("Input the country code, an example would be \"US\"") - print(f"({' | '.join(f'{region}-{flag}' for region, flag in self.region_flags.items())})") - - chosen_region = "" - - while chosen_region.upper() not in self.region_instances: - chosen_region = input("nearest country: ").strip().upper() - - return chosen_region - - def choose_instance(self): - instance_list = self.region_instances[self.region] - instance_list.sort(key=lambda x: x.users, reverse=True) - - # output the options - print("Choose your instance (input needs to be a digit):") - for i, instance in enumerate(instance_list): - print(f"{i}) {instance}") - - print() - - # ask for index - index = "" - while not index.isdigit() or int(index) >= len(instance_list): - index = input("> ").strip() - - instance = instance_list[int(index)] - print() - print(f"Setting the instance to {instance}") - - config.set_name_to_value("invidious_instance", instance.uri) - write() - \ No newline at end of file diff --git a/src/music_kraken/objects/__init__.py b/src/music_kraken/objects/__init__.py index f458303..4a17b9b 100644 --- a/src/music_kraken/objects/__init__.py +++ b/src/music_kraken/objects/__init__.py @@ -16,3 +16,5 @@ from .song import ( from .formatted_text import FormattedText from .collection import Collection + +from .country import Country diff --git a/src/music_kraken/objects/country.py b/src/music_kraken/objects/country.py new file mode 100644 index 0000000..aed4842 --- /dev/null +++ b/src/music_kraken/objects/country.py @@ -0,0 +1,66 @@ +from dataclasses import dataclass + +import pycountry + + +CountryTyping = type(list(pycountry.countries)[0]) + + +emoji_map = {'AD': '🇦🇩', 'AE': '🇦🇪', 'AF': '🇦🇫', 'AG': '🇦🇬', 'AI': '🇦🇮', 'AL': '🇦🇱', 'AM': '🇦🇲', 'AO': '🇦🇴', 'AQ': '🇦🇶', 'AR': '🇦🇷', 'AS': '🇦🇸', 'AT': '🇦🇹', 'AU': '🇦🇺', 'AW': '🇦🇼', 'AX': '🇦🇽', 'AZ': '🇦🇿', 'BA': '🇧🇦', 'BB': '🇧🇧', 'BD': '🇧🇩', 'BE': '🇧🇪', 'BF': '🇧🇫', 'BG': '🇧🇬', 'BH': '🇧🇭', 'BI': '🇧🇮', 'BJ': '🇧🇯', 'BL': '🇧🇱', 'BM': '🇧🇲', 'BN': '🇧🇳', 'BO': '🇧🇴', 'BQ': '🇧🇶', 'BR': '🇧🇷', 'BS': '🇧🇸', 'BT': '🇧🇹', 'BV': '🇧🇻', 'BW': '🇧🇼', 'BY': '🇧🇾', 'BZ': '🇧🇿', 'CA': '🇨🇦', 'CC': '🇨🇨', 'CD': '🇨🇩', 'CF': '🇨🇫', 'CG': '🇨🇬', 'CH': '🇨🇭', 'CI': '🇨🇮', 'CK': '🇨🇰', 'CL': '🇨🇱', 'CM': '🇨🇲', 'CN': '🇨🇳', 'CO': '🇨🇴', 'CR': '🇨🇷', 'CU': '🇨🇺', 'CV': '🇨🇻', 'CW': '🇨🇼', 'CX': '🇨🇽', 'CY': '🇨🇾', 'CZ': '🇨🇿', 'DE': '🇩🇪', 'DJ': '🇩🇯', 'DK': '🇩🇰', 'DM': '🇩🇲', 'DO': '🇩🇴', 'DZ': '🇩🇿', 'EC': '🇪🇨', 'EE': '🇪🇪', 'EG': '🇪🇬', 'EH': '🇪🇭', 'ER': '🇪🇷', 'ES': '🇪🇸', 'ET': '🇪🇹', 'FI': '🇫🇮', 'FJ': '🇫🇯', 'FK': '🇫🇰', 'FM': '🇫🇲', 'FO': '🇫🇴', 'FR': '🇫🇷', 'GA': '🇬🇦', 'GB': '🇬🇧', 'GD': '🇬🇩', 'GE': '🇬🇪', 'GF': '🇬🇫', 'GG': '🇬🇬', 'GH': '🇬🇭', 'GI': '🇬🇮', 'GL': '🇬🇱', 'GM': '🇬🇲', 'GN': '🇬🇳', 'GP': '🇬🇵', 'GQ': '🇬🇶', 'GR': '🇬🇷', 'GS': '🇬🇸', 'GT': '🇬🇹', 'GU': '🇬🇺', 'GW': '🇬🇼', 'GY': '🇬🇾', 'HK': '🇭🇰', 'HM': '🇭🇲', 'HN': '🇭🇳', 'HR': '🇭🇷', 'HT': '🇭🇹', 'HU': '🇭🇺', 'ID': '🇮🇩', 'IE': '🇮🇪', 'IL': '🇮🇱', 'IM': '🇮🇲', 'IN': '🇮🇳', 'IO': '🇮🇴', 'IQ': '🇮🇶', 'IR': '🇮🇷', 'IS': '🇮🇸', 'IT': '🇮🇹', 'JE': '🇯🇪', 'JM': '🇯🇲', 'JO': '🇯🇴', 'JP': '🇯🇵', 'KE': '🇰🇪', 'KG': '🇰🇬', 'KH': '🇰🇭', 'KI': '🇰🇮', 'KM': '🇰🇲', 'KN': '🇰🇳', 'KP': '🇰🇵', 'KR': '🇰🇷', 'KW': '🇰🇼', 'KY': '🇰🇾', 'KZ': '🇰🇿', 'LA': '🇱🇦', 'LB': '🇱🇧', 'LC': '🇱🇨', 'LI': '🇱🇮', 'LK': '🇱🇰', 'LR': '🇱🇷', 'LS': '🇱🇸', 'LT': '🇱🇹', 'LU': '🇱🇺', 'LV': '🇱🇻', 'LY': '🇱🇾', 'MA': '🇲🇦', 'MC': '🇲🇨', 'MD': '🇲🇩', 'ME': '🇲🇪', 'MF': '🇲🇫', 'MG': '🇲🇬', 'MH': '🇲🇭', 'MK': '🇲🇰', 'ML': '🇲🇱', 'MM': '🇲🇲', 'MN': '🇲🇳', 'MO': '🇲🇴', 'MP': '🇲🇵', 'MQ': '🇲🇶', 'MR': '🇲🇷', 'MS': '🇲🇸', 'MT': '🇲🇹', 'MU': '🇲🇺', 'MV': '🇲🇻', 'MW': '🇲🇼', 'MX': '🇲🇽', 'MY': '🇲🇾', 'MZ': '🇲🇿', 'NA': '🇳🇦', 'NC': '🇳🇨', 'NE': '🇳🇪', 'NF': '🇳🇫', 'NG': '🇳🇬', 'NI': '🇳🇮', 'NL': '🇳🇱', 'NO': '🇳🇴', 'NP': '🇳🇵', 'NR': '🇳🇷', 'NU': '🇳🇺', 'NZ': '🇳🇿', 'OM': '🇴🇲', 'PA': '🇵🇦', 'PE': '🇵🇪', 'PF': '🇵🇫', 'PG': '🇵🇬', 'PH': '🇵🇭', 'PK': '🇵🇰', 'PL': '🇵🇱', 'PM': '🇵🇲', 'PN': '🇵🇳', 'PR': '🇵🇷', 'PS': '🇵🇸', 'PT': '🇵🇹', 'PW': '🇵🇼', 'PY': '🇵🇾', 'QA': '🇶🇦', 'RE': '🇷🇪', 'RO': '🇷🇴', 'RS': '🇷🇸', 'RU': '🇷🇺', 'RW': '🇷🇼', 'SA': '🇸🇦', 'SB': '🇸🇧', 'SC': '🇸🇨', 'SD': '🇸🇩', 'SE': '🇸🇪', 'SG': '🇸🇬', 'SH': '🇸🇭', 'SI': '🇸🇮', 'SJ': '🇸🇯', 'SK': '🇸🇰', 'SL': '🇸🇱', 'SM': '🇸🇲', 'SN': '🇸🇳', 'SO': '🇸🇴', 'SR': '🇸🇷', 'SS': '🇸🇸', 'ST': '🇸🇹', 'SV': '🇸🇻', 'SX': '🇸🇽', 'SY': '🇸🇾', 'SZ': '🇸🇿', 'TC': '🇹🇨', 'TD': '🇹🇩', 'TF': '🇹🇫', 'TG': '🇹🇬', 'TH': '🇹🇭', 'TJ': '🇹🇯', 'TK': '🇹🇰', 'TL': '🇹🇱', 'TM': '🇹🇲', 'TN': '🇹🇳', 'TO': '🇹🇴', 'TR': '🇹🇷', 'TT': '🇹🇹', 'TV': '🇹🇻', 'TW': '🇹🇼', 'TZ': '🇹🇿', 'UA': '🇺🇦', 'UG': '🇺🇬', 'UM': '🇺🇲', 'US': '🇺🇸', 'UY': '🇺🇾', 'UZ': '🇺🇿', 'VA': '🇻🇦', 'VC': '🇻🇨', 'VE': '🇻🇪', 'VG': '🇻🇬', 'VI': '🇻🇮', 'VN': '🇻🇳', 'VU': '🇻🇺', 'WF': '🇼🇫', 'WS': '🇼🇸', 'YE': '🇾🇪', 'YT': '🇾🇹', 'ZA': '🇿🇦', 'ZM': '🇿🇲', 'ZW': '🇿🇼', '🇦🇩': 'AD', '🇦🇪': 'AE', '🇦🇫': 'AF', '🇦🇬': 'AG', '🇦🇮': 'AI', '🇦🇱': 'AL', '🇦🇲': 'AM', '🇦🇴': 'AO', '🇦🇶': 'AQ', '🇦🇷': 'AR', '🇦🇸': 'AS', '🇦🇹': 'AT', '🇦🇺': 'AU', '🇦🇼': 'AW', '🇦🇽': 'AX', '🇦🇿': 'AZ', '🇧🇦': 'BA', '🇧🇧': 'BB', '🇧🇩': 'BD', '🇧🇪': 'BE', '🇧🇫': 'BF', '🇧🇬': 'BG', '🇧🇭': 'BH', '🇧🇮': 'BI', '🇧🇯': 'BJ', '🇧🇱': 'BL', '🇧🇲': 'BM', '🇧🇳': 'BN', '🇧🇴': 'BO', '🇧🇶': 'BQ', '🇧🇷': 'BR', '🇧🇸': 'BS', '🇧🇹': 'BT', '🇧🇻': 'BV', '🇧🇼': 'BW', '🇧🇾': 'BY', '🇧🇿': 'BZ', '🇨🇦': 'CA', '🇨🇨': 'CC', '🇨🇩': 'CD', '🇨🇫': 'CF', '🇨🇬': 'CG', '🇨🇭': 'CH', '🇨🇮': 'CI', '🇨🇰': 'CK', '🇨🇱': 'CL', '🇨🇲': 'CM', '🇨🇳': 'CN', '🇨🇴': 'CO', '🇨🇷': 'CR', '🇨🇺': 'CU', '🇨🇻': 'CV', '🇨🇼': 'CW', '🇨🇽': 'CX', '🇨🇾': 'CY', '🇨🇿': 'CZ', '🇩🇪': 'DE', '🇩🇯': 'DJ', '🇩🇰': 'DK', '🇩🇲': 'DM', '🇩🇴': 'DO', '🇩🇿': 'DZ', '🇪🇨': 'EC', '🇪🇪': 'EE', '🇪🇬': 'EG', '🇪🇭': 'EH', '🇪🇷': 'ER', '🇪🇸': 'ES', '🇪🇹': 'ET', '🇫🇮': 'FI', '🇫🇯': 'FJ', '🇫🇰': 'FK', '🇫🇲': 'FM', '🇫🇴': 'FO', '🇫🇷': 'FR', '🇬🇦': 'GA', '🇬🇧': 'GB', '🇬🇩': 'GD', '🇬🇪': 'GE', '🇬🇫': 'GF', '🇬🇬': 'GG', '🇬🇭': 'GH', '🇬🇮': 'GI', '🇬🇱': 'GL', '🇬🇲': 'GM', '🇬🇳': 'GN', '🇬🇵': 'GP', '🇬🇶': 'GQ', '🇬🇷': 'GR', '🇬🇸': 'GS', '🇬🇹': 'GT', '🇬🇺': 'GU', '🇬🇼': 'GW', '🇬🇾': 'GY', '🇭🇰': 'HK', '🇭🇲': 'HM', '🇭🇳': 'HN', '🇭🇷': 'HR', '🇭🇹': 'HT', '🇭🇺': 'HU', '🇮🇩': 'ID', '🇮🇪': 'IE', '🇮🇱': 'IL', '🇮🇲': 'IM', '🇮🇳': 'IN', '🇮🇴': 'IO', '🇮🇶': 'IQ', '🇮🇷': 'IR', '🇮🇸': 'IS', '🇮🇹': 'IT', '🇯🇪': 'JE', '🇯🇲': 'JM', '🇯🇴': 'JO', '🇯🇵': 'JP', '🇰🇪': 'KE', '🇰🇬': 'KG', '🇰🇭': 'KH', '🇰🇮': 'KI', '🇰🇲': 'KM', '🇰🇳': 'KN', '🇰🇵': 'KP', '🇰🇷': 'KR', '🇰🇼': 'KW', '🇰🇾': 'KY', '🇰🇿': 'KZ', '🇱🇦': 'LA', '🇱🇧': 'LB', '🇱🇨': 'LC', '🇱🇮': 'LI', '🇱🇰': 'LK', '🇱🇷': 'LR', '🇱🇸': 'LS', '🇱🇹': 'LT', '🇱🇺': 'LU', '🇱🇻': 'LV', '🇱🇾': 'LY', '🇲🇦': 'MA', '🇲🇨': 'MC', '🇲🇩': 'MD', '🇲🇪': 'ME', '🇲🇫': 'MF', '🇲🇬': 'MG', '🇲🇭': 'MH', '🇲🇰': 'MK', '🇲🇱': 'ML', '🇲🇲': 'MM', '🇲🇳': 'MN', '🇲🇴': 'MO', '🇲🇵': 'MP', '🇲🇶': 'MQ', '🇲🇷': 'MR', '🇲🇸': 'MS', '🇲🇹': 'MT', '🇲🇺': 'MU', '🇲🇻': 'MV', '🇲🇼': 'MW', '🇲🇽': 'MX', '🇲🇾': 'MY', '🇲🇿': 'MZ', '🇳🇦': 'NA', '🇳🇨': 'NC', '🇳🇪': 'NE', '🇳🇫': 'NF', '🇳🇬': 'NG', '🇳🇮': 'NI', '🇳🇱': 'NL', '🇳🇴': 'NO', '🇳🇵': 'NP', '🇳🇷': 'NR', '🇳🇺': 'NU', '🇳🇿': 'NZ', '🇴🇲': 'OM', '🇵🇦': 'PA', '🇵🇪': 'PE', '🇵🇫': 'PF', '🇵🇬': 'PG', '🇵🇭': 'PH', '🇵🇰': 'PK', '🇵🇱': 'PL', '🇵🇲': 'PM', '🇵🇳': 'PN', '🇵🇷': 'PR', '🇵🇸': 'PS', '🇵🇹': 'PT', '🇵🇼': 'PW', '🇵🇾': 'PY', '🇶🇦': 'QA', '🇷🇪': 'RE', '🇷🇴': 'RO', '🇷🇸': 'RS', '🇷🇺': 'RU', '🇷🇼': 'RW', '🇸🇦': 'SA', '🇸🇧': 'SB', '🇸🇨': 'SC', '🇸🇩': 'SD', '🇸🇪': 'SE', '🇸🇬': 'SG', '🇸🇭': 'SH', '🇸🇮': 'SI', '🇸🇯': 'SJ', '🇸🇰': 'SK', '🇸🇱': 'SL', '🇸🇲': 'SM', '🇸🇳': 'SN', '🇸🇴': 'SO', '🇸🇷': 'SR', '🇸🇸': 'SS', '🇸🇹': 'ST', '🇸🇻': 'SV', '🇸🇽': 'SX', '🇸🇾': 'SY', '🇸🇿': 'SZ', '🇹🇨': 'TC', '🇹🇩': 'TD', '🇹🇫': 'TF', '🇹🇬': 'TG', '🇹🇭': 'TH', '🇹🇯': 'TJ', '🇹🇰': 'TK', '🇹🇱': 'TL', '🇹🇲': 'TM', '🇹🇳': 'TN', '🇹🇴': 'TO', '🇹🇷': 'TR', '🇹🇹': 'TT', '🇹🇻': 'TV', '🇹🇼': 'TW', '🇹🇿': 'TZ', '🇺🇦': 'UA', '🇺🇬': 'UG', '🇺🇲': 'UM', '🇺🇸': 'US', '🇺🇾': 'UY', '🇺🇿': 'UZ', '🇻🇦': 'VA', '🇻🇨': 'VC', '🇻🇪': 'VE', '🇻🇬': 'VG', '🇻🇮': 'VI', '🇻🇳': 'VN', '🇻🇺': 'VU', '🇼🇫': 'WF', '🇼🇸': 'WS', '🇾🇪': 'YE', '🇾🇹': 'YT', '🇿🇦': 'ZA', '🇿🇲': 'ZM', '🇿🇼': 'ZW'} + + + +@dataclass +class Country: + alpha_2: str + alpha_3: str + name: str + numeric: int + emoji: str + + @classmethod + def by_pycountry(cls, country: CountryTyping, emoji: str = "") -> "Country": + emoji = "" + alpha_2 = country.alpha_2.upper() + + if alpha_2 in emoji_map: + emoji=emoji_map[alpha_2] + + + return cls( + alpha_2=country.alpha_2, + alpha_3=country.alpha_3, + name=country.name, + numeric=country.numeric, + emoji=emoji + ) + + @classmethod + def by_alpha_2(cls, alpha_2: str) -> "Country": + return cls.by_pycountry(pycountry.countries.get(alpha_2=alpha_2)) + + @classmethod + def by_apha_3(cls, alpha_3: str) -> "Country": + return cls.by_pycountry(pycountry.countries.get(alpha_3=alpha_3)) + + @classmethod + def by_numeric(cls, numeric: int) -> "Country": + return cls.by_pycountry(pycountry.countries.get(numeric=numeric)) + + @classmethod + def by_name(cls, name: str) -> "Country": + return cls.by_pycountry(pycountry.countries.get(name=name)) + + @classmethod + def by_emoji(cls, flag: str) -> "Country": + return cls.by_alpha_2(emoji_map[flag]) + + @classmethod + def by_official_name(cls, official_name: str) -> "Country": + return cls.by_pycountry(pycountry.countries.get(official_name=official_name)) + + def __hash__(self) -> int: + return hash(self.alpha_3) + + def __eq__(self, __value: object) -> bool: + return self.__hash__() == __value.__hash__() \ No newline at end of file diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index b562686..f8d82bb 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -1,5 +1,5 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum from .musify import Musify -from .youtube import YouTube +from .youtube.youtube import YouTube from .abstract import Page, INDEPENDENT_DB_OBJECTS diff --git a/src/music_kraken/cli/options/invidious/__init__.py b/src/music_kraken/pages/youtube/__init__.py similarity index 100% rename from src/music_kraken/cli/options/invidious/__init__.py rename to src/music_kraken/pages/youtube/__init__.py diff --git a/src/music_kraken/pages/youtube/piped.py b/src/music_kraken/pages/youtube/piped.py new file mode 100644 index 0000000..e69de29 diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube/youtube.py similarity index 97% rename from src/music_kraken/pages/youtube.py rename to src/music_kraken/pages/youtube/youtube.py index 40c62b5..5c7eba1 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube/youtube.py @@ -5,9 +5,9 @@ from enum import Enum import sponsorblock from sponsorblock.errors import HTTPException, NotFoundException -from ..objects import Source, DatabaseObject, Song, Target -from .abstract import Page -from ..objects import ( +from ...objects import Source, DatabaseObject, Song, Target +from ..abstract import Page +from ...objects import ( Artist, Source, SourcePages, @@ -18,9 +18,9 @@ from ..objects import ( FormattedText, ID3Timestamp ) -from ..connection import Connection -from ..utils.support_classes import DownloadResult -from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK +from ...connection import Connection +from ...utils.support_classes import DownloadResult +from ...utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK """ diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index d6e8225..8996f38 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -67,18 +67,22 @@ class ConnectionSection(Section): # INVIDIOUS INSTANCES LIST self.INVIDIOUS_INSTANCE = UrlStringAttribute( name="invidious_instance", - description="This is a List, where you can define the invidious instances,\n" + description="This is an attribute, where you can define the invidious instances,\n" "the youtube downloader should use.\n" "Here is a list of active ones: https://docs.invidious.io/instances/\n" "Instances that use cloudflare or have source code changes could cause issues.\n" "Hidden instances (.onion) will only work, when setting 'tor=true'.", value="https://yt.artemislena.eu/" ) - # INVIDIOUS PROXY - self.INVIDIOUS_PROXY_VIDEOS = BoolAttribute( - name="invidious_proxy_video", - value="false", - description="Downloads the videos using the given instances." + + self.PIPED_INSTANCE = UrlStringAttribute( + name="piped_instance", + description="This is an attribute, where you can define the pioed instances,\n" + "the youtube downloader should use.\n" + "Here is a list of active ones: https://github.com/TeamPiped/Piped/wiki/Instances\n" + "Instances that use cloudflare or have source code changes could cause issues.\n" + "Hidden instances (.onion) will only work, when setting 'tor=true'.", + value="https://pipedapi.kavin.rocks" ) self.SPONSOR_BLOCK = BoolAttribute( @@ -93,7 +97,7 @@ class ConnectionSection(Section): self.CHUNK_SIZE, self.SHOW_DOWNLOAD_ERRORS_THRESHOLD, self.INVIDIOUS_INSTANCE, - self.INVIDIOUS_PROXY_VIDEOS, + self.PIPED_INSTANCE, self.SPONSOR_BLOCK ] From 6a44cb827b9130138bb3721b5fb20e6b16a63aa2 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 19 Jun 2023 14:54:09 +0200 Subject: [PATCH 69/81] added a collection of youtube and its frontend mirrors --- src/music_kraken/cli/options/frontend.py | 5 ++-- src/music_kraken/utils/config/base_classes.py | 3 ++ src/music_kraken/utils/config/connection.py | 29 +++++++++++++++++++ src/music_kraken/utils/shared.py | 4 ++- 4 files changed, 38 insertions(+), 3 deletions(-) diff --git a/src/music_kraken/cli/options/frontend.py b/src/music_kraken/cli/options/frontend.py index 34395f4..809ebe8 100644 --- a/src/music_kraken/cli/options/frontend.py +++ b/src/music_kraken/cli/options/frontend.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from collections import defaultdict from ...objects import Country -from ...utils import config, write +from ...utils import config, write, shared from ...connection import Connection @@ -34,6 +34,8 @@ class FrontendInstance: def add_instance(self, instance: Instance): self.all_instances.append(instance) + config.set_name_to_value("youtube_url", instance.uri) + for region in instance.regions: self.region_instances[region].append(instance) @@ -118,7 +120,6 @@ class Invidious(FrontendInstance): self.add_instance(instance) def fetch(self, silent: bool): - print("jhdflashfö") r = self.connection.get(self.endpoint) if r is None: return diff --git a/src/music_kraken/utils/config/base_classes.py b/src/music_kraken/utils/config/base_classes.py index f14ebb8..b5fcbce 100644 --- a/src/music_kraken/utils/config/base_classes.py +++ b/src/music_kraken/utils/config/base_classes.py @@ -144,6 +144,9 @@ class ListAttribute(Attribute): self.value = [] self.has_default_values = False + if value in self.value: + return + self.value.append(value) def __str__(self): diff --git a/src/music_kraken/utils/config/connection.py b/src/music_kraken/utils/config/connection.py index 8996f38..43fc782 100644 --- a/src/music_kraken/utils/config/connection.py +++ b/src/music_kraken/utils/config/connection.py @@ -31,6 +31,22 @@ class UrlStringAttribute(StringAttribute): return urlparse(self.value) +class UrlListAttribute(ListAttribute): + def validate(self, value: str): + v = value.strip() + url = re.match(URL_PATTERN, v) + if url is None: + raise SettingValueError( + setting_name=self.name, + setting_value=v, + rule="has to be a valid url" + ) + + def single_object_from_element(self, value: str): + return urlparse(value) + + + class ConnectionSection(Section): def __init__(self): self.PROXIES = ProxAttribute( @@ -85,6 +101,18 @@ class ConnectionSection(Section): value="https://pipedapi.kavin.rocks" ) + self.ALL_YOUTUBE_URLS = UrlListAttribute( + name="youtube_url", + description="This is used to detect, if an url is from youtube, or any alternativ frontend.\n" + "If any instance seems to be missing, run music kraken with the -f flag.", + value=[ + "https://www.youtube.com/", + "https://www.youtu.be/", + "https://redirect.invidious.io/", + "https://piped.kavin.rocks/" + ] + ) + self.SPONSOR_BLOCK = BoolAttribute( name="use_sponsor_block", value="true", @@ -98,6 +126,7 @@ class ConnectionSection(Section): self.SHOW_DOWNLOAD_ERRORS_THRESHOLD, self.INVIDIOUS_INSTANCE, self.PIPED_INSTANCE, + self.ALL_YOUTUBE_URLS, self.SPONSOR_BLOCK ] diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index addcca7..22a361e 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -101,4 +101,6 @@ ALBUM_TYPE_BLACKLIST: Set[AlbumType] = set(AUDIO_SECTION.ALBUM_TYPE_BLACKLIST.ob THREADED = False ENABLE_RESULT_HISTORY: bool = MISC_SECTION.ENABLE_RESULT_HISTORY.object_from_value -HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value \ No newline at end of file +HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value + +ALL_YOUTUBE_URLS: List[ParseResult] = CONNECTION_SECTION.ALL_YOUTUBE_URLS.object_from_value \ No newline at end of file From 8278449ea39614fe64e9150f71431a1cd02bd58e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 19 Jun 2023 15:10:57 +0200 Subject: [PATCH 70/81] small adjustment --- src/music_kraken/objects/source.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/music_kraken/objects/source.py b/src/music_kraken/objects/source.py index 1680821..a55dbd0 100644 --- a/src/music_kraken/objects/source.py +++ b/src/music_kraken/objects/source.py @@ -4,6 +4,7 @@ from typing import List, Dict, Set, Tuple, Optional from urllib.parse import urlparse from ..utils.enums.source import SourcePages, SourceTypes +from ..utils.shared import ALL_YOUTUBE_URLS from .metadata import Mapping, Metadata from .parents import DatabaseObject from .collection import Collection @@ -53,7 +54,7 @@ class Source(DatabaseObject): if "musify" in parsed.netloc: return cls(SourcePages.MUSIFY, url, referer_page=referer_page) - if url.startswith("https://www.youtube"): + if parsed.netloc in [url.netloc for url in ALL_YOUTUBE_URLS]: return cls(SourcePages.YOUTUBE, url, referer_page=referer_page) if url.startswith("https://www.deezer"): From 2a7452dc44f530521103ff8009c974fc9e86584c Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 19 Jun 2023 16:27:59 +0200 Subject: [PATCH 71/81] implemented the ability to fetch playlists from piped --- src/music_kraken/pages/__init__.py | 2 +- .../pages/{youtube => }/youtube.py | 73 ++++++++++++++++--- src/music_kraken/pages/youtube/__init__.py | 0 src/music_kraken/pages/youtube/piped.py | 0 src/music_kraken/utils/shared.py | 5 +- 5 files changed, 65 insertions(+), 15 deletions(-) rename src/music_kraken/pages/{youtube => }/youtube.py (85%) delete mode 100644 src/music_kraken/pages/youtube/__init__.py delete mode 100644 src/music_kraken/pages/youtube/piped.py diff --git a/src/music_kraken/pages/__init__.py b/src/music_kraken/pages/__init__.py index f8d82bb..b562686 100644 --- a/src/music_kraken/pages/__init__.py +++ b/src/music_kraken/pages/__init__.py @@ -1,5 +1,5 @@ from .encyclopaedia_metallum import EncyclopaediaMetallum from .musify import Musify -from .youtube.youtube import YouTube +from .youtube import YouTube from .abstract import Page, INDEPENDENT_DB_OBJECTS diff --git a/src/music_kraken/pages/youtube/youtube.py b/src/music_kraken/pages/youtube.py similarity index 85% rename from src/music_kraken/pages/youtube/youtube.py rename to src/music_kraken/pages/youtube.py index 5c7eba1..55f6ed3 100644 --- a/src/music_kraken/pages/youtube/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -5,9 +5,9 @@ from enum import Enum import sponsorblock from sponsorblock.errors import HTTPException, NotFoundException -from ...objects import Source, DatabaseObject, Song, Target -from ..abstract import Page -from ...objects import ( +from ..objects import Source, DatabaseObject, Song, Target +from .abstract import Page +from ..objects import ( Artist, Source, SourcePages, @@ -18,9 +18,9 @@ from ...objects import ( FormattedText, ID3Timestamp ) -from ...connection import Connection -from ...utils.support_classes import DownloadResult -from ...utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK +from ..connection import Connection +from ..utils.support_classes import DownloadResult +from ..utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_SPONSOR_BLOCK, PIPED_INSTANCE """ @@ -34,6 +34,9 @@ from ...utils.shared import YOUTUBE_LOGGER, INVIDIOUS_INSTANCE, BITRATE, ENABLE_ def get_invidious_url(path: str = "", params: str = "", query: str = "", fragment: str = "") -> str: return urlunparse((INVIDIOUS_INSTANCE.scheme, INVIDIOUS_INSTANCE.netloc, path, params, query, fragment)) +def get_piped_url(path: str = "", params: str = "", query: str = "", fragment: str = "") -> str: + return urlunparse((PIPED_INSTANCE.scheme, PIPED_INSTANCE.netloc, path, params, query, fragment)) + class YouTubeUrlType(Enum): CHANNEL = "channel" @@ -137,6 +140,11 @@ class YouTube(Page): host=get_invidious_url(), logger=self.LOGGER ) + + self.piped_connection: Connection = Connection( + host=get_piped_url(), + logger=self.LOGGER + ) self.download_connection: Connection = Connection( host="https://www.youtube.com/", @@ -294,17 +302,13 @@ class YouTube(Page): date=ID3Timestamp.fromtimestamp(round(sum(timestamps) / len(timestamps))) ) - def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: - parsed = YouTubeUrl(source.url) - if parsed.url_type != YouTubeUrlType.CHANNEL: - return Artist(source_list=[source]) - + def fetch_invidious_album_list(self, yt_id: str): artist_name = None album_list = [] # playlist # https://yt.artemislena.eu/api/v1/channels/playlists/UCV0Ntl3lVR7xDXKoCU6uUXA - r = self.connection.get(get_invidious_url(f"/api/v1/channels/playlists/{parsed.id}")) + r = self.connection.get(get_invidious_url(f"/api/v1/channels/playlists/{yt_id}")) if r is None: return Artist() @@ -328,6 +332,51 @@ class YouTube(Page): )] )) + return album_list, artist_name + + def fetch_piped_album_list(self, yt_id: str): + endpoint = get_piped_url(path=f"/channels/tabs", query='data={"originalUrl":"https://www.youtube.com/' + yt_id + '/playlists","url":"https://www.youtube.com/' + yt_id + 'playlists","id":"' + yt_id + '","contentFilters":["playlists"],"sortFilter":"","baseUrl":"https://www.youtube.com"}') + + r = self.piped_connection.get(endpoint) + if r is None: + return [], None + + content = r.json()["content"] + + artist_name = None + album_list = [] + + for playlist in content: + if playlist["type"] != "playlist": + continue + + artist_name = playlist["uploaderName"].replace(" - Topic", "") + + album_list.append(Album( + title=playlist["name"], + source_list=[Source( + self.SOURCE_TYPE, get_invidious_url() + playlist["url"] + )], + artist_list=[Artist( + name=artist_name, + source_list=[ + Source(self.SOURCE_TYPE, get_invidious_url(path=playlist["uploaderUrl"])) + ] + )] + )) + + return album_list, artist_name + + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + parsed = YouTubeUrl(source.url) + if parsed.url_type != YouTubeUrlType.CHANNEL: + return Artist(source_list=[source]) + + album_list, artist_name = self.fetch_piped_album_list(parsed.id) + if len(album_list) <= 0: + self.LOGGER.warning(f"didn't found any playlists with piped, falling back to invidious. (it is unusual)") + album_list, artist_name = self.fetch_invidious_album_list(parsed.id) + return Artist(name=artist_name, main_album_list=album_list, source_list=[source]) def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: diff --git a/src/music_kraken/pages/youtube/__init__.py b/src/music_kraken/pages/youtube/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/music_kraken/pages/youtube/piped.py b/src/music_kraken/pages/youtube/piped.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index 22a361e..f9bf8a2 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -83,6 +83,9 @@ if TOR: 'https': f'socks5h://127.0.0.1:{CONNECTION_SECTION.TOR_PORT.object_from_value}' } INVIDIOUS_INSTANCE: ParseResult = CONNECTION_SECTION.INVIDIOUS_INSTANCE.object_from_value +PIPED_INSTANCE: ParseResult = CONNECTION_SECTION.PIPED_INSTANCE.object_from_value + +ALL_YOUTUBE_URLS: List[ParseResult] = CONNECTION_SECTION.ALL_YOUTUBE_URLS.object_from_value ENABLE_SPONSOR_BLOCK: bool = CONNECTION_SECTION.SPONSOR_BLOCK.object_from_value # size of the chunks that are streamed @@ -102,5 +105,3 @@ THREADED = False ENABLE_RESULT_HISTORY: bool = MISC_SECTION.ENABLE_RESULT_HISTORY.object_from_value HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value - -ALL_YOUTUBE_URLS: List[ParseResult] = CONNECTION_SECTION.ALL_YOUTUBE_URLS.object_from_value \ No newline at end of file From f58534155f157e38e5c374c029bdc8d30ddd556f Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Mon, 19 Jun 2023 16:30:29 +0200 Subject: [PATCH 72/81] addedlogging message --- src/music_kraken/pages/youtube.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 55f6ed3..1ef1323 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -250,6 +250,8 @@ class YouTube(Page): return song def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + self.LOGGER.info(f"Getting the metadata of an album may take slightly longer, only panic in a couple minutes <333") + parsed = YouTubeUrl(source.url) if parsed.url_type != YouTubeUrlType.PLAYLIST: return Album() From 7eee1a8ace53c5cd0c8d1e8ef2ceebcc9454ee7e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 12:03:08 +0200 Subject: [PATCH 73/81] continued musify fetching from song --- documentation/html/musify/song_details.html | 204 ++++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 documentation/html/musify/song_details.html diff --git a/documentation/html/musify/song_details.html b/documentation/html/musify/song_details.html new file mode 100644 index 0000000..57c7368 --- /dev/null +++ b/documentation/html/musify/song_details.html @@ -0,0 +1,204 @@ + +

+
+
+
+ + + + + +
    +
  1. + + Главная + + +
    2. +
  2. + + Исполнители + + +
    3. + +
  3. + + Linkin Park + + +
    4. + +
  4. + + Numb + + +
    5. + +
  5. Numb
    6. +
+ + + +
+
+ +
+ Numb + +
+ +
+
+
+
+ + Скачать MP3 +
+
+
+ +

+ #Alternative +

+
+
+ +
+ + +
+
+
+
+

Текст "Linkin Park - Numb"

+
+ I'm tired of being what you want me to be + Feeling so faithless lost under the surface + Don't know what you're expecting of me + Put under the pressure of walking in your shoes + (Caught in the undertone just caught in the undertone) + Every step I take is another mistake to you + (Caught in the undertone just caught in the undertone) + + I've become so numb I can't feel you there + I've become so tired so much more aware + I've becoming this all I want to do + Is be more like me and be less like you + + Can't you see that you're smothering me + Holding too tightly afraid to lose control + Cause everything that you thought I would be + Has fallen apart right in front of you + (Caught in the undertone just caught in the undertone) + Every step that I take is another mistake to you + (Caught in the undertone just caught in the undertone) + And every second I waste is more than I can take + + I've become so numb I can't feel you there + I've become so tired so much more aware + I've becoming this all I want to do + Is be more like me and be less like you + + And I know + I may end up failing too + But I know + You were just like me with someone disappointed in you + + I've become so numb I can't feel you there + I've become so tired so much more aware + I've becoming this all I want to do + Is be more like me and be less like you + + I've become so numb I can't feel you there + Is everything what you want me to be + I've become so numb I can't feel you there + Is everything what you want me to be +
+ Обновить текст +
+
+
+ +
+
+

Смотреть видео клип "Linkin Park - + Numb" онлайн

+
+
+ +
+
+ +
+
+
+
+
+
+
+
+
+
+ + \ No newline at end of file From 4c3bce9e0c9ea661a235a475d65f5ee092a4fbd1 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 12:03:11 +0200 Subject: [PATCH 74/81] actuall continued --- documentation/html/musify/details.md | 3 + src/actual_donwload.py | 6 +- src/music_kraken/objects/lyrics.py | 2 +- src/music_kraken/pages/musify.py | 91 +++++++++++++++++++++++++--- 4 files changed, 91 insertions(+), 11 deletions(-) create mode 100644 documentation/html/musify/details.md diff --git a/documentation/html/musify/details.md b/documentation/html/musify/details.md new file mode 100644 index 0000000..3427cf0 --- /dev/null +++ b/documentation/html/musify/details.md @@ -0,0 +1,3 @@ +title | url +--- | --- +song | https://musify.club/track/linkin-park-numb-210765 \ No newline at end of file diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 9e8bd89..ac8779e 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -12,10 +12,14 @@ if __name__ == "__main__": "d: https://musify.club/release/crystal-f-x-2012-795181" ] + fetch_musify_song = [ + "s: https://musify.club/track/linkin-park-numb-210765" + ] + youtube_search = [ "s: #a Zombiez", "10", "d: 5" ] - music_kraken.cli(genre="test", command_list=youtube_search) + music_kraken.cli(genre="test", command_list=fetch_musify_song) diff --git a/src/music_kraken/objects/lyrics.py b/src/music_kraken/objects/lyrics.py index d2ba425..465b96d 100644 --- a/src/music_kraken/objects/lyrics.py +++ b/src/music_kraken/objects/lyrics.py @@ -17,7 +17,7 @@ class Lyrics(DatabaseObject): def __init__( self, text: FormattedText, - language: pycountry.Languages, + language: pycountry.Languages = pycountry.languages.get(alpha_2="en"), _id: str = None, dynamic: bool = False, source_list: List[Source] = None, diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index ed045b6..51560b7 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -20,7 +20,8 @@ from ..objects import ( FormattedText, Label, Target, - DatabaseObject + DatabaseObject, + Lyrics ) from ..utils.shared import MUSIFY_LOGGER from ..utils import string_processing, shared @@ -381,7 +382,85 @@ class Musify(Page): return search_results def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: - return Song() + # https://musify.club/track/linkin-park-numb-210765 + r = self.connection.get(source.url) + if r is None: + return Song + + soup = self.get_soup_from_response(r) + + track_name: str = None + source_list: List[Source] = [source] + lyrics_list: List[Lyrics] = [] + artist_list: List[Artist] = [] + album_list: List[Album] = [] + + # breadcrums + breadcrumb_list_element_list: List[BeautifulSoup] = soup.find_all("ol", {"class": "breadcrumb"}) + for breadcrumb_list_element in breadcrumb_list_element_list: + list_points: List[BeautifulSoup] = breadcrumb_list_element.find_all("li", "breadcrumb-item") + if len(list_points) != 5: + self.LOGGER.warning(f"breadcrumbs of song doesn't have 5 items: {breadcrumb_list_element.prettify()}") + break + + artist_anchor: BeautifulSoup = list_points[2].find("a") + if artist_anchor is not None: + artist_src_list = [] + artist_name = None + + href = artist_anchor["href"] + if href is not None: + artist_src_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) + + name_elem: BeautifulSoup = artist_anchor.find("span", {"itemprop": "name"}) + if name_elem is not None: + artist_name = name_elem.text.strip() + + artist_list.append(Artist(name=artist_name, source_list=artist_src_list)) + + album_anchor: BeautifulSoup = list_points[3].find("a") + if album_anchor is not None: + album_source_list = [] + album_name = None + + href = artist_anchor["href"] + if href is not None: + album_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) + + name_elem: BeautifulSoup = album_anchor.find("span", {"itemprop": "name"}) + if name_elem is not None: + album_name = name_elem.text.strip() + + album_list.append(Album(title=album_name, source_list=album_source_list)) + + + track_name = list_points[4].text.strip() + + + # lyrics + lyrics_container: List[BeautifulSoup] = soup.find_all("div", {"id": "tabLyrics"}) + for lyrics in lyrics_container: + lyrics_text = lyrics.find("div", {"style": "white-space: pre-line"}).text.strip() + lyrics_list.append(Lyrics(text=FormattedText(html=lyrics_text))) + + # youtube video + video_container_list: List[BeautifulSoup] = soup.find_all("div", {"id": "tabVideo"}) + for video_container in video_container_list: + iframe_list: List[BeautifulSoup] = video_container.findAll("iframe") + for iframe in iframe_list: + source_list.append(Source( + SourcePages.YOUTUBE, + iframe["src"], + referer_page=self.SOURCE_TYPE + )) + + return Song( + title=track_name, + source_list=source_list, + lyrics_list=lyrics_list, + main_artist_list=artist_list, + album_list=album_list, + ) def _parse_song_card(self, song_card: BeautifulSoup) -> Song: """ @@ -1007,10 +1086,4 @@ class Musify(Page): self.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - r = self.connection.get(endpoint, stream=True, raw_url=True) - if r is None: - return DownloadResult(error_message=f"couldn't connect to {endpoint}") - - if target.stream_into(r, desc=desc): - return DownloadResult(total=1) - return DownloadResult(error_message=f"Streaming to the file went wrong: {endpoint}, {str(target.file_path)}") + return self.connection.stream_into(endpoint, target, stream=True, raw_url=True) From d5717c9a6bd3a559447abd7a39e7df92fe418a3e Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 12:28:48 +0200 Subject: [PATCH 75/81] continued musify --- src/music_kraken/pages/musify.py | 76 ++++++++++++++++++++------------ 1 file changed, 47 insertions(+), 29 deletions(-) diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index 51560b7..da1bb0c 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -395,6 +395,49 @@ class Musify(Page): artist_list: List[Artist] = [] album_list: List[Album] = [] + def _parse_artist_anchor(artist_soup: BeautifulSoup): + nonlocal artist_list + if artist_anchor is None: + return + + artist_src_list = [] + artist_name = None + + href = artist_soup["href"] + if href is not None: + artist_src_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) + + name_elem: BeautifulSoup = artist_soup.find("span", {"itemprop": "name"}) + if name_elem is not None: + artist_name = name_elem.text.strip() + + artist_list.append(Artist(name=artist_name, source_list=artist_src_list)) + + def _parse_album_anchor(album_soup: BeautifulSoup): + nonlocal album_list + if album_anchor is None: + return + album_source_list = [] + album_name = None + + href = album_soup["href"] + if href is not None: + album_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) + + name_elem: BeautifulSoup = album_soup.find("span", {"itemprop": "name"}) + if name_elem is not None: + album_name = name_elem.text.strip() + + album_list.append(Album(title=album_name, source_list=album_source_list)) + + # download url + anchor: BeautifulSoup + for anchor in soup.find_all("a", {"itemprop": "audio"}): + href = anchor["href"] + if href is not None: + source.audio_url = self.HOST + href + + # breadcrums breadcrumb_list_element_list: List[BeautifulSoup] = soup.find_all("ol", {"class": "breadcrumb"}) for breadcrumb_list_element in breadcrumb_list_element_list: @@ -404,36 +447,11 @@ class Musify(Page): break artist_anchor: BeautifulSoup = list_points[2].find("a") - if artist_anchor is not None: - artist_src_list = [] - artist_name = None - - href = artist_anchor["href"] - if href is not None: - artist_src_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) - - name_elem: BeautifulSoup = artist_anchor.find("span", {"itemprop": "name"}) - if name_elem is not None: - artist_name = name_elem.text.strip() - - artist_list.append(Artist(name=artist_name, source_list=artist_src_list)) - + _parse_artist_anchor(artist_anchor) + album_anchor: BeautifulSoup = list_points[3].find("a") - if album_anchor is not None: - album_source_list = [] - album_name = None - - href = artist_anchor["href"] - if href is not None: - album_source_list.append(Source(self.SOURCE_TYPE, self.HOST + href)) - - name_elem: BeautifulSoup = album_anchor.find("span", {"itemprop": "name"}) - if name_elem is not None: - album_name = name_elem.text.strip() - - album_list.append(Album(title=album_name, source_list=album_source_list)) - - + _parse_album_anchor(album_anchor) + track_name = list_points[4].text.strip() From 0a8a96c5468f864ba9f0e8129508b41c772160b2 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 12:52:00 +0200 Subject: [PATCH 76/81] finished fetching of single song --- src/actual_donwload.py | 2 +- src/music_kraken/pages/musify.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index ac8779e..bafae70 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -13,7 +13,7 @@ if __name__ == "__main__": ] fetch_musify_song = [ - "s: https://musify.club/track/linkin-park-numb-210765" + "s: https://musify.club/track/blokkmonsta-schwartz-crystal-f-purer-hass-8369115" ] youtube_search = [ diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index da1bb0c..e732969 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -397,7 +397,7 @@ class Musify(Page): def _parse_artist_anchor(artist_soup: BeautifulSoup): nonlocal artist_list - if artist_anchor is None: + if artist_soup is None: return artist_src_list = [] @@ -437,6 +437,22 @@ class Musify(Page): if href is not None: source.audio_url = self.HOST + href + # song detail + album_info: BeautifulSoup + for album_info in soup.find_all("ul", {"class": "album-info"}): + list_element: BeautifulSoup = album_info.find("li") + + if list_element is not None: + artist_soup: BeautifulSoup + for artist_soup in list_element.find_all("a"): + artist_source_list = [] + href = artist_soup["href"] + if href is not None: + artist_source_list = [Source(self.SOURCE_TYPE, self.HOST + href)] + artist_list.append(Artist( + name=artist_soup.text.strip(), + source_list=artist_source_list + )) # breadcrums breadcrumb_list_element_list: List[BeautifulSoup] = soup.find_all("ol", {"class": "breadcrumb"}) From 08b47b7f598d68c1ad50f7df70366158d6fad8e8 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 13:39:06 +0200 Subject: [PATCH 77/81] musify --- src/music_kraken/pages/musify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/music_kraken/pages/musify.py b/src/music_kraken/pages/musify.py index e732969..93fc01a 100644 --- a/src/music_kraken/pages/musify.py +++ b/src/music_kraken/pages/musify.py @@ -1120,4 +1120,4 @@ class Musify(Page): self.LOGGER.warning(f"The source has no audio link. Falling back to {endpoint}.") - return self.connection.stream_into(endpoint, target, stream=True, raw_url=True) + return self.connection.stream_into(endpoint, target, raw_url=True) From f95083050e04c48f6afbe04b2298ecbc57e6e908 Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Tue, 20 Jun 2023 16:40:34 +0200 Subject: [PATCH 78/81] refactored cli --- src/music_kraken/__init__.py | 115 +----------------- src/music_kraken/__main__.py | 57 ++++----- src/music_kraken/cli/__init__.py | 6 +- src/music_kraken/cli/download/__init__.py | 0 src/music_kraken/cli/informations/paths.py | 3 + .../{download/shell.py => main_downloader.py} | 63 +++++----- src/music_kraken/cli/options/frontend.py | 11 +- src/music_kraken/cli/options/settings.py | 71 +++++++++++ src/music_kraken/cli/utils.py | 32 +++++ src/music_kraken/utils/__init__.py | 5 +- src/music_kraken/utils/config/__init__.py | 17 ++- src/music_kraken/utils/config/config.py | 13 -- src/music_kraken/utils/shared.py | 15 +++ 13 files changed, 209 insertions(+), 199 deletions(-) delete mode 100644 src/music_kraken/cli/download/__init__.py rename src/music_kraken/cli/{download/shell.py => main_downloader.py} (90%) create mode 100644 src/music_kraken/cli/options/settings.py create mode 100644 src/music_kraken/cli/utils.py diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index 28d505a..03e84a5 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -6,12 +6,7 @@ from typing import List import gc import musicbrainzngs -from .cli import Shell -from . import objects, pages, download -from .utils import exception, shared, path_manager -from .utils.config import config, read, write, PATHS_SECTION -from .utils.shared import MUSIC_DIR, MODIFY_GC, NOT_A_GENRE_REGEX, get_random_message -from .utils.string_processing import fit_to_file_system +from .utils.shared import MODIFY_GC if MODIFY_GC: @@ -31,111 +26,3 @@ if MODIFY_GC: logging.getLogger("musicbrainzngs").setLevel(logging.WARNING) musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeIIow2/music-downloader") - -URL_REGEX = 'https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+' -DOWNLOAD_COMMANDS = { - "ok", - "download", - "\\d", - "hs" -} - -EXIT_COMMANDS = { - "exit", - "quit" -} - - -def print_cute_message(): - message = get_random_message() - try: - print(message) - except UnicodeEncodeError: - message = str(c for c in message if 0 < ord(c) < 127) - print(message) - - -def exit_message(): - print() - print_cute_message() - print("See you soon! :3") - - -def settings( - name: str = None, - value: str = None, -): - def modify_setting(_name: str, _value: str, invalid_ok: bool = True) -> bool: - try: - config.set_name_to_value(_name, _value) - except exception.config.SettingException as e: - if invalid_ok: - print(e) - return False - else: - raise e - - write() - return True - - def print_settings(): - for i, attribute in enumerate(config): - print(f"{i:0>2}: {attribute.name}={attribute.value}") - - def modify_setting_by_index(index: int) -> bool: - attribute = list(config)[index] - - print() - print(attribute) - - input__ = input(f"{attribute.name}=") - if not modify_setting(attribute.name, input__.strip()): - return modify_setting_by_index(index) - - return True - - if name is not None and value is not None: - modify_setting(name, value, invalid_ok=True) - - print() - print_cute_message() - print() - return - - while True: - print_settings() - - input_ = input("Id of setting to modify: ") - print() - if input_.isdigit() and int(input_) < len(config): - if modify_setting_by_index(int(input_)): - print() - print_cute_message() - print() - return - else: - print("Please input a valid ID.") - print() - - -def cli( - genre: str = None, - download_all: bool = False, - direct_download_url: str = None, - command_list: List[str] = None -): - shell = Shell(genre=genre) - - if command_list is not None: - for command in command_list: - shell.process_input(command) - return - - if direct_download_url is not None: - if shell.download(direct_download_url, download_all=download_all): - exit_message() - return - - shell.mainloop() - - exit_message() diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 776622b..7a249a8 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -80,44 +80,39 @@ if __name__ == "__main__": print("Setting logging-level to DEBUG") logging.getLogger().setLevel(logging.DEBUG) - import music_kraken - - music_kraken.read() - - if arguments.setting is not None: - music_kraken.settings(*arguments.setting) - exit() - - if arguments.settings: - music_kraken.settings() - exit() - - if arguments.paths: - music_kraken.paths() - exit() - + from . import cli + from .utils.config import read_config + from .utils import shared + if arguments.r: import os - if os.path.exists(music_kraken.shared.CONFIG_FILE): - os.remove(music_kraken.shared.CONFIG_FILE) - music_kraken.read() + if os.path.exists(shared.CONFIG_FILE): + os.remove(shared.CONFIG_FILE) + read_config() + + exit() + + read_config() + + if arguments.setting is not None: + cli.settings(*arguments.setting) + + if arguments.settings: + cli.settings() + + if arguments.paths: + cli.print_paths() if arguments.frontend: - from .cli.options.frontend import set_frontend - set_frontend() - exit() + cli.set_frontend(silent=False) # getting the genre genre: str = arguments.genre if arguments.test: genre = "test" - try: - music_kraken.cli( - genre=genre, - download_all=arguments.all, - direct_download_url=arguments.url - ) - except KeyboardInterrupt: - print("\n\nRaise an issue if I fucked up:\nhttps://github.com/HeIIow2/music-downloader/issues") - music_kraken.exit_message() + cli.download( + genre=genre, + download_all=arguments.all, + direct_download_url=arguments.url + ) \ No newline at end of file diff --git a/src/music_kraken/cli/__init__.py b/src/music_kraken/cli/__init__.py index c21af81..1dbf213 100644 --- a/src/music_kraken/cli/__init__.py +++ b/src/music_kraken/cli/__init__.py @@ -1 +1,5 @@ -from .download.shell import Shell \ No newline at end of file +from .informations import print_paths +from .main_downloader import download +from .options.settings import settings +from .options.frontend import set_frontend + diff --git a/src/music_kraken/cli/download/__init__.py b/src/music_kraken/cli/download/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/music_kraken/cli/informations/paths.py b/src/music_kraken/cli/informations/paths.py index d6bff30..cb1ba98 100644 --- a/src/music_kraken/cli/informations/paths.py +++ b/src/music_kraken/cli/informations/paths.py @@ -1,3 +1,5 @@ +from ..utils import cli_function + from ...utils.path_manager import LOCATIONS from ...utils import shared @@ -12,6 +14,7 @@ def all_paths(): } +@cli_function def print_paths(): for name, path in all_paths().items(): print(f"{name}:\t{path}") \ No newline at end of file diff --git a/src/music_kraken/cli/download/shell.py b/src/music_kraken/cli/main_downloader.py similarity index 90% rename from src/music_kraken/cli/download/shell.py rename to src/music_kraken/cli/main_downloader.py index 03e804f..b9076ca 100644 --- a/src/music_kraken/cli/download/shell.py +++ b/src/music_kraken/cli/main_downloader.py @@ -2,14 +2,16 @@ from typing import Set, Type, Dict, List from pathlib import Path import re -from ...utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX, ENABLE_RESULT_HISTORY, HISTORY_LENGTH -from ...utils.regex import URL_PATTERN -from ...utils.string_processing import fit_to_file_system -from ...utils.support_classes import Query, DownloadResult -from ...download.results import Results, SearchResults, Option, PageResults -from ...download.page_attributes import Pages -from ...pages import Page -from ...objects import Song, Album, Artist, DatabaseObject +from .utils import cli_function + +from ..utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX, ENABLE_RESULT_HISTORY, HISTORY_LENGTH, HELP_MESSAGE +from ..utils.regex import URL_PATTERN +from ..utils.string_processing import fit_to_file_system +from ..utils.support_classes import Query, DownloadResult +from ..download.results import Results, SearchResults, Option, PageResults +from ..download.page_attributes import Pages +from ..pages import Page +from ..objects import Song, Album, Artist, DatabaseObject """ @@ -128,30 +130,12 @@ def get_genre(): def help_message(): print() - print(""" -to search: -> s: {query or url} -> s: https://musify.club/release/some-random-release-183028492 -> s: #a {artist} #r {release} #t {track} - -to download: -> d: {option ids or direct url} -> d: 0, 3, 4 -> d: 1 -> d: https://musify.club/release/some-random-release-183028492 - -have fun :3 - """.strip()) + print(HELP_MESSAGE) print() -class Shell: - """ - TODO: - - - Implement search and download for direct urls - """ - + +class Downloader: def __init__( self, exclude_pages: Set[Type[Page]] = None, @@ -388,4 +372,23 @@ class Shell: while True: if self.process_input(input("> ")): return - \ No newline at end of file + +@cli_function +def download( + genre: str = None, + download_all: bool = False, + direct_download_url: str = None, + command_list: List[str] = None +): + shell = Downloader(genre=genre) + + if command_list is not None: + for command in command_list: + shell.process_input(command) + return + + if direct_download_url is not None: + if shell.download(direct_download_url, download_all=download_all): + return + + shell.mainloop() diff --git a/src/music_kraken/cli/options/frontend.py b/src/music_kraken/cli/options/frontend.py index 809ebe8..28f9239 100644 --- a/src/music_kraken/cli/options/frontend.py +++ b/src/music_kraken/cli/options/frontend.py @@ -1,10 +1,11 @@ from typing import Dict, List -import requests from dataclasses import dataclass from collections import defaultdict +from ..utils import cli_function + from ...objects import Country -from ...utils import config, write, shared +from ...utils import config, write_config from ...connection import Connection @@ -45,7 +46,7 @@ class FrontendInstance: def set_instance(self, instance: Instance): config.set_name_to_value(self.SETTING_NAME, instance.uri) - write() + write_config() def _choose_country(self) -> List[Instance]: print("Input the country code, an example would be \"US\"") @@ -123,7 +124,7 @@ class Invidious(FrontendInstance): r = self.connection.get(self.endpoint) if r is None: return - + for instance in r.json(): self._process_instance(all_instance_data=instance) @@ -177,6 +178,8 @@ class FrontendSelection: self.piped.choose(silent) +@cli_function def set_frontend(silent: bool = False): shell = FrontendSelection() shell.choose(silent=silent) + \ No newline at end of file diff --git a/src/music_kraken/cli/options/settings.py b/src/music_kraken/cli/options/settings.py new file mode 100644 index 0000000..3ba0ade --- /dev/null +++ b/src/music_kraken/cli/options/settings.py @@ -0,0 +1,71 @@ +from ..utils import cli_function + +from ...utils.config import config, write_config +from ...utils import exception + + +def modify_setting(_name: str, _value: str, invalid_ok: bool = True) -> bool: + try: + config.set_name_to_value(_name, _value) + except exception.config.SettingException as e: + if invalid_ok: + print(e) + return False + else: + raise e + + write_config() + return True + + +def print_settings(): + for i, attribute in enumerate(config): + print(f"{i:0>2}: {attribute.name}={attribute.value}") + + + def modify_setting_by_index(index: int) -> bool: + attribute = list(config)[index] + + print() + print(attribute) + + input__ = input(f"{attribute.name}=") + if not modify_setting(attribute.name, input__.strip()): + return modify_setting_by_index(index) + + return True + + +def modify_setting_by_index(index: int) -> bool: + attribute = list(config)[index] + + print() + print(attribute) + + input__ = input(f"{attribute.name}=") + if not modify_setting(attribute.name, input__.strip()): + return modify_setting_by_index(index) + + return True + + +@cli_function +def settings( + name: str = None, + value: str = None, +): + if name is not None and value is not None: + modify_setting(name, value, invalid_ok=True) + return + + while True: + print_settings() + + input_ = input("Id of setting to modify: ") + print() + if input_.isdigit() and int(input_) < len(config): + if modify_setting_by_index(int(input_)): + return + else: + print("Please input a valid ID.") + print() \ No newline at end of file diff --git a/src/music_kraken/cli/utils.py b/src/music_kraken/cli/utils.py new file mode 100644 index 0000000..ffd629f --- /dev/null +++ b/src/music_kraken/cli/utils.py @@ -0,0 +1,32 @@ +from ..utils.shared import get_random_message + + +def cli_function(function): + def wrapper(*args, **kwargs): + print_cute_message() + print() + try: + function(*args, **kwargs) + except KeyboardInterrupt: + print("\n\nRaise an issue if I fucked up:\nhttps://github.com/HeIIow2/music-downloader/issues") + + finally: + print() + print_cute_message() + print("See you soon! :3") + + exit() + + return wrapper + + +def print_cute_message(): + message = get_random_message() + try: + print(message) + except UnicodeEncodeError: + message = str(c for c in message if 0 < ord(c) < 127) + print(message) + + + \ No newline at end of file diff --git a/src/music_kraken/utils/__init__.py b/src/music_kraken/utils/__init__.py index fb3447a..89186a6 100644 --- a/src/music_kraken/utils/__init__.py +++ b/src/music_kraken/utils/__init__.py @@ -1,4 +1 @@ -from .config import config, read, write - -# tells what exists -__all__ = ["shared", "object_handeling", "phonetic_compares", "functions"] +from .config import config, read_config, write_config diff --git a/src/music_kraken/utils/config/__init__.py b/src/music_kraken/utils/config/__init__.py index 0313788..3d57aed 100644 --- a/src/music_kraken/utils/config/__init__.py +++ b/src/music_kraken/utils/config/__init__.py @@ -4,7 +4,20 @@ from .connection import CONNECTION_SECTION from .misc import MISC_SECTION from .paths import PATHS_SECTION -from .config import read, write, config +from .paths import LOCATIONS +from .config import Config -read() +config = Config() + + +def read_config(): + if not LOCATIONS.CONFIG_FILE.is_file(): + write_config() + config.read_from_config_file(LOCATIONS.CONFIG_FILE) + + +def write_config(): + config.write_to_config_file(LOCATIONS.CONFIG_FILE) + +set_name_to_value = config.set_name_to_value diff --git a/src/music_kraken/utils/config/config.py b/src/music_kraken/utils/config/config.py index 1ecfe28..50b1ac0 100644 --- a/src/music_kraken/utils/config/config.py +++ b/src/music_kraken/utils/config/config.py @@ -125,16 +125,3 @@ class Config: for section in self._section_list: for name, attribute in section.name_attribute_map.items(): yield attribute - - -config = Config() - - -def read(): - if not LOCATIONS.CONFIG_FILE.is_file(): - write() - config.read_from_config_file(LOCATIONS.CONFIG_FILE) - - -def write(): - config.write_to_config_file(LOCATIONS.CONFIG_FILE) diff --git a/src/music_kraken/utils/shared.py b/src/music_kraken/utils/shared.py index f9bf8a2..bceba4a 100644 --- a/src/music_kraken/utils/shared.py +++ b/src/music_kraken/utils/shared.py @@ -105,3 +105,18 @@ THREADED = False ENABLE_RESULT_HISTORY: bool = MISC_SECTION.ENABLE_RESULT_HISTORY.object_from_value HISTORY_LENGTH: int = MISC_SECTION.HISTORY_LENGTH.object_from_value + +HELP_MESSAGE = """ +to search: +> s: {query or url} +> s: https://musify.club/release/some-random-release-183028492 +> s: #a {artist} #r {release} #t {track} + +to download: +> d: {option ids or direct url} +> d: 0, 3, 4 +> d: 1 +> d: https://musify.club/release/some-random-release-183028492 + +have fun :3 +""".strip() From dc540e48297d5c157d67b01c5f9cbc7615e3c62d Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 20 Jun 2023 17:07:32 +0200 Subject: [PATCH 79/81] fixed issue with reading the config file at the wrong point --- src/actual_donwload.py | 6 +++++- src/music_kraken/__init__.py | 6 ++---- src/music_kraken/cli/main_downloader.py | 12 ++++++++++-- src/music_kraken/objects/source.py | 2 +- src/music_kraken/utils/config/__init__.py | 2 ++ 5 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/actual_donwload.py b/src/actual_donwload.py index bafae70..c60650b 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -15,6 +15,10 @@ if __name__ == "__main__": fetch_musify_song = [ "s: https://musify.club/track/blokkmonsta-schwartz-crystal-f-purer-hass-8369115" ] + + fetch_youtube_playlist = [ + "s: https://yt.artemislena.eu/playlist?list=OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM" + ] youtube_search = [ "s: #a Zombiez", @@ -22,4 +26,4 @@ if __name__ == "__main__": "d: 5" ] - music_kraken.cli(genre="test", command_list=fetch_musify_song) + music_kraken.cli.download(genre="test", command_list=fetch_youtube_playlist) diff --git a/src/music_kraken/__init__.py b/src/music_kraken/__init__.py index 03e84a5..599ad89 100644 --- a/src/music_kraken/__init__.py +++ b/src/music_kraken/__init__.py @@ -1,13 +1,11 @@ import logging -import re -from pathlib import Path -from typing import List import gc import musicbrainzngs +from .utils.config import read_config from .utils.shared import MODIFY_GC - +from . import cli if MODIFY_GC: """ diff --git a/src/music_kraken/cli/main_downloader.py b/src/music_kraken/cli/main_downloader.py index b9076ca..c0affea 100644 --- a/src/music_kraken/cli/main_downloader.py +++ b/src/music_kraken/cli/main_downloader.py @@ -8,7 +8,8 @@ from ..utils.shared import MUSIC_DIR, NOT_A_GENRE_REGEX, ENABLE_RESULT_HISTORY, from ..utils.regex import URL_PATTERN from ..utils.string_processing import fit_to_file_system from ..utils.support_classes import Query, DownloadResult -from ..download.results import Results, SearchResults, Option, PageResults +from ..utils.exception.download import UrlNotFoundException +from ..download.results import Results, Option, PageResults from ..download.page_attributes import Pages from ..pages import Page from ..objects import Song, Album, Artist, DatabaseObject @@ -224,7 +225,14 @@ class Downloader: def search(self, query: str): if re.match(URL_PATTERN, query) is not None: - page, data_object = self.pages.fetch_url(query) + try: + page, data_object = self.pages.fetch_url(query) + except UrlNotFoundException as e: + print(f"{e.url} could not be attributed/parsed to any yet implemented site.\n" + f"PR appreciated if the site isn't implemented.\n" + f"Recommendations and suggestions on sites to implement appreciated.\n" + f"But don't be a bitch if I don't end up implementing it.") + return self.set_current_options(PageResults(page, data_object.options)) self.print_current_options() return diff --git a/src/music_kraken/objects/source.py b/src/music_kraken/objects/source.py index a55dbd0..7cbdd44 100644 --- a/src/music_kraken/objects/source.py +++ b/src/music_kraken/objects/source.py @@ -54,7 +54,7 @@ class Source(DatabaseObject): if "musify" in parsed.netloc: return cls(SourcePages.MUSIFY, url, referer_page=referer_page) - if parsed.netloc in [url.netloc for url in ALL_YOUTUBE_URLS]: + if parsed.netloc in [_url.netloc for _url in ALL_YOUTUBE_URLS]: return cls(SourcePages.YOUTUBE, url, referer_page=referer_page) if url.startswith("https://www.deezer"): diff --git a/src/music_kraken/utils/config/__init__.py b/src/music_kraken/utils/config/__init__.py index 3d57aed..1da4dab 100644 --- a/src/music_kraken/utils/config/__init__.py +++ b/src/music_kraken/utils/config/__init__.py @@ -21,3 +21,5 @@ def write_config(): config.write_to_config_file(LOCATIONS.CONFIG_FILE) set_name_to_value = config.set_name_to_value + +read_config() From ae547c7de269b404be8d096f2f148366f3fed3f2 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 20 Jun 2023 19:30:48 +0200 Subject: [PATCH 80/81] refactor and fix --- .idea/sqldialects.xml | 6 - src/actual_donwload.py | 4 +- src/music_kraken/__main__.py | 11 +- src/music_kraken/cli/main_downloader.py | 11 +- src/music_kraken/download/page_attributes.py | 4 +- src/music_kraken/objects/metadata.py | 8 +- src/music_kraken/objects/song.py | 6 +- src/music_kraken/pages/abstract.py | 27 ++-- src/music_kraken/static_files/new_db.sql | 81 ---------- .../static_files/temp_database_structure.sql | 144 ------------------ 10 files changed, 50 insertions(+), 252 deletions(-) delete mode 100644 .idea/sqldialects.xml delete mode 100644 src/music_kraken/static_files/new_db.sql delete mode 100644 src/music_kraken/static_files/temp_database_structure.sql diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml deleted file mode 100644 index af742c7..0000000 --- a/.idea/sqldialects.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/src/actual_donwload.py b/src/actual_donwload.py index c60650b..591accb 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -19,6 +19,8 @@ if __name__ == "__main__": fetch_youtube_playlist = [ "s: https://yt.artemislena.eu/playlist?list=OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM" ] + + download_youtube_playlist = ["d: https://www.youtube.com/playlist?list=OLAK5uy_lqI_c6aDF9q4DWJ4TBzt1AFQYx_FXfU4E"] youtube_search = [ "s: #a Zombiez", @@ -26,4 +28,4 @@ if __name__ == "__main__": "d: 5" ] - music_kraken.cli.download(genre="test", command_list=fetch_youtube_playlist) + music_kraken.cli.download(genre="test", command_list=download_youtube_playlist, process_metadata_anyway=True) diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 7a249a8..8aab5a9 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -16,10 +16,16 @@ if __name__ == "__main__": help="Sets the logging level to debug." ) + parser.add_argument( + '-p', '--force-post-process', + action="store_true", + help="If a to downloaded thing is skipped due to being found on disc,\nit will still update the metadata accordingly." + ) + parser.add_argument( '-t', '--test', action="store_true", - help="For the sake of testing. Equals: '-v -g test'" + help="For the sake of testing. Equals: '-vp -g test'" ) # general arguments @@ -114,5 +120,6 @@ if __name__ == "__main__": cli.download( genre=genre, download_all=arguments.all, - direct_download_url=arguments.url + direct_download_url=arguments.url, + process_metadata_anyway=arguments.force_post_process or arguments.test ) \ No newline at end of file diff --git a/src/music_kraken/cli/main_downloader.py b/src/music_kraken/cli/main_downloader.py index c0affea..72b91f9 100644 --- a/src/music_kraken/cli/main_downloader.py +++ b/src/music_kraken/cli/main_downloader.py @@ -143,7 +143,8 @@ class Downloader: exclude_shady: bool = False, max_displayed_options: int = 10, option_digits: int = 3, - genre: str = None + genre: str = None, + process_metadata_anyway: bool = False, ) -> None: self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) @@ -156,6 +157,7 @@ class Downloader: self._result_history: List[Results] = [] self.genre = genre or get_genre() + self.process_metadata_anyway = process_metadata_anyway print() print(f"Downloading to: \"{self.genre}\"") @@ -334,7 +336,7 @@ class Downloader: _result_map: Dict[DatabaseObject, DownloadResult] = dict() for database_object in to_download: - r = self.pages.download(music_object=database_object, genre=self.genre, download_all=download_all) + r = self.pages.download(music_object=database_object, genre=self.genre, download_all=download_all, process_metadata_anyway=self.process_metadata_anyway) _result_map[database_object] = r for music_object, result in _result_map.items(): @@ -386,9 +388,10 @@ def download( genre: str = None, download_all: bool = False, direct_download_url: str = None, - command_list: List[str] = None + command_list: List[str] = None, + process_metadata_anyway: bool = False, ): - shell = Downloader(genre=genre) + shell = Downloader(genre=genre, process_metadata_anyway=process_metadata_anyway) if command_list is not None: for command in command_list: diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index ee7c18d..9b58087 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -74,7 +74,7 @@ class Pages: return music_object - def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: if not isinstance(music_object, INDEPENDENT_DB_OBJECTS): return DownloadResult(error_message=f"{type(music_object).__name__} can't be downloaded.") @@ -82,7 +82,7 @@ class Pages: audio_pages = self._audio_pages_set.intersection(_page_types) for download_page in audio_pages: - return self._page_instances[download_page].download(music_object=music_object, genre=genre, download_all=download_all) + return self._page_instances[download_page].download(music_object=music_object, genre=genre, download_all=download_all, process_metadata_anyway=process_metadata_anyway) return DownloadResult(error_message=f"No audio source has been found for {music_object}.") diff --git a/src/music_kraken/objects/metadata.py b/src/music_kraken/objects/metadata.py index 488af12..69fff8d 100644 --- a/src/music_kraken/objects/metadata.py +++ b/src/music_kraken/objects/metadata.py @@ -16,7 +16,11 @@ class Mapping(Enum): TITLE = "TIT2" ISRC = "TSRC" LENGTH = "TLEN" # in milliseconds - DATE = "TYER" + # The 'Date' frame is a numeric string in the DDMM format containing the date for the recording. This field is always four characters long. + DATE = "TDAT" + # The 'Time' frame is a numeric string in the HHMM format containing the time for the recording. This field is always four characters long. + TIME = "TIME" + YEAR = "TYER" TRACKNUMBER = "TRCK" TOTALTRACKS = "TRCK" # Stored in the same frame with TRACKNUMBER, separated by '/': e.g. '4/9'. TITLESORTORDER = "TSOT" @@ -298,7 +302,7 @@ class Metadata: if id3_dict is not None: self.add_metadata_dict(id3_dict) - def __setitem__(self, frame, value_list: list, override_existing: bool = True): + def __setitem__(self, frame: Mapping, value_list: list, override_existing: bool = True): if type(value_list) != list: raise ValueError(f"can only set attribute to list, not {type(value_list)}") diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index 6df2001..0845d11 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -295,7 +295,11 @@ class Album(MainObject): id3Mapping.COPYRIGHT: [self.copyright], id3Mapping.LANGUAGE: [self.iso_639_2_lang], id3Mapping.ALBUM_ARTIST: [a.name for a in self.artist_collection], - id3Mapping.DATE: [self.date.timestamp], + id3Mapping.DATE: [self.date.strftime("%d%m")] if self.date.has_year and self.date.has_month else [], + id3Mapping.TIME: [self.date.strftime(("%H%M"))] if self.date.has_hour and self.date.has_minute else [], + id3Mapping.YEAR: [str(self.date.year).zfill(4)] if self.date.has_year else [], + id3Mapping.RELEASE_DATE: [self.date.timestamp], + id3Mapping.ORIGINAL_RELEASE_DATE: [self.date.timestamp], id3Mapping.ALBUMSORTORDER: [str(self.albumsort)] if self.albumsort is not None else [] }) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 65ad85f..ba982ba 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -322,7 +322,7 @@ class Page: def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() - def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: naming_dict: NamingDict = NamingDict({"genre": genre}) def fill_naming_objects(naming_music_object: DatabaseObject): @@ -340,10 +340,10 @@ class Page: fill_naming_objects(music_object) - return self._download(music_object, naming_dict, download_all) + return self._download(music_object, naming_dict, download_all, process_metadata_anyway=process_metadata_anyway) - def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False, skip_details: bool = False) -> DownloadResult: + def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False, skip_details: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: skip_next_details = skip_details # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False @@ -360,7 +360,7 @@ class Page: naming_dict.add_object(music_object) if isinstance(music_object, Song): - return self._download_song(music_object, naming_dict) + return self._download_song(music_object, naming_dict, process_metadata_anyway=process_metadata_anyway) download_result: DownloadResult = DownloadResult() @@ -369,11 +369,17 @@ class Page: sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: - download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all, skip_details=skip_next_details)) + download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all, skip_details=skip_next_details, process_metadata_anyway=process_metadata_anyway)) return download_result - def _download_song(self, song: Song, naming_dict: NamingDict): + def _download_song(self, song: Song, naming_dict: NamingDict, process_metadata_anyway: bool = False): + if "genre" not in naming_dict and song.genre is not None: + naming_dict["genre"] = song.genre + + if song.genre is None: + song.genre = naming_dict["genre"] + path_parts = Formatter().parse(DOWNLOAD_PATH) file_parts = Formatter().parse(DOWNLOAD_FILE) new_target = Target( @@ -401,18 +407,21 @@ class Page: target: Target for target in song.target_collection: if target.exists: + if process_metadata_anyway: + target.copy_content(temp_target) found_on_disc = True r.found_on_disk += 1 r.add_target(target) - if found_on_disc: + if found_on_disc and not process_metadata_anyway: self.LOGGER.info(f"{song.option_string} already exists, thus not downloading again.") return r source = sources[0] - - r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) + + if not found_on_disc: + r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) if not r.is_fatal_error: diff --git a/src/music_kraken/static_files/new_db.sql b/src/music_kraken/static_files/new_db.sql deleted file mode 100644 index fd9cd0d..0000000 --- a/src/music_kraken/static_files/new_db.sql +++ /dev/null @@ -1,81 +0,0 @@ -CREATE TABLE Song -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - name TEXT, - isrc TEXT, - length INT, -- length is in milliseconds (could be wrong) - tracksort INT, - genre TEXT, - album_id BIGINT, - FOREIGN KEY(album_id) REFERENCES Album(id) -); - - -CREATE TABLE Source -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - type TEXT NOT NULL, - src TEXT NOT NULL, - url TEXT NOT NULL, - certainty INT NOT NULL DEFAULT 0, -- certainty=0 -> it is definitely a valid source - valid BOOLEAN NOT NULL DEFAULT 1, - song_id BIGINT, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - - -CREATE TABLE Album -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - title TEXT, - label TEXT, - album_status TEXT, - language TEXT, - date TEXT, - date_format TEXT, - country TEXT, - barcode TEXT, - albumsort INT, - is_split BOOLEAN NOT NULL DEFAULT 0 -); - -CREATE TABLE Target -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - file TEXT, - path TEXT, - song_id BIGINT UNIQUE, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - -CREATE TABLE Lyrics -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - text TEXT, - language TEXT, - song_id BIGINT, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - -CREATE TABLE Artist -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - name TEXT -); - -CREATE TABLE SongArtist -( - song_id BIGINT NOT NULL, - artist_id BIGINT NOT NULL, - is_feature BOOLEAN NOT NULL DEFAULT 0, - FOREIGN KEY(song_id) REFERENCES Song(id), - FOREIGN KEY(artist_id) REFERENCES Artist(id) -); - -CREATE TABLE AlbumArtist -( - album_id BIGINT, - artist_id BIGINT, - FOREIGN KEY(album_id) REFERENCES Album(id), - FOREIGN KEY(artist_id) REFERENCES Artist(id) -); diff --git a/src/music_kraken/static_files/temp_database_structure.sql b/src/music_kraken/static_files/temp_database_structure.sql deleted file mode 100644 index b75653a..0000000 --- a/src/music_kraken/static_files/temp_database_structure.sql +++ /dev/null @@ -1,144 +0,0 @@ -DROP TABLE IF EXISTS artist; -CREATE TABLE artist ( - id TEXT PRIMARY KEY NOT NULL, - mb_id TEXT, - name TEXT -); - -DROP TABLE IF EXISTS artist_release_group; -CREATE TABLE artist_release_group ( - artist_id TEXT NOT NULL, - release_group_id TEXT NOT NULL -); - -DROP TABLE IF EXISTS artist_track; -CREATE TABLE artist_track ( - artist_id TEXT NOT NULL, - track_id TEXT NOT NULL -); - -DROP TABLE IF EXISTS release_group; -CREATE TABLE release_group ( - id TEXT PRIMARY KEY NOT NULL, - albumartist TEXT, - albumsort INT, - musicbrainz_albumtype TEXT, - compilation TEXT, - album_artist_id TEXT -); - -DROP TABLE IF EXISTS release_; -CREATE TABLE release_ ( - id TEXT PRIMARY KEY NOT NULL, - release_group_id TEXT NOT NULL, - title TEXT, - copyright TEXT, - album_status TEXT, - language TEXT, - year TEXT, - date TEXT, - country TEXT, - barcode TEXT -); - -DROP TABLE IF EXISTS track; -CREATE TABLE track ( - id TEXT PRIMARY KEY NOT NULL, - downloaded BOOLEAN NOT NULL DEFAULT 0, - release_id TEXT NOT NULL, - mb_id TEXT, - track TEXT, - length INT, - tracknumber TEXT, - isrc TEXT, - genre TEXT, - lyrics TEXT, - path TEXT, - file TEXT, - url TEXT, - src TEXT -); - -DROP TABLE IF EXISTS lyrics; -CREATE TABLE lyrics ( - track_id TEXT NOT NULL, - text TEXT, - language TEXT -); - -DROP TABLE IF EXISTS target; -CREATE TABLE target ( - track_id TEXT NOT NULL, - file TEXT, - path TEXT -); - -DROP TABLE IF EXISTS source; -CREATE TABLE source ( - track_id TEXT NOT NULL, - src TEXT NOT NULL, - url TEXT NOT NULL, - certainty INT NOT NULL DEFAULT 0, -- certainty=0 -> it is definitly a valid source - valid BOOLEAN NOT NULL DEFAULT 1 -); - -DROP TABLE IF EXISTS easy_id3; -CREATE TABLE easy_id3 ( - track_id TEXT NOT NULL, - album TEXT, - bpm TEXT, - compilation TEXT, - composer TEXT, - copyright TEXT, - encodedby TEXT, - lyricist TEXT, - length TEXT, - media TEXT, - mood TEXT, - grouping TEXT, - title TEXT, - version TEXT, - artist TEXT, - albumartist TEXT, - conductor TEXT, - arranger TEXT, - discnumber TEXT, - organization TEXT, - tracknumber TEXT, - author TEXT, - albumartistsort TEXT, - albumsort TEXT, - composersort TEXT, - artistsort TEXT, - titlesort TEXT, - isrc TEXT, - discsubtitle TEXT, - language TEXT, - genre TEXT, - date TEXT, - originaldate TEXT, - performer TEXT, - musicbrainz_trackid TEXT, - website TEXT, - replaygain_gain TEXT, - replaygain_peak TEXT, - musicbrainz_artistid TEXT, - musicbrainz_albumid TEXT, - musicbrainz_albumartistid TEXT, - musicbrainz_trmid TEXT, - musicip_puid TEXT, - musicip_fingerprint TEXT, - musicbrainz_albumstatus TEXT, - musicbrainz_albumtype TEXT, - releasecountry TEXT, - musicbrainz_discid TEXT, - asin TEXT, - performer TEXT, - barcode TEXT, - catalognumber TEXT, - musicbrainz_releasetrackid TEXT, - musicbrainz_releasegroupid TEXT, - musicbrainz_workid TEXT, - acoustid_fingerprint TEXT, - acoustid_id TEXT -); From adc7716dff0794287f4709107314b56b630c336b Mon Sep 17 00:00:00 2001 From: Hellow2 Date: Wed, 21 Jun 2023 08:19:39 +0200 Subject: [PATCH 81/81] Update __main__.py --- src/music_kraken/__main__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 8aab5a9..0425d5d 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -1,4 +1,4 @@ -if __name__ == "__main__": +def cli(): import argparse parser = argparse.ArgumentParser( @@ -17,7 +17,7 @@ if __name__ == "__main__": ) parser.add_argument( - '-p', '--force-post-process', + '-m', '--force-post-process', action="store_true", help="If a to downloaded thing is skipped due to being found on disc,\nit will still update the metadata accordingly." ) @@ -122,4 +122,8 @@ if __name__ == "__main__": download_all=arguments.all, direct_download_url=arguments.url, process_metadata_anyway=arguments.force_post_process or arguments.test - ) \ No newline at end of file + ) + + +if __name__ == "__main__": + cli()