From 8255ad5264049303f95992e2753b24eed37a00fb Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 14:24:20 +0200 Subject: [PATCH 01/35] feat: added detection to autoscann pages --- music_kraken/pages/__init__.py | 62 ++++++++++++++++--- .../pages/{abstract.py => _abstract.py} | 5 +- .../pages/{bandcamp.py => _bandcamp.py} | 2 +- ...metallum.py => _encyclopaedia_metallum.py} | 3 +- music_kraken/pages/{genius.py => _genius.py} | 2 +- music_kraken/pages/{musify.py => _musify.py} | 2 +- .../pages/{youtube.py => _youtube.py} | 6 +- .../__init__.py | 0 .../_list_render.py | 1 - .../_music_object_render.py | 1 - .../super_youtube.py | 2 +- .../youtube_music.py | 2 +- music_kraken/utils/enums/__init__.py | 3 + 13 files changed, 72 insertions(+), 19 deletions(-) rename music_kraken/pages/{abstract.py => _abstract.py} (98%) rename music_kraken/pages/{bandcamp.py => _bandcamp.py} (99%) rename music_kraken/pages/{encyclopaedia_metallum.py => _encyclopaedia_metallum.py} (99%) rename music_kraken/pages/{genius.py => _genius.py} (99%) rename music_kraken/pages/{musify.py => _musify.py} (99%) rename music_kraken/pages/{youtube.py => _youtube.py} (98%) rename music_kraken/pages/{youtube_music => _youtube_music}/__init__.py (100%) rename music_kraken/pages/{youtube_music => _youtube_music}/_list_render.py (99%) rename music_kraken/pages/{youtube_music => _youtube_music}/_music_object_render.py (99%) rename music_kraken/pages/{youtube_music => _youtube_music}/super_youtube.py (99%) rename music_kraken/pages/{youtube_music => _youtube_music}/youtube_music.py (99%) diff --git a/music_kraken/pages/__init__.py b/music_kraken/pages/__init__.py index ba24501..5f82511 100644 --- a/music_kraken/pages/__init__.py +++ b/music_kraken/pages/__init__.py @@ -1,8 +1,56 @@ -from .encyclopaedia_metallum import EncyclopaediaMetallum -from .musify import Musify -from .youtube import YouTube -from .youtube_music import YoutubeMusic -from .bandcamp import Bandcamp -from .genius import Genius +from typing import Type, Generator, Set, Dict, List +from collections import defaultdict -from .abstract import Page, INDEPENDENT_DB_OBJECTS +from ._encyclopaedia_metallum import EncyclopaediaMetallum +from ._musify import Musify +from ._youtube import YouTube +from ._youtube_music import YoutubeMusic +from ._bandcamp import Bandcamp +from ._genius import Genius +from ._abstract import Page, INDEPENDENT_DB_OBJECTS + + +_registered_pages: Dict[Type[Page], Set[Page]] = defaultdict(set) + + +def get_pages(*page_types: List[Type[Page]]) -> Generator[Page, None, None]: + if len(page_types) == 0: + page_types = _registered_pages.keys() + + for page_type in page_types: + yield from _registered_pages[page_type] + + +def register_page(page_type: Type[Page], **kwargs): + if page_type in _registered_pages: + return + + _registered_pages[page_type].add(page_type(**kwargs)) + + +def deregister_page(page_type: Type[Page]): + if page_type not in _registered_pages: + return + + for p in _registered_pages[page_type]: + p.__del__() + del _registered_pages[page_type] + +def scan_for_pages(): + # assuming the wanted pages are the leaf classes of the interface + leaf_classes = [] + + _class_list = [Page] + while len(_class_list): + _class = _class_list.pop() + _class_subclasses = _class.__subclasses__() + + if len(_class_subclasses) == 0: + if _class.REGISTER: + leaf_classes.append(_class) + else: + _class_list.extend(_class_subclasses) + + print(leaf_classes) + for leaf_class in leaf_classes: + register_page(leaf_class) diff --git a/music_kraken/pages/abstract.py b/music_kraken/pages/_abstract.py similarity index 98% rename from music_kraken/pages/abstract.py rename to music_kraken/pages/_abstract.py index 8783dbb..e6e35f7 100644 --- a/music_kraken/pages/abstract.py +++ b/music_kraken/pages/_abstract.py @@ -48,12 +48,12 @@ class DownloadOptions: process_metadata_if_found: bool = True class Page: + REGISTER = True SOURCE_TYPE: SourceType LOGGER: logging.Logger def __new__(cls, *args, **kwargs): cls.LOGGER = logging.getLogger(cls.__name__) - return super().__new__(cls) def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None): @@ -62,6 +62,9 @@ class Page: self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() + def __del__(self): + self.SOURCE_TYPE.deregister_page(self) + def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): """ Perform a regex search on the given string, using a single or a list of diff --git a/music_kraken/pages/bandcamp.py b/music_kraken/pages/_bandcamp.py similarity index 99% rename from music_kraken/pages/bandcamp.py rename to music_kraken/pages/_bandcamp.py index 1caf803..658b448 100644 --- a/music_kraken/pages/bandcamp.py +++ b/music_kraken/pages/_bandcamp.py @@ -6,7 +6,7 @@ from bs4 import BeautifulSoup import pycountry from ..objects import Source, DatabaseObject -from .abstract import Page +from ._abstract import Page from ..objects import ( Artist, Source, diff --git a/music_kraken/pages/encyclopaedia_metallum.py b/music_kraken/pages/_encyclopaedia_metallum.py similarity index 99% rename from music_kraken/pages/encyclopaedia_metallum.py rename to music_kraken/pages/_encyclopaedia_metallum.py index 52d9ea3..cd6ec4c 100644 --- a/music_kraken/pages/encyclopaedia_metallum.py +++ b/music_kraken/pages/_encyclopaedia_metallum.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse, urlencode from ..connection import Connection from ..utils.config import logging_settings -from .abstract import Page +from ._abstract import Page from ..utils.enums import SourceType, ALL_SOURCE_TYPES from ..utils.enums.album import AlbumType from ..utils.support_classes.query import Query @@ -207,6 +207,7 @@ def create_grid( class EncyclopaediaMetallum(Page): + REGISTER = False SOURCE_TYPE = ALL_SOURCE_TYPES.ENCYCLOPAEDIA_METALLUM LOGGER = logging_settings["metal_archives_logger"] diff --git a/music_kraken/pages/genius.py b/music_kraken/pages/_genius.py similarity index 99% rename from music_kraken/pages/genius.py rename to music_kraken/pages/_genius.py index 1719025..08d38f0 100644 --- a/music_kraken/pages/genius.py +++ b/music_kraken/pages/_genius.py @@ -6,7 +6,7 @@ from bs4 import BeautifulSoup import pycountry from ..objects import Source, DatabaseObject -from .abstract import Page +from ._abstract import Page from ..objects import ( Artist, Source, diff --git a/music_kraken/pages/musify.py b/music_kraken/pages/_musify.py similarity index 99% rename from music_kraken/pages/musify.py rename to music_kraken/pages/_musify.py index e8078fb..0d00d26 100644 --- a/music_kraken/pages/musify.py +++ b/music_kraken/pages/_musify.py @@ -8,7 +8,7 @@ import pycountry from bs4 import BeautifulSoup from ..connection import Connection -from .abstract import Page +from ._abstract import Page from ..utils.enums import SourceType, ALL_SOURCE_TYPES from ..utils.enums.album import AlbumType, AlbumStatus from ..objects import ( diff --git a/music_kraken/pages/youtube.py b/music_kraken/pages/_youtube.py similarity index 98% rename from music_kraken/pages/youtube.py rename to music_kraken/pages/_youtube.py index 8f21c73..2530aa1 100644 --- a/music_kraken/pages/youtube.py +++ b/music_kraken/pages/_youtube.py @@ -5,7 +5,7 @@ from enum import Enum import python_sponsorblock from ..objects import Source, DatabaseObject, Song, Target -from .abstract import Page +from ._abstract import Page from ..objects import ( Artist, Source, @@ -22,7 +22,7 @@ from ..utils.enums import SourceType, ALL_SOURCE_TYPES from ..utils.support_classes.download_result import DownloadResult from ..utils.config import youtube_settings, main_settings, logging_settings -from .youtube_music.super_youtube import SuperYouTube, YouTubeUrl, get_invidious_url, YouTubeUrlType +from ._youtube_music.super_youtube import SuperYouTube, YouTubeUrl, get_invidious_url, YouTubeUrlType """ @@ -38,7 +38,7 @@ def get_piped_url(path: str = "", params: str = "", query: str = "", fragment: s class YouTube(SuperYouTube): - # CHANGE + REGISTER = youtube_settings["use_youtube_alongside_youtube_music"] SOURCE_TYPE = ALL_SOURCE_TYPES.YOUTUBE def __init__(self, *args, **kwargs): diff --git a/music_kraken/pages/youtube_music/__init__.py b/music_kraken/pages/_youtube_music/__init__.py similarity index 100% rename from music_kraken/pages/youtube_music/__init__.py rename to music_kraken/pages/_youtube_music/__init__.py diff --git a/music_kraken/pages/youtube_music/_list_render.py b/music_kraken/pages/_youtube_music/_list_render.py similarity index 99% rename from music_kraken/pages/youtube_music/_list_render.py rename to music_kraken/pages/_youtube_music/_list_render.py index 7dcc8cf..29474cd 100644 --- a/music_kraken/pages/youtube_music/_list_render.py +++ b/music_kraken/pages/_youtube_music/_list_render.py @@ -3,7 +3,6 @@ from enum import Enum from ...utils.config import logging_settings from ...objects import Source, DatabaseObject -from ..abstract import Page from ...objects import ( Artist, Source, diff --git a/music_kraken/pages/youtube_music/_music_object_render.py b/music_kraken/pages/_youtube_music/_music_object_render.py similarity index 99% rename from music_kraken/pages/youtube_music/_music_object_render.py rename to music_kraken/pages/_youtube_music/_music_object_render.py index 43aee3e..474f395 100644 --- a/music_kraken/pages/youtube_music/_music_object_render.py +++ b/music_kraken/pages/_youtube_music/_music_object_render.py @@ -6,7 +6,6 @@ from ...utils.string_processing import clean_song_title from ...utils.enums import SourceType, ALL_SOURCE_TYPES from ...objects import Source, DatabaseObject -from ..abstract import Page from ...objects import ( Artist, Source, diff --git a/music_kraken/pages/youtube_music/super_youtube.py b/music_kraken/pages/_youtube_music/super_youtube.py similarity index 99% rename from music_kraken/pages/youtube_music/super_youtube.py rename to music_kraken/pages/_youtube_music/super_youtube.py index df900a1..fa5ce1c 100644 --- a/music_kraken/pages/youtube_music/super_youtube.py +++ b/music_kraken/pages/_youtube_music/super_youtube.py @@ -6,7 +6,7 @@ import requests import python_sponsorblock from ...objects import Source, DatabaseObject, Song, Target -from ..abstract import Page +from .._abstract import Page from ...objects import ( Artist, Source, diff --git a/music_kraken/pages/youtube_music/youtube_music.py b/music_kraken/pages/_youtube_music/youtube_music.py similarity index 99% rename from music_kraken/pages/youtube_music/youtube_music.py rename to music_kraken/pages/_youtube_music/youtube_music.py index 08e2207..b2d4aa2 100644 --- a/music_kraken/pages/youtube_music/youtube_music.py +++ b/music_kraken/pages/_youtube_music/youtube_music.py @@ -22,7 +22,7 @@ from ...utils import get_current_millis, traverse_json_path from ...utils import dump_to_file -from ..abstract import Page +from .._abstract import Page from ...objects import ( DatabaseObject as DataObject, Source, diff --git a/music_kraken/utils/enums/__init__.py b/music_kraken/utils/enums/__init__.py index 28f0b9f..38fc03f 100644 --- a/music_kraken/utils/enums/__init__.py +++ b/music_kraken/utils/enums/__init__.py @@ -17,6 +17,9 @@ class SourceType: def register_page(self, page: Page): self.page = page + def deregister_page(self): + self.page = None + def __hash__(self): return hash(self.name) -- 2.45.2 From 40e9366a0bd8dfcabb188c19c247af0cf4a66d7e Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 14:32:31 +0200 Subject: [PATCH 02/35] feat: implemented the new page mechanics in the downloader --- music_kraken/download/page_attributes.py | 80 +++--------------------- 1 file changed, 9 insertions(+), 71 deletions(-) diff --git a/music_kraken/download/page_attributes.py b/music_kraken/download/page_attributes.py index 1db24be..8be35b8 100644 --- a/music_kraken/download/page_attributes.py +++ b/music_kraken/download/page_attributes.py @@ -30,31 +30,9 @@ from ..utils.exception import MKMissingNameException from ..utils.exception.download import UrlNotFoundException from ..utils.shared import DEBUG_PAGES -from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, Bandcamp, Genius, INDEPENDENT_DB_OBJECTS +from ..pages import scan_for_pages, get_pages -ALL_PAGES: Set[Type[Page]] = { - # EncyclopaediaMetallum, - Genius, - Musify, - YoutubeMusic, - Bandcamp -} - -if youtube_settings["use_youtube_alongside_youtube_music"]: - ALL_PAGES.add(YouTube) - -AUDIO_PAGES: Set[Type[Page]] = { - Musify, - YouTube, - YoutubeMusic, - Bandcamp -} - -SHADY_PAGES: Set[Type[Page]] = { - Musify, -} - fetch_map = { Song: "fetch_song", Album: "fetch_album", @@ -62,66 +40,28 @@ fetch_map = { Label: "fetch_label", } -if DEBUG_PAGES: - DEBUGGING_PAGE = Bandcamp - print(f"Only downloading from page {DEBUGGING_PAGE}.") - - ALL_PAGES = {DEBUGGING_PAGE} - AUDIO_PAGES = ALL_PAGES.union(AUDIO_PAGES) - class Pages: - def __init__(self, exclude_pages: Set[Type[Page]] = None, exclude_shady: bool = False, download_options: DownloadOptions = None, fetch_options: FetchOptions = None): + def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): self.LOGGER = logging.getLogger("download") self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() - # initialize all page instances - self._page_instances: Dict[Type[Page], Page] = dict() - self._source_to_page: Dict[SourceType, Type[Page]] = dict() - - exclude_pages = exclude_pages if exclude_pages is not None else set() - - if exclude_shady: - exclude_pages = exclude_pages.union(SHADY_PAGES) - - if not exclude_pages.issubset(ALL_PAGES): - raise ValueError(f"The excluded pages have to be a subset of all pages: {exclude_pages} | {ALL_PAGES}") - - def _set_to_tuple(page_set: Set[Type[Page]]) -> Tuple[Type[Page], ...]: - return tuple(sorted(page_set, key=lambda page: page.__name__)) - - self._pages_set: Set[Type[Page]] = ALL_PAGES.difference(exclude_pages) - self.pages: Tuple[Type[Page], ...] = _set_to_tuple(self._pages_set) - - self._audio_pages_set: Set[Type[Page]] = self._pages_set.intersection(AUDIO_PAGES) - self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(self._audio_pages_set) - - for page_type in self.pages: - self._page_instances[page_type] = page_type(fetch_options=self.fetch_options, download_options=self.download_options) - self._source_to_page[page_type.SOURCE_TYPE] = page_type - - def _get_page_from_enum(self, source_page: SourceType) -> Page: - if source_page not in self._source_to_page: - return None - return self._page_instances[self._source_to_page[source_page]] + scan_for_pages() def search(self, query: Query) -> SearchResults: result = SearchResults() - for page_type in self.pages: + for page in get_pages(): result.add( - page=page_type, - search_result=self._page_instances[page_type].search(query=query) + page=type(page), + search_result=page.search(query=query) ) return result def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: - if not isinstance(data_object, INDEPENDENT_DB_OBJECTS): - return data_object - source: Source for source in data_object.source_collection.get_sources(source_type_sorting={ "only_with_page": True, @@ -317,12 +257,10 @@ class Pages: tmp.delete() return r - def fetch_url(self, url: str, stop_at_level: int = 2) -> Tuple[Type[Page], DataObject]: + def fetch_url(self, url: str, **kwargs) -> DataObject: source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) - if source is None: + if source is None or source.page is None: raise UrlNotFoundException(url=url) - _actual_page = self._source_to_page[source.source_type] - - return _actual_page, self._page_instances[_actual_page].fetch_object_from_source(source=source, stop_at_level=stop_at_level) \ No newline at end of file + return source.page.fetch_object_from_source(source=source, **kwargs) \ No newline at end of file -- 2.45.2 From aafbba3b1cc5a38de1a3b16eab8dcb05393faafe Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 14:36:19 +0200 Subject: [PATCH 03/35] feat: implemented consistent settings --- music_kraken/download/page_attributes.py | 2 +- music_kraken/pages/__init__.py | 5 ++--- music_kraken/pages/_abstract.py | 22 +++++++--------------- 3 files changed, 10 insertions(+), 19 deletions(-) diff --git a/music_kraken/download/page_attributes.py b/music_kraken/download/page_attributes.py index 8be35b8..c897cfe 100644 --- a/music_kraken/download/page_attributes.py +++ b/music_kraken/download/page_attributes.py @@ -48,7 +48,7 @@ class Pages: self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() - scan_for_pages() + scan_for_pages(download_options=self.download_options, fetch_options=self.fetch_options, **kwargs) def search(self, query: Query) -> SearchResults: result = SearchResults() diff --git a/music_kraken/pages/__init__.py b/music_kraken/pages/__init__.py index 5f82511..1e015a6 100644 --- a/music_kraken/pages/__init__.py +++ b/music_kraken/pages/__init__.py @@ -36,7 +36,7 @@ def deregister_page(page_type: Type[Page]): p.__del__() del _registered_pages[page_type] -def scan_for_pages(): +def scan_for_pages(**kwargs): # assuming the wanted pages are the leaf classes of the interface leaf_classes = [] @@ -51,6 +51,5 @@ def scan_for_pages(): else: _class_list.extend(_class_subclasses) - print(leaf_classes) for leaf_class in leaf_classes: - register_page(leaf_class) + register_page(leaf_class, **kwargs) diff --git a/music_kraken/pages/_abstract.py b/music_kraken/pages/_abstract.py index e6e35f7..2b76c59 100644 --- a/music_kraken/pages/_abstract.py +++ b/music_kraken/pages/_abstract.py @@ -1,15 +1,19 @@ +from __future__ import annotations + import logging import random import re from copy import copy from pathlib import Path -from typing import Optional, Union, Type, Dict, Set, List, Tuple, TypedDict +from typing import Optional, Union, Type, Dict, Set, List, Tuple, TypedDict, TYPE_CHECKING from string import Formatter from dataclasses import dataclass, field import requests from bs4 import BeautifulSoup +if TYPE_CHECKING: + from ..download.page_attributes import DownloadOptions, FetchOptions from ..connection import Connection from ..objects import ( Song, @@ -34,18 +38,6 @@ from ..utils import trace, output, BColors INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] -@dataclass -class FetchOptions: - download_all: bool = False - album_type_blacklist: Set[AlbumType] = field(default_factory=lambda: set(AlbumType(a) for a in main_settings["album_type_blacklist"])) - -@dataclass -class DownloadOptions: - download_all: bool = False - album_type_blacklist: Set[AlbumType] = field(default_factory=lambda: set(AlbumType(a) for a in main_settings["album_type_blacklist"])) - - process_audio_if_found: bool = False - process_metadata_if_found: bool = True class Page: REGISTER = True @@ -56,14 +48,14 @@ class Page: cls.LOGGER = logging.getLogger(cls.__name__) return super().__new__(cls) - def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None): + def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): self.SOURCE_TYPE.register_page(self) self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() def __del__(self): - self.SOURCE_TYPE.deregister_page(self) + self.SOURCE_TYPE.deregister_page() def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): """ -- 2.45.2 From c683394228633ac9c152b16dc33bac946c48c643 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 16:13:47 +0200 Subject: [PATCH 04/35] feat: imports --- music_kraken/download/page_attributes.py | 45 +++++++++--------------- 1 file changed, 17 insertions(+), 28 deletions(-) diff --git a/music_kraken/download/page_attributes.py b/music_kraken/download/page_attributes.py index c897cfe..0622bf1 100644 --- a/music_kraken/download/page_attributes.py +++ b/music_kraken/download/page_attributes.py @@ -1,37 +1,26 @@ -from typing import Tuple, Type, Dict, Set, Optional, List +import logging +import re from collections import defaultdict from pathlib import Path -import re -import logging +from typing import Dict, List, Optional, Set, Tuple, Type -from . import FetchOptions, DownloadOptions -from .results import SearchResults -from ..objects import ( - DatabaseObject as DataObject, - Collection, - Target, - Source, - Options, - Song, - Album, - Artist, - Label, -) -from ..audio import write_metadata_to_target, correct_codec -from ..utils import output, BColors -from ..utils.string_processing import fit_to_file_system -from ..utils.config import youtube_settings, main_settings -from ..utils.path_manager import LOCATIONS -from ..utils.enums import SourceType, ALL_SOURCE_TYPES -from ..utils.support_classes.download_result import DownloadResult -from ..utils.support_classes.query import Query -from ..utils.support_classes.download_result import DownloadResult +from ..audio import correct_codec, write_metadata_to_target +from ..objects import Album, Artist, Collection +from ..objects import DatabaseObject as DataObject +from ..objects import Label, Options, Song, Source, Target +from ..pages import get_pages, scan_for_pages +from ..utils import BColors, output +from ..utils.config import main_settings, youtube_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.exception import MKMissingNameException from ..utils.exception.download import UrlNotFoundException +from ..utils.path_manager import LOCATIONS from ..utils.shared import DEBUG_PAGES - -from ..pages import scan_for_pages, get_pages - +from ..utils.string_processing import fit_to_file_system +from ..utils.support_classes.download_result import DownloadResult +from ..utils.support_classes.query import Query +from . import DownloadOptions, FetchOptions +from .results import SearchResults fetch_map = { Song: "fetch_song", -- 2.45.2 From cd2e7d7173c9e297b1a06d8bebe28d22e7d89290 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 16:33:40 +0200 Subject: [PATCH 05/35] draft: moving page interface to downloader module --- music_kraken/cli/main_downloader.py | 103 ++----- music_kraken/download/__init__.py | 432 +++++++++++++++++++++++++++- 2 files changed, 448 insertions(+), 87 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index c5ba8a9..2356f8e 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -1,89 +1,24 @@ import random -from typing import Set, Type, Dict, List -from pathlib import Path import re +from pathlib import Path +from typing import Dict, List, Set, Type -from .utils import cli_function -from .options.first_config import initial_config - -from ..utils import output, BColors -from ..utils.config import write_config, main_settings -from ..utils.shared import URL_PATTERN -from ..utils.string_processing import fit_to_file_system -from ..utils.support_classes.query import Query -from ..utils.support_classes.download_result import DownloadResult +from .. import console +from ..download import Downloader +from ..download.results import GoToResults, Option, PageResults, Results +from ..objects import Album, Artist, DatabaseObject, Song +from ..pages import Page +from ..utils import BColors, output +from ..utils.config import main_settings, write_config +from ..utils.enums.colors import BColors from ..utils.exception import MKInvalidInputException from ..utils.exception.download import UrlNotFoundException -from ..utils.enums.colors import BColors -from .. import console - -from ..download.results import Results, Option, PageResults, GoToResults -from ..download.page_attributes import Pages -from ..pages import Page -from ..objects import Song, Album, Artist, DatabaseObject - -""" -This is the implementation of the Shell - -# Behaviour - -## Searching - -```mkshell -> s: {querry or url} - -# examples -> s: https://musify.club/release/some-random-release-183028492 -> s: r: #a an Artist #r some random Release -``` - -Searches for an url, or an query - -### Query Syntax - -``` -#a {artist} #r {release} #t {track} -``` - -You can escape stuff like `#` doing this: `\#` - -## Downloading - -To download something, you either need a direct link, or you need to have already searched for options - -```mkshell -> d: {option ids or direct url} - -# examples -> d: 0, 3, 4 -> d: 1 -> d: https://musify.club/release/some-random-release-183028492 -``` - -## Misc - -### Exit - -```mkshell -> q -> quit -> exit -> abort -``` - -### Current Options - -```mkshell -> . -``` - -### Previous Options - -``` -> .. -``` - -""" +from ..utils.shared import URL_PATTERN +from ..utils.string_processing import fit_to_file_system +from ..utils.support_classes.download_result import DownloadResult +from ..utils.support_classes.query import Query +from .options.first_config import initial_config +from .utils import cli_function EXIT_COMMANDS = {"q", "quit", "exit", "abort"} ALPHABET = "abcdefghijklmnopqrstuvwxyz" @@ -143,7 +78,7 @@ def help_message(): print() -class Downloader: +class CliDownloader: def __init__( self, exclude_pages: Set[Type[Page]] = None, @@ -153,7 +88,7 @@ class Downloader: genre: str = None, process_metadata_anyway: bool = False, ) -> None: - self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) + self.pages: Downloader = Downloader(exclude_pages=exclude_pages, exclude_shady=exclude_shady) self.page_dict: Dict[str, Type[Page]] = dict() @@ -446,7 +381,7 @@ def download( else: print(f"{BColors.FAIL.value}Something went wrong configuring.{BColors.ENDC.value}") - shell = Downloader(genre=genre, process_metadata_anyway=process_metadata_anyway) + shell = CliDownloader(genre=genre, process_metadata_anyway=process_metadata_anyway) if command_list is not None: for command in command_list: diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 7ca0086..e9f0fd0 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -1,8 +1,36 @@ -from dataclasses import dataclass, field -from typing import Set +from __future__ import annotations -from ..utils.config import main_settings +import logging +import random +import re +from collections import defaultdict +from copy import copy +from dataclasses import dataclass, field +from pathlib import Path +from string import Formatter +from typing import (TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type, + TypedDict, Union) + +import requests +from bs4 import BeautifulSoup + +from ..audio import correct_codec, write_metadata_to_target +from ..connection import Connection +from ..objects import Album, Artist, Collection +from ..objects import DatabaseObject as DataObject +from ..objects import Label, Options, Song, Source, Target +from ..utils import BColors, output, trace +from ..utils.config import main_settings, youtube_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.enums.album import AlbumType +from ..utils.exception import MKMissingNameException +from ..utils.exception.download import UrlNotFoundException +from ..utils.path_manager import LOCATIONS +from ..utils.shared import DEBUG_PAGES +from ..utils.string_processing import fit_to_file_system +from ..utils.support_classes.download_result import DownloadResult +from ..utils.support_classes.query import Query +from .results import SearchResults @dataclass @@ -19,3 +47,401 @@ class DownloadOptions: download_again_if_found: bool = False process_audio_if_found: bool = False process_metadata_if_found: bool = True + + +fetch_map = { + Song: "fetch_song", + Album: "fetch_album", + Artist: "fetch_artist", + Label: "fetch_label", +} + + +class Downloader: + def __init__( + self, + auto_register_pages: bool = True, + download_options: DownloadOptions = None, + fetch_options: FetchOptions = None, + **kwargs + ): + self.LOGGER = logging.getLogger("download") + + self.download_options: DownloadOptions = download_options or DownloadOptions() + self.fetch_options: FetchOptions = fetch_options or FetchOptions() + + self._registered_pages: Dict[Type[Page], Set[Page]] = defaultdict(set) + if auto_register_pages: + self.scan_for_pages(**kwargs) + + def register_page(self, page_type: Type[Page], **kwargs): + if page_type in _registered_pages: + return + + self._registered_pages[page_type].add(page_type( + download_options=self.download_options, + fetch_options=self.fetch_options, + **kwargs + )) + + def deregister_page(self, page_type: Type[Page]): + if page_type not in _registered_pages: + return + + for p in self._registered_pages[page_type]: + p.__del__() + del self._registered_pages[page_type] + + def scan_for_pages(self, **kwargs): + # assuming the wanted pages are the leaf classes of the interface + leaf_classes = [] + + class_list = [Page] + while len(class_list): + _class = class_list.pop() + class_subclasses = _class.__subclasses__() + + if len(class_subclasses) == 0: + if _class.REGISTER: + leaf_classes.append(_class) + else: + class_list.extend(class_subclasses) + + for leaf_class in leaf_classes: + self.register_page(leaf_class, **kwargs) + + def get_pages(self, *page_types: List[Type[Page]]) -> Generator[Page, None, None]: + if len(page_types) == 0: + page_types = _registered_pages.keys() + + for page_type in page_types: + yield from self._registered_pages[page_type] + + def search(self, query: Query) -> SearchResults: + result = SearchResults() + + for page in self.get_pages(): + result.add( + page=type(page), + search_result=page.search(query=query) + ) + + return result + + def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: + source: Source + for source in data_object.source_collection.get_sources(source_type_sorting={ + "only_with_page": True, + }): + new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level) + if new_data_object is not None: + data_object.merge(new_data_object) + + return data_object + + def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]: + if not source.has_page: + return None + + source_type = source.page.get_source_type(source=source) + if source_type is None: + self.LOGGER.debug(f"Could not determine source type for {source}.") + return None + + func = getattr(source.page, fetch_map[source_type]) + + # fetching the data object and marking it as fetched + data_object: DataObject = func(source=source, **kwargs) + data_object.mark_as_fetched(source.hash_url) + return data_object + + def fetch_from_url(self, url: str) -> Optional[DataObject]: + source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) + if source is None: + return None + + return self.fetch_from_source(source=source) + + def _skip_object(self, data_object: DataObject) -> bool: + if isinstance(data_object, Album): + if not self.download_options.download_all and data_object.album_type in self.download_options.album_type_blacklist: + return True + + return False + + def download(self, data_object: DataObject, genre: str, **kwargs) -> DownloadResult: + # fetch the given object + self.fetch_details(data_object) + output(f"\nDownloading {data_object.option_string}...", color=BColors.BOLD) + + # fetching all parent objects (e.g. if you only download a song) + if not kwargs.get("fetched_upwards", False): + to_fetch: List[DataObject] = [data_object] + + while len(to_fetch) > 0: + new_to_fetch = [] + for d in to_fetch: + if self._skip_object(d): + continue + + self.fetch_details(d) + + for c in d.get_parent_collections(): + new_to_fetch.extend(c) + + to_fetch = new_to_fetch + + kwargs["fetched_upwards"] = True + + # download all children + download_result: DownloadResult = DownloadResult() + for c in data_object.get_child_collections(): + for d in c: + if self._skip_object(d): + continue + + download_result.merge(self.download(d, genre, **kwargs)) + + # actually download if the object is a song + if isinstance(data_object, Song): + """ + TODO + add the traced artist and album to the naming. + I am able to do that, because duplicate values are removed later on. + """ + + self._download_song(data_object, naming={ + "genre": [genre], + "audio_format": [main_settings["audio_format"]], + }) + + return download_result + + def _extract_fields_from_template(self, path_template: str) -> Set[str]: + return set(re.findall(r"{([^}]+)}", path_template)) + + def _parse_path_template(self, path_template: str, naming: Dict[str, List[str]]) -> str: + field_names: Set[str] = self._extract_fields_from_template(path_template) + + for field in field_names: + if len(naming[field]) == 0: + raise MKMissingNameException(f"Missing field for {field}.") + + path_template = path_template.replace(f"{{{field}}}", naming[field][0]) + + return path_template + + def _download_song(self, song: Song, naming: dict) -> DownloadOptions: + """ + TODO + Search the song in the file system. + """ + r = DownloadResult(total=1) + + # pre process the data recursively + song.compile() + + # manage the naming + naming: Dict[str, List[str]] = defaultdict(list, naming) + naming["song"].append(song.title_value) + naming["isrc"].append(song.isrc) + naming["album"].extend(a.title_value for a in song.album_collection) + naming["album_type"].extend(a.album_type.value for a in song.album_collection) + naming["artist"].extend(a.name for a in song.artist_collection) + naming["artist"].extend(a.name for a in song.feature_artist_collection) + for a in song.album_collection: + naming["label"].extend([l.title_value for l in a.label_collection]) + # removing duplicates from the naming, and process the strings + for key, value in naming.items(): + # https://stackoverflow.com/a/17016257 + naming[key] = list(dict.fromkeys(value)) + song.genre = naming["genre"][0] + + # manage the targets + tmp: Target = Target.temp(file_extension=main_settings["audio_format"]) + + song.target_collection.append(Target( + relative_to_music_dir=True, + file_path=Path( + self._parse_path_template(main_settings["download_path"], naming=naming), + self._parse_path_template(main_settings["download_file"], naming=naming), + ) + )) + for target in song.target_collection: + if target.exists: + output(f'{target.file_path} {BColors.OKGREEN.value}[already exists]', color=BColors.GREY) + r.found_on_disk += 1 + + if not self.download_options.download_again_if_found: + target.copy_content(tmp) + else: + target.create_path() + output(f'{target.file_path}', color=BColors.GREY) + + # this streams from every available source until something succeeds, setting the skip intervals to the values of the according source + used_source: Optional[Source] = None + skip_intervals: List[Tuple[float, float]] = [] + for source in song.source_collection.get_sources(source_type_sorting={ + "only_with_page": True, + "sort_key": lambda page: page.download_priority, + "reverse": True, + }): + if tmp.exists: + break + + used_source = source + streaming_results = source.page.download_song_to_target(source=source, target=tmp, desc="download") + skip_intervals = source.page.get_skip_intervals(song=song, source=source) + + # if something has been downloaded but it somehow failed, delete the file + if streaming_results.is_fatal_error and tmp.exists: + tmp.delete() + + # if everything went right, the file should exist now + if not tmp.exists: + if used_source is None: + r.error_message = f"No source found for {song.option_string}." + else: + r.error_message = f"Something went wrong downloading {song.option_string}." + return r + + # post process the audio + found_on_disk = used_source is None + if not found_on_disk or self.download_options.process_audio_if_found: + correct_codec(target=tmp, skip_intervals=skip_intervals) + r.sponsor_segments = len(skip_intervals) + + if used_source is not None: + used_source.page.post_process_hook(song=song, temp_target=tmp) + + if not found_on_disk or self.download_options.process_metadata_if_found: + write_metadata_to_target(metadata=song.metadata, target=tmp, song=song) + + # copy the tmp target to the final locations + for target in song.target_collection: + tmp.copy_content(target) + + tmp.delete() + return r + + def fetch_url(self, url: str, **kwargs) -> DataObject: + source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) + + if source is None or source.page is None: + raise UrlNotFoundException(url=url) + + return source.page.fetch_object_from_source(source=source, **kwargs) + + +class Page: + REGISTER = True + SOURCE_TYPE: SourceType + LOGGER: logging.Logger + + def __new__(cls, *args, **kwargs): + cls.LOGGER = logging.getLogger(cls.__name__) + return super().__new__(cls) + + def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): + self.SOURCE_TYPE.register_page(self) + + self.download_options: DownloadOptions = download_options or DownloadOptions() + self.fetch_options: FetchOptions = fetch_options or FetchOptions() + + def __del__(self): + self.SOURCE_TYPE.deregister_page() + + def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): + """ + Perform a regex search on the given string, using a single or a list of + patterns returning the first matching group. + In case of failure return a default value or raise a WARNING or a + RegexNotFoundError, depending on fatal, specifying the field name. + """ + + if isinstance(pattern, str): + mobj = re.search(pattern, string, flags) + else: + for p in pattern: + mobj = re.search(p, string, flags) + if mobj: + break + + if mobj: + if group is None: + # return the first matching group + return next(g for g in mobj.groups() if g is not None) + elif isinstance(group, (list, tuple)): + return tuple(mobj.group(g) for g in group) + else: + return mobj.group(group) + + return default + + def get_source_type(self, source: Source) -> Optional[Type[DataObject]]: + return None + + def get_soup_from_response(self, r: requests.Response) -> BeautifulSoup: + return BeautifulSoup(r.content, "html.parser") + + # to search stuff + def search(self, query: Query) -> List[DataObject]: + music_object = query.music_object + + search_functions = { + Song: self.song_search, + Album: self.album_search, + Artist: self.artist_search, + Label: self.label_search + } + + if type(music_object) in search_functions: + r = search_functions[type(music_object)](music_object) + if r is not None and len(r) > 0: + return r + + r = [] + for default_query in query.default_search: + for single_option in self.general_search(default_query): + r.append(single_option) + + return r + + def general_search(self, search_query: str) -> List[DataObject]: + return [] + + def label_search(self, label: Label) -> List[Label]: + return [] + + def artist_search(self, artist: Artist) -> List[Artist]: + return [] + + def album_search(self, album: Album) -> List[Album]: + return [] + + def song_search(self, song: Song) -> List[Song]: + return [] + + # to fetch stuff + def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: + return Song() + + def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: + return Album() + + def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: + return Artist() + + def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: + return Label() + + # to download stuff + def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]: + return [] + + def post_process_hook(self, song: Song, temp_target: Target, **kwargs): + pass + + def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: + return DownloadResult() + -- 2.45.2 From 906ddb679d51e4dcc82c9163abde2afef8e9099b Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 17:27:24 +0200 Subject: [PATCH 06/35] draft: --- music_kraken/download/page_attributes.py | 255 ------------------ music_kraken/download/results.py | 4 +- music_kraken/pages/__init__.py | 52 +--- music_kraken/pages/_abstract.py | 152 ----------- music_kraken/pages/_bandcamp.py | 37 +-- music_kraken/pages/_encyclopaedia_metallum.py | 35 +-- music_kraken/pages/_genius.py | 37 +-- music_kraken/pages/_musify.py | 27 +- music_kraken/pages/_youtube.py | 28 +- .../pages/_youtube_music/youtube_music.py | 45 ++-- 10 files changed, 76 insertions(+), 596 deletions(-) delete mode 100644 music_kraken/download/page_attributes.py delete mode 100644 music_kraken/pages/_abstract.py diff --git a/music_kraken/download/page_attributes.py b/music_kraken/download/page_attributes.py deleted file mode 100644 index 0622bf1..0000000 --- a/music_kraken/download/page_attributes.py +++ /dev/null @@ -1,255 +0,0 @@ -import logging -import re -from collections import defaultdict -from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple, Type - -from ..audio import correct_codec, write_metadata_to_target -from ..objects import Album, Artist, Collection -from ..objects import DatabaseObject as DataObject -from ..objects import Label, Options, Song, Source, Target -from ..pages import get_pages, scan_for_pages -from ..utils import BColors, output -from ..utils.config import main_settings, youtube_settings -from ..utils.enums import ALL_SOURCE_TYPES, SourceType -from ..utils.exception import MKMissingNameException -from ..utils.exception.download import UrlNotFoundException -from ..utils.path_manager import LOCATIONS -from ..utils.shared import DEBUG_PAGES -from ..utils.string_processing import fit_to_file_system -from ..utils.support_classes.download_result import DownloadResult -from ..utils.support_classes.query import Query -from . import DownloadOptions, FetchOptions -from .results import SearchResults - -fetch_map = { - Song: "fetch_song", - Album: "fetch_album", - Artist: "fetch_artist", - Label: "fetch_label", -} - - -class Pages: - def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): - self.LOGGER = logging.getLogger("download") - - self.download_options: DownloadOptions = download_options or DownloadOptions() - self.fetch_options: FetchOptions = fetch_options or FetchOptions() - - scan_for_pages(download_options=self.download_options, fetch_options=self.fetch_options, **kwargs) - - def search(self, query: Query) -> SearchResults: - result = SearchResults() - - for page in get_pages(): - result.add( - page=type(page), - search_result=page.search(query=query) - ) - - return result - - def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: - source: Source - for source in data_object.source_collection.get_sources(source_type_sorting={ - "only_with_page": True, - }): - new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level) - if new_data_object is not None: - data_object.merge(new_data_object) - - return data_object - - def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]: - if not source.has_page: - return None - - source_type = source.page.get_source_type(source=source) - if source_type is None: - self.LOGGER.debug(f"Could not determine source type for {source}.") - return None - - func = getattr(source.page, fetch_map[source_type]) - - # fetching the data object and marking it as fetched - data_object: DataObject = func(source=source, **kwargs) - data_object.mark_as_fetched(source.hash_url) - return data_object - - def fetch_from_url(self, url: str) -> Optional[DataObject]: - source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) - if source is None: - return None - - return self.fetch_from_source(source=source) - - def _skip_object(self, data_object: DataObject) -> bool: - if isinstance(data_object, Album): - if not self.download_options.download_all and data_object.album_type in self.download_options.album_type_blacklist: - return True - - return False - - def download(self, data_object: DataObject, genre: str, **kwargs) -> DownloadResult: - # fetch the given object - self.fetch_details(data_object) - output(f"\nDownloading {data_object.option_string}...", color=BColors.BOLD) - - # fetching all parent objects (e.g. if you only download a song) - if not kwargs.get("fetched_upwards", False): - to_fetch: List[DataObject] = [data_object] - - while len(to_fetch) > 0: - new_to_fetch = [] - for d in to_fetch: - if self._skip_object(d): - continue - - self.fetch_details(d) - - for c in d.get_parent_collections(): - new_to_fetch.extend(c) - - to_fetch = new_to_fetch - - kwargs["fetched_upwards"] = True - - # download all children - download_result: DownloadResult = DownloadResult() - for c in data_object.get_child_collections(): - for d in c: - if self._skip_object(d): - continue - - download_result.merge(self.download(d, genre, **kwargs)) - - # actually download if the object is a song - if isinstance(data_object, Song): - """ - TODO - add the traced artist and album to the naming. - I am able to do that, because duplicate values are removed later on. - """ - - self._download_song(data_object, naming={ - "genre": [genre], - "audio_format": [main_settings["audio_format"]], - }) - - return download_result - - def _extract_fields_from_template(self, path_template: str) -> Set[str]: - return set(re.findall(r"{([^}]+)}", path_template)) - - def _parse_path_template(self, path_template: str, naming: Dict[str, List[str]]) -> str: - field_names: Set[str] = self._extract_fields_from_template(path_template) - - for field in field_names: - if len(naming[field]) == 0: - raise MKMissingNameException(f"Missing field for {field}.") - - path_template = path_template.replace(f"{{{field}}}", naming[field][0]) - - return path_template - - def _download_song(self, song: Song, naming: dict) -> DownloadOptions: - """ - TODO - Search the song in the file system. - """ - r = DownloadResult(total=1) - - # pre process the data recursively - song.compile() - - # manage the naming - naming: Dict[str, List[str]] = defaultdict(list, naming) - naming["song"].append(song.title_value) - naming["isrc"].append(song.isrc) - naming["album"].extend(a.title_value for a in song.album_collection) - naming["album_type"].extend(a.album_type.value for a in song.album_collection) - naming["artist"].extend(a.name for a in song.artist_collection) - naming["artist"].extend(a.name for a in song.feature_artist_collection) - for a in song.album_collection: - naming["label"].extend([l.title_value for l in a.label_collection]) - # removing duplicates from the naming, and process the strings - for key, value in naming.items(): - # https://stackoverflow.com/a/17016257 - naming[key] = list(dict.fromkeys(value)) - song.genre = naming["genre"][0] - - # manage the targets - tmp: Target = Target.temp(file_extension=main_settings["audio_format"]) - - song.target_collection.append(Target( - relative_to_music_dir=True, - file_path=Path( - self._parse_path_template(main_settings["download_path"], naming=naming), - self._parse_path_template(main_settings["download_file"], naming=naming), - ) - )) - for target in song.target_collection: - if target.exists: - output(f'{target.file_path} {BColors.OKGREEN.value}[already exists]', color=BColors.GREY) - r.found_on_disk += 1 - - if not self.download_options.download_again_if_found: - target.copy_content(tmp) - else: - target.create_path() - output(f'{target.file_path}', color=BColors.GREY) - - # this streams from every available source until something succeeds, setting the skip intervals to the values of the according source - used_source: Optional[Source] = None - skip_intervals: List[Tuple[float, float]] = [] - for source in song.source_collection.get_sources(source_type_sorting={ - "only_with_page": True, - "sort_key": lambda page: page.download_priority, - "reverse": True, - }): - if tmp.exists: - break - - used_source = source - streaming_results = source.page.download_song_to_target(source=source, target=tmp, desc="download") - skip_intervals = source.page.get_skip_intervals(song=song, source=source) - - # if something has been downloaded but it somehow failed, delete the file - if streaming_results.is_fatal_error and tmp.exists: - tmp.delete() - - # if everything went right, the file should exist now - if not tmp.exists: - if used_source is None: - r.error_message = f"No source found for {song.option_string}." - else: - r.error_message = f"Something went wrong downloading {song.option_string}." - return r - - # post process the audio - found_on_disk = used_source is None - if not found_on_disk or self.download_options.process_audio_if_found: - correct_codec(target=tmp, skip_intervals=skip_intervals) - r.sponsor_segments = len(skip_intervals) - - if used_source is not None: - used_source.page.post_process_hook(song=song, temp_target=tmp) - - if not found_on_disk or self.download_options.process_metadata_if_found: - write_metadata_to_target(metadata=song.metadata, target=tmp, song=song) - - # copy the tmp target to the final locations - for target in song.target_collection: - tmp.copy_content(target) - - tmp.delete() - return r - - def fetch_url(self, url: str, **kwargs) -> DataObject: - source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) - - if source is None or source.page is None: - raise UrlNotFoundException(url=url) - - return source.page.fetch_object_from_source(source=source, **kwargs) \ No newline at end of file diff --git a/music_kraken/download/results.py b/music_kraken/download/results.py index 2486c26..c6ac522 100644 --- a/music_kraken/download/results.py +++ b/music_kraken/download/results.py @@ -1,8 +1,8 @@ -from typing import Tuple, Type, Dict, List, Generator, Union from dataclasses import dataclass +from typing import Dict, Generator, List, Tuple, Type, Union from ..objects import DatabaseObject -from ..pages import Page, EncyclopaediaMetallum, Musify +from . import Page @dataclass diff --git a/music_kraken/pages/__init__.py b/music_kraken/pages/__init__.py index 1e015a6..83a147c 100644 --- a/music_kraken/pages/__init__.py +++ b/music_kraken/pages/__init__.py @@ -1,55 +1,9 @@ -from typing import Type, Generator, Set, Dict, List from collections import defaultdict +from typing import Dict, Generator, List, Set, Type +from ._bandcamp import Bandcamp from ._encyclopaedia_metallum import EncyclopaediaMetallum +from ._genius import Genius from ._musify import Musify from ._youtube import YouTube from ._youtube_music import YoutubeMusic -from ._bandcamp import Bandcamp -from ._genius import Genius -from ._abstract import Page, INDEPENDENT_DB_OBJECTS - - -_registered_pages: Dict[Type[Page], Set[Page]] = defaultdict(set) - - -def get_pages(*page_types: List[Type[Page]]) -> Generator[Page, None, None]: - if len(page_types) == 0: - page_types = _registered_pages.keys() - - for page_type in page_types: - yield from _registered_pages[page_type] - - -def register_page(page_type: Type[Page], **kwargs): - if page_type in _registered_pages: - return - - _registered_pages[page_type].add(page_type(**kwargs)) - - -def deregister_page(page_type: Type[Page]): - if page_type not in _registered_pages: - return - - for p in _registered_pages[page_type]: - p.__del__() - del _registered_pages[page_type] - -def scan_for_pages(**kwargs): - # assuming the wanted pages are the leaf classes of the interface - leaf_classes = [] - - _class_list = [Page] - while len(_class_list): - _class = _class_list.pop() - _class_subclasses = _class.__subclasses__() - - if len(_class_subclasses) == 0: - if _class.REGISTER: - leaf_classes.append(_class) - else: - _class_list.extend(_class_subclasses) - - for leaf_class in leaf_classes: - register_page(leaf_class, **kwargs) diff --git a/music_kraken/pages/_abstract.py b/music_kraken/pages/_abstract.py deleted file mode 100644 index 2b76c59..0000000 --- a/music_kraken/pages/_abstract.py +++ /dev/null @@ -1,152 +0,0 @@ -from __future__ import annotations - -import logging -import random -import re -from copy import copy -from pathlib import Path -from typing import Optional, Union, Type, Dict, Set, List, Tuple, TypedDict, TYPE_CHECKING -from string import Formatter -from dataclasses import dataclass, field - -import requests -from bs4 import BeautifulSoup - -if TYPE_CHECKING: - from ..download.page_attributes import DownloadOptions, FetchOptions -from ..connection import Connection -from ..objects import ( - Song, - Source, - Album, - Artist, - Target, - DatabaseObject, - Options, - Collection, - Label, -) -from ..utils.enums import SourceType -from ..utils.enums.album import AlbumType -from ..audio import write_metadata_to_target, correct_codec -from ..utils.config import main_settings -from ..utils.support_classes.query import Query -from ..utils.support_classes.download_result import DownloadResult -from ..utils.string_processing import fit_to_file_system -from ..utils import trace, output, BColors - -INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song] -INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]] - - -class Page: - REGISTER = True - SOURCE_TYPE: SourceType - LOGGER: logging.Logger - - def __new__(cls, *args, **kwargs): - cls.LOGGER = logging.getLogger(cls.__name__) - return super().__new__(cls) - - def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): - self.SOURCE_TYPE.register_page(self) - - self.download_options: DownloadOptions = download_options or DownloadOptions() - self.fetch_options: FetchOptions = fetch_options or FetchOptions() - - def __del__(self): - self.SOURCE_TYPE.deregister_page() - - def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): - """ - Perform a regex search on the given string, using a single or a list of - patterns returning the first matching group. - In case of failure return a default value or raise a WARNING or a - RegexNotFoundError, depending on fatal, specifying the field name. - """ - - if isinstance(pattern, str): - mobj = re.search(pattern, string, flags) - else: - for p in pattern: - mobj = re.search(p, string, flags) - if mobj: - break - - if mobj: - if group is None: - # return the first matching group - return next(g for g in mobj.groups() if g is not None) - elif isinstance(group, (list, tuple)): - return tuple(mobj.group(g) for g in group) - else: - return mobj.group(group) - - return default - - def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]: - return None - - def get_soup_from_response(self, r: requests.Response) -> BeautifulSoup: - return BeautifulSoup(r.content, "html.parser") - - # to search stuff - def search(self, query: Query) -> List[DatabaseObject]: - music_object = query.music_object - - search_functions = { - Song: self.song_search, - Album: self.album_search, - Artist: self.artist_search, - Label: self.label_search - } - - if type(music_object) in search_functions: - r = search_functions[type(music_object)](music_object) - if r is not None and len(r) > 0: - return r - - r = [] - for default_query in query.default_search: - for single_option in self.general_search(default_query): - r.append(single_option) - - return r - - def general_search(self, search_query: str) -> List[DatabaseObject]: - return [] - - def label_search(self, label: Label) -> List[Label]: - return [] - - def artist_search(self, artist: Artist) -> List[Artist]: - return [] - - def album_search(self, album: Album) -> List[Album]: - return [] - - def song_search(self, song: Song) -> List[Song]: - return [] - - # to fetch stuff - def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song: - return Song() - - def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album: - return Album() - - def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist: - return Artist() - - def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: - return Label() - - # to download stuff - def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]: - return [] - - def post_process_hook(self, song: Song, temp_target: Target, **kwargs): - pass - - def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: - return DownloadResult() diff --git a/music_kraken/pages/_bandcamp.py b/music_kraken/pages/_bandcamp.py index 658b448..ab4977e 100644 --- a/music_kraken/pages/_bandcamp.py +++ b/music_kraken/pages/_bandcamp.py @@ -1,33 +1,22 @@ -from typing import List, Optional, Type -from urllib.parse import urlparse, urlunparse import json from enum import Enum -from bs4 import BeautifulSoup -import pycountry +from typing import List, Optional, Type +from urllib.parse import urlparse, urlunparse + +import pycountry +from bs4 import BeautifulSoup -from ..objects import Source, DatabaseObject -from ._abstract import Page -from ..objects import ( - Artist, - Source, - SourceType, - Song, - Album, - Label, - Target, - Contact, - ID3Timestamp, - Lyrics, - FormattedText, - Artwork, -) from ..connection import Connection +from ..download import Page +from ..objects import (Album, Artist, Artwork, Contact, DatabaseObject, + FormattedText, ID3Timestamp, Label, Lyrics, Song, + Source, SourceType, Target) from ..utils import dump_to_file -from ..utils.enums import SourceType, ALL_SOURCE_TYPES -from ..utils.support_classes.download_result import DownloadResult -from ..utils.string_processing import clean_song_title -from ..utils.config import main_settings, logging_settings +from ..utils.config import logging_settings, main_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.shared import DEBUG +from ..utils.string_processing import clean_song_title +from ..utils.support_classes.download_result import DownloadResult if DEBUG: from ..utils import dump_to_file diff --git a/music_kraken/pages/_encyclopaedia_metallum.py b/music_kraken/pages/_encyclopaedia_metallum.py index cd6ec4c..99ca57e 100644 --- a/music_kraken/pages/_encyclopaedia_metallum.py +++ b/music_kraken/pages/_encyclopaedia_metallum.py @@ -1,31 +1,20 @@ from collections import defaultdict -from typing import List, Optional, Dict, Type, Union -from bs4 import BeautifulSoup +from typing import Dict, List, Optional, Type, Union +from urllib.parse import urlencode, urlparse + import pycountry -from urllib.parse import urlparse, urlencode +from bs4 import BeautifulSoup from ..connection import Connection -from ..utils.config import logging_settings -from ._abstract import Page -from ..utils.enums import SourceType, ALL_SOURCE_TYPES -from ..utils.enums.album import AlbumType -from ..utils.support_classes.query import Query -from ..objects import ( - Lyrics, - Artist, - Source, - Song, - Album, - ID3Timestamp, - FormattedText, - Label, - Options, - DatabaseObject -) -from ..utils.shared import DEBUG +from ..download import Page +from ..objects import (Album, Artist, DatabaseObject, FormattedText, + ID3Timestamp, Label, Lyrics, Options, Song, Source) from ..utils import dump_to_file - - +from ..utils.config import logging_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType +from ..utils.enums.album import AlbumType +from ..utils.shared import DEBUG +from ..utils.support_classes.query import Query ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, { "Full-length": AlbumType.STUDIO_ALBUM, diff --git a/music_kraken/pages/_genius.py b/music_kraken/pages/_genius.py index 08d38f0..f79a1db 100644 --- a/music_kraken/pages/_genius.py +++ b/music_kraken/pages/_genius.py @@ -1,33 +1,22 @@ -from typing import List, Optional, Type -from urllib.parse import urlparse, urlunparse, urlencode import json from enum import Enum -from bs4 import BeautifulSoup -import pycountry +from typing import List, Optional, Type +from urllib.parse import urlencode, urlparse, urlunparse + +import pycountry +from bs4 import BeautifulSoup -from ..objects import Source, DatabaseObject -from ._abstract import Page -from ..objects import ( - Artist, - Source, - SourceType, - Song, - Album, - Label, - Target, - Contact, - ID3Timestamp, - Lyrics, - FormattedText, - Artwork, -) from ..connection import Connection +from ..download import Page +from ..objects import (Album, Artist, Artwork, Contact, DatabaseObject, + FormattedText, ID3Timestamp, Label, Lyrics, Song, + Source, SourceType, Target) from ..utils import dump_to_file, traverse_json_path -from ..utils.enums import SourceType, ALL_SOURCE_TYPES -from ..utils.support_classes.download_result import DownloadResult -from ..utils.string_processing import clean_song_title -from ..utils.config import main_settings, logging_settings +from ..utils.config import logging_settings, main_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.shared import DEBUG +from ..utils.string_processing import clean_song_title +from ..utils.support_classes.download_result import DownloadResult if DEBUG: from ..utils import dump_to_file diff --git a/music_kraken/pages/_musify.py b/music_kraken/pages/_musify.py index 0d00d26..9e4c6ae 100644 --- a/music_kraken/pages/_musify.py +++ b/music_kraken/pages/_musify.py @@ -1,34 +1,23 @@ from collections import defaultdict from dataclasses import dataclass from enum import Enum -from typing import List, Optional, Type, Union, Generator, Dict, Any +from typing import Any, Dict, Generator, List, Optional, Type, Union from urllib.parse import urlparse import pycountry from bs4 import BeautifulSoup from ..connection import Connection -from ._abstract import Page -from ..utils.enums import SourceType, ALL_SOURCE_TYPES -from ..utils.enums.album import AlbumType, AlbumStatus -from ..objects import ( - Artist, - Source, - Song, - Album, - ID3Timestamp, - FormattedText, - Label, - Target, - DatabaseObject, - Lyrics, - Artwork -) +from ..download import Page +from ..objects import (Album, Artist, Artwork, DatabaseObject, FormattedText, + ID3Timestamp, Label, Lyrics, Song, Source, Target) +from ..utils import shared, string_processing from ..utils.config import logging_settings, main_settings -from ..utils import string_processing, shared +from ..utils.enums import ALL_SOURCE_TYPES, SourceType +from ..utils.enums.album import AlbumStatus, AlbumType from ..utils.string_processing import clean_song_title -from ..utils.support_classes.query import Query from ..utils.support_classes.download_result import DownloadResult +from ..utils.support_classes.query import Query """ https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent diff --git a/music_kraken/pages/_youtube.py b/music_kraken/pages/_youtube.py index 2530aa1..5bbee4e 100644 --- a/music_kraken/pages/_youtube.py +++ b/music_kraken/pages/_youtube.py @@ -1,29 +1,19 @@ -from typing import List, Optional, Type, Tuple -from urllib.parse import urlparse, urlunparse, parse_qs from enum import Enum +from typing import List, Optional, Tuple, Type +from urllib.parse import parse_qs, urlparse, urlunparse import python_sponsorblock -from ..objects import Source, DatabaseObject, Song, Target -from ._abstract import Page -from ..objects import ( - Artist, - Source, - Song, - Album, - Label, - Target, - FormattedText, - ID3Timestamp -) from ..connection import Connection +from ..download import Page +from ..objects import (Album, Artist, DatabaseObject, FormattedText, + ID3Timestamp, Label, Song, Source, Target) +from ..utils.config import logging_settings, main_settings, youtube_settings +from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.string_processing import clean_song_title -from ..utils.enums import SourceType, ALL_SOURCE_TYPES from ..utils.support_classes.download_result import DownloadResult -from ..utils.config import youtube_settings, main_settings, logging_settings - -from ._youtube_music.super_youtube import SuperYouTube, YouTubeUrl, get_invidious_url, YouTubeUrlType - +from ._youtube_music.super_youtube import (SuperYouTube, YouTubeUrl, + YouTubeUrlType, get_invidious_url) """ - https://yt.artemislena.eu/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance diff --git a/music_kraken/pages/_youtube_music/youtube_music.py b/music_kraken/pages/_youtube_music/youtube_music.py index b2d4aa2..ad66844 100644 --- a/music_kraken/pages/_youtube_music/youtube_music.py +++ b/music_kraken/pages/_youtube_music/youtube_music.py @@ -1,46 +1,33 @@ -from __future__ import unicode_literals, annotations +from __future__ import annotations, unicode_literals -from typing import Dict, List, Optional, Set, Type -from urllib.parse import urlparse, urlunparse, quote, parse_qs, urlencode +import json import logging import random -import json -from dataclasses import dataclass import re -from functools import lru_cache from collections import defaultdict +from dataclasses import dataclass +from functools import lru_cache +from typing import Dict, List, Optional, Set, Type +from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse import youtube_dl from youtube_dl.extractor.youtube import YoutubeIE from youtube_dl.utils import DownloadError +from ...connection import Connection +from ...download import Page +from ...objects import Album, Artist, Artwork +from ...objects import DatabaseObject as DataObject +from ...objects import (FormattedText, ID3Timestamp, Label, Lyrics, Song, + Source, Target) +from ...utils import dump_to_file, get_current_millis, traverse_json_path +from ...utils.config import logging_settings, main_settings, youtube_settings +from ...utils.enums import ALL_SOURCE_TYPES, SourceType +from ...utils.enums.album import AlbumType from ...utils.exception.config import SettingValueError -from ...utils.config import main_settings, youtube_settings, logging_settings from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING from ...utils.string_processing import clean_song_title -from ...utils import get_current_millis, traverse_json_path - -from ...utils import dump_to_file - -from .._abstract import Page -from ...objects import ( - DatabaseObject as DataObject, - Source, - FormattedText, - ID3Timestamp, - Artwork, - Artist, - Song, - Album, - Label, - Target, - Lyrics, -) -from ...connection import Connection -from ...utils.enums import SourceType, ALL_SOURCE_TYPES -from ...utils.enums.album import AlbumType from ...utils.support_classes.download_result import DownloadResult - from ._list_render import parse_renderer from ._music_object_render import parse_run_element from .super_youtube import SuperYouTube -- 2.45.2 From b5a5559f7ba4390e15e9777a81e0409f30b693c2 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Thu, 23 May 2024 18:05:18 +0200 Subject: [PATCH 07/35] feat: created layout for option --- music_kraken/download/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index e9f0fd0..11b0f3b 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -445,3 +445,16 @@ class Page: def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: return DownloadResult() + +class Option: + """ + This could represent a data object, a string or a page. + """ + + def __init__(self, value: Any, text: Optional[str] = None, keys: Set[str] = None): + self.value = value + self.text = text or str(value) + + self.keys = keys or set() + self.keys.add(self.text) + -- 2.45.2 From c0fbd169296b55ccc903233819e35ba89fb6a9bd Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 24 May 2024 13:21:07 +0200 Subject: [PATCH 08/35] feat: basic select layout --- music_kraken/download/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 11b0f3b..2a07672 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -451,10 +451,27 @@ class Option: This could represent a data object, a string or a page. """ - def __init__(self, value: Any, text: Optional[str] = None, keys: Set[str] = None): + def __init__(self, value: Any, text: Optional[str] = None, keys: Set[str] = None, hidden: bool = False): self.value = value self.text = text or str(value) + self.hidden = hidden self.keys = keys or set() self.keys.add(self.text) + +class SelectOption: + def __init__(self, options: List[Option] = None): + self._key_to_option: Dict[Any, Option] = dict() + self._options: List[Option] = options + + self.extend(options or []) + + def append(self, option: Option): + self._options.append(option) + for key in option.keys: + self._key_to_option[key] = option + + def extend(self, options: List[Option]): + for option in options: + self.append(option) \ No newline at end of file -- 2.45.2 From cef87460a7ca208ed8c00e55292d05b4ffc6ced1 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 24 May 2024 14:46:38 +0200 Subject: [PATCH 09/35] draft --- .vscode/launch.json | 6 + music_kraken/cli/main_downloader.py | 3 +- music_kraken/download/__init__.py | 109 +++++++++++++++--- music_kraken/download/results.py | 8 +- .../pages/_youtube_music/super_youtube.py | 25 ++-- music_kraken/utils/exception/__init__.py | 3 + 6 files changed, 119 insertions(+), 35 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 24c2088..f864249 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -17,6 +17,12 @@ "request": "launch", "program": "development/actual_donwload.py", "console": "integratedTerminal" + }, + { + "name": "Python Debugger: Music Kraken", + "type": "debugpy", + "request": "launch", // run the module + "module": "music_kraken", } ] } \ No newline at end of file diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 2356f8e..b21b265 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -4,10 +4,9 @@ from pathlib import Path from typing import Dict, List, Set, Type from .. import console -from ..download import Downloader +from ..download import Downloader, Page from ..download.results import GoToResults, Option, PageResults, Results from ..objects import Album, Artist, DatabaseObject, Song -from ..pages import Page from ..utils import BColors, output from ..utils.config import main_settings, write_config from ..utils.enums.colors import BColors diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 2a07672..3aa27fc 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -8,8 +8,8 @@ from copy import copy from dataclasses import dataclass, field from pathlib import Path from string import Formatter -from typing import (TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Type, - TypedDict, Union) +from typing import (TYPE_CHECKING, Any, Callable, Dict, Generator, List, + Optional, Set, Tuple, Type, TypedDict, Union) import requests from bs4 import BeautifulSoup @@ -23,7 +23,7 @@ from ..utils import BColors, output, trace from ..utils.config import main_settings, youtube_settings from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.enums.album import AlbumType -from ..utils.exception import MKMissingNameException +from ..utils.exception import MKComposeException, MKMissingNameException from ..utils.exception.download import UrlNotFoundException from ..utils.path_manager import LOCATIONS from ..utils.shared import DEBUG_PAGES @@ -75,7 +75,7 @@ class Downloader: self.scan_for_pages(**kwargs) def register_page(self, page_type: Type[Page], **kwargs): - if page_type in _registered_pages: + if page_type in self._registered_pages: return self._registered_pages[page_type].add(page_type( @@ -343,13 +343,15 @@ class Page: return super().__new__(cls) def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): - self.SOURCE_TYPE.register_page(self) + if self.SOURCE_TYPE is not None: + self.SOURCE_TYPE.register_page(self) self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() def __del__(self): - self.SOURCE_TYPE.deregister_page() + if self.SOURCE_TYPE is not None: + self.SOURCE_TYPE.deregister_page() def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): """ @@ -451,27 +453,106 @@ class Option: This could represent a data object, a string or a page. """ - def __init__(self, value: Any, text: Optional[str] = None, keys: Set[str] = None, hidden: bool = False): + def __init__( + self, + value: Any, + text: Optional[str] = None, + keys: List[Any] = None, + hidden: bool = False, + parse_key: Callable[[Any], Any] = lambda x: x, + ): + self._parse_key: Callable[[Any], Any] = parse_key + self.value = value self.text = text or str(value) self.hidden = hidden - self.keys = keys or set() - self.keys.add(self.text) + self._raw_keys = set(keys or []) + self._raw_keys.add(self.text) + self.keys = set(self.parse_key(key) for key in self._raw_keys) + + def register_key(self, key: Any): + self._raw_keys.add(key) + self.keys.add(self._parse_key(key)) + + @property + def parse_key(self) -> Callable[[Any], Any]: + return self._parse_key + + @parse_key.setter + def parse_key(self, value: Callable[[Any], Any]): + self._parse_key = value + self.keys = set(self._parse_key(key) for key in self._raw_keys) + + def __str__(self): + return self.text -class SelectOption: - def __init__(self, options: List[Option] = None): +class Select: + def __init__( + self, + options: List[Option] = None, + option_factory: Callable[[Any], Option] = None, + raw_options: List[Any] = None, + parse_option_key: Callable[[Any], Any] = lambda x: x, + ask_for_creating_option: Callable[[Option], bool] = lambda x: True, + **kwargs + ): + self._parse_option_key: Callable[[Any], Any] = parse_option_key + self._ask_for_creating_option: Callable[[Option], bool] = ask_for_creating_option + self._key_to_option: Dict[Any, Option] = dict() - self._options: List[Option] = options + self._options: List[Option] = [] - self.extend(options or []) + options = options or [] + self.option_factory: Optional[Callable[[Any], Option]] = option_factory + if self.can_create_options: + for raw_option in raw_options or []: + self.append(self.option_factory(raw_option)) + elif raw_options is not None: + raise MKComposeException("Cannot create options without a factory.") + + self.extend(options) + + @property + def can_create_options(self) -> bool: + return self.option_factory is not None def append(self, option: Option): + option.parse_key = self._parse_option_key self._options.append(option) for key in option.keys: self._key_to_option[key] = option def extend(self, options: List[Option]): for option in options: - self.append(option) \ No newline at end of file + self.append(option) + + def __iter__(self) -> Generator[Option, None, None]: + for option in self._options: + if option.hidden: + continue + + yield option + + def __contains__(self, key: Any) -> bool: + return key in self._key_to_option + + def __getitem__(self, key: Any) -> Option: + return self._key_to_option[key] + + def create_option(self, key: Any, **kwargs) -> Option: + if not self.can_create_options: + raise MKComposeException("Cannot create options without a factory.") + + option = self.option_factory(key, **kwargs) + self.append(option) + return option + + def choose(self, key: Any) -> Optional[Option]: + if key not in self: + if self.can_create_options and self._ask_for_creating_option(key): + return self.create_option(key) + return None + + return self[key] diff --git a/music_kraken/download/results.py b/music_kraken/download/results.py index c6ac522..3587da4 100644 --- a/music_kraken/download/results.py +++ b/music_kraken/download/results.py @@ -1,8 +1,12 @@ +from __future__ import annotations + from dataclasses import dataclass -from typing import Dict, Generator, List, Tuple, Type, Union +from typing import TYPE_CHECKING, Dict, Generator, List, Tuple, Type, Union from ..objects import DatabaseObject -from . import Page + +if TYPE_CHECKING: + from . import Page @dataclass diff --git a/music_kraken/pages/_youtube_music/super_youtube.py b/music_kraken/pages/_youtube_music/super_youtube.py index fa5ce1c..3e3dced 100644 --- a/music_kraken/pages/_youtube_music/super_youtube.py +++ b/music_kraken/pages/_youtube_music/super_youtube.py @@ -1,26 +1,17 @@ -from typing import List, Optional, Type, Tuple -from urllib.parse import urlparse, urlunparse, parse_qs from enum import Enum -import requests +from typing import List, Optional, Tuple, Type +from urllib.parse import parse_qs, urlparse, urlunparse import python_sponsorblock +import requests -from ...objects import Source, DatabaseObject, Song, Target -from .._abstract import Page -from ...objects import ( - Artist, - Source, - Song, - Album, - Label, - Target, - FormattedText, - ID3Timestamp -) from ...connection import Connection +from ...download import Page +from ...objects import (Album, Artist, DatabaseObject, FormattedText, + ID3Timestamp, Label, Song, Source, Target) +from ...utils.config import logging_settings, main_settings, youtube_settings +from ...utils.enums import ALL_SOURCE_TYPES, SourceType from ...utils.support_classes.download_result import DownloadResult -from ...utils.config import youtube_settings, logging_settings, main_settings -from ...utils.enums import SourceType, ALL_SOURCE_TYPES def get_invidious_url(path: str = "", params: str = "", query: str = "", fragment: str = "") -> str: diff --git a/music_kraken/utils/exception/__init__.py b/music_kraken/utils/exception/__init__.py index 8f139fb..42b26d7 100644 --- a/music_kraken/utils/exception/__init__.py +++ b/music_kraken/utils/exception/__init__.py @@ -3,6 +3,9 @@ class MKBaseException(Exception): self.message = message super().__init__(message, **kwargs) +# Compose exceptions. Those usually mean a bug on my side. +class MKComposeException(MKBaseException): + pass # Downloading class MKDownloadException(MKBaseException): -- 2.45.2 From c24cf701c1992bd51253757c432bf5a0abeeee50 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 24 May 2024 15:28:47 +0200 Subject: [PATCH 10/35] fix: pages were not in the subclasses because the module was never importet --- music_kraken/download/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 3aa27fc..39e8934 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -78,6 +78,8 @@ class Downloader: if page_type in self._registered_pages: return + print(page_type) + self._registered_pages[page_type].add(page_type( download_options=self.download_options, fetch_options=self.fetch_options, @@ -94,6 +96,7 @@ class Downloader: def scan_for_pages(self, **kwargs): # assuming the wanted pages are the leaf classes of the interface + from .. import pages leaf_classes = [] class_list = [Page] @@ -342,16 +345,18 @@ class Page: cls.LOGGER = logging.getLogger(cls.__name__) return super().__new__(cls) + @classmethod + def is_leaf_page(cls) -> bool: + return len(cls.__subclasses__()) == 0 + def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): - if self.SOURCE_TYPE is not None: - self.SOURCE_TYPE.register_page(self) + self.SOURCE_TYPE.register_page(self) self.download_options: DownloadOptions = download_options or DownloadOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions() def __del__(self): - if self.SOURCE_TYPE is not None: - self.SOURCE_TYPE.deregister_page() + self.SOURCE_TYPE.deregister_page() def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): """ -- 2.45.2 From 5af95f1b0310b4db6b559d0825990d54d3d63553 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 24 May 2024 15:46:42 +0200 Subject: [PATCH 11/35] feat: auto import pages in page module --- music_kraken/pages/__init__.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/music_kraken/pages/__init__.py b/music_kraken/pages/__init__.py index 83a147c..bda24f4 100644 --- a/music_kraken/pages/__init__.py +++ b/music_kraken/pages/__init__.py @@ -1,9 +1,23 @@ +import logging from collections import defaultdict +from pathlib import Path from typing import Dict, Generator, List, Set, Type +""" from ._bandcamp import Bandcamp from ._encyclopaedia_metallum import EncyclopaediaMetallum from ._genius import Genius from ._musify import Musify from ._youtube import YouTube from ._youtube_music import YoutubeMusic +""" + +_page_directory = Path(__file__).parent +_stem_blacklist = set(["__pycache__", "__init__"]) + +for _file in _page_directory.iterdir(): + if _file.stem in _stem_blacklist: + continue + + logging.debug(f"importing {_file.absolute()}") + exec(f"from . import {_file.stem}") -- 2.45.2 From 0f2229b0f2b95e88a1f14633a689829905915704 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Fri, 24 May 2024 17:00:39 +0200 Subject: [PATCH 12/35] feat: completely dynamified the datasource import --- music_kraken/download/__init__.py | 6 ++-- music_kraken/pages/__init__.py | 47 +++++++++++++++++++++++++------ 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 39e8934..0d806b3 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -78,8 +78,6 @@ class Downloader: if page_type in self._registered_pages: return - print(page_type) - self._registered_pages[page_type].add(page_type( download_options=self.download_options, fetch_options=self.fetch_options, @@ -97,6 +95,7 @@ class Downloader: def scan_for_pages(self, **kwargs): # assuming the wanted pages are the leaf classes of the interface from .. import pages + leaf_classes = [] class_list = [Page] @@ -110,6 +109,9 @@ class Downloader: else: class_list.extend(class_subclasses) + if Page in leaf_classes: + self.LOGGER.warn("couldn't find any data source") + return for leaf_class in leaf_classes: self.register_page(leaf_class, **kwargs) diff --git a/music_kraken/pages/__init__.py b/music_kraken/pages/__init__.py index bda24f4..a9fe75b 100644 --- a/music_kraken/pages/__init__.py +++ b/music_kraken/pages/__init__.py @@ -1,23 +1,52 @@ +import importlib +import inspect import logging +import pkgutil +import sys from collections import defaultdict +from copy import copy from pathlib import Path from typing import Dict, Generator, List, Set, Type -""" from ._bandcamp import Bandcamp from ._encyclopaedia_metallum import EncyclopaediaMetallum from ._genius import Genius from ._musify import Musify from ._youtube import YouTube from ._youtube_music import YoutubeMusic + + +def import_children(): + _page_directory = Path(__file__).parent + _stem_blacklist = set(["__pycache__", "__init__"]) + + for _file in _page_directory.iterdir(): + if _file.stem in _stem_blacklist: + continue + + logging.debug(f"importing {_file.absolute()}") + exec(f"from . import {_file.stem}") + +# module_blacklist = set(sys.modules.keys()) +import_children() + """ +classes = set() -_page_directory = Path(__file__).parent -_stem_blacklist = set(["__pycache__", "__init__"]) - -for _file in _page_directory.iterdir(): - if _file.stem in _stem_blacklist: +print(__name__) +for module_name, module in sys.modules.items(): + if module_name in module_blacklist or not module_name.startswith(__name__): continue - - logging.debug(f"importing {_file.absolute()}") - exec(f"from . import {_file.stem}") + + print("scanning module", module_name) + for name, obj in inspect.getmembers(module, predicate=inspect.isclass): + _module = obj.__module__ + if _module.startswith(__name__) and hasattr(obj, "SOURCE_TYPE"): + print("checking object", name, obj.__module__) + classes.add(obj) + print() + +print(*(c.__name__ for c in classes), sep=",\t") + +__all__ = [c.__name__ for c in classes] +""" \ No newline at end of file -- 2.45.2 From 49145a7d93b8e5e773a614417a7ca3d36127b492 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 13:41:24 +0200 Subject: [PATCH 13/35] feat: moved cli helpers to seperate file --- music_kraken/cli/main_downloader.py | 59 +++-------- music_kraken/cli/utils.py | 4 + music_kraken/download/__init__.py | 110 ------------------- music_kraken/download/components.py | 158 ++++++++++++++++++++++++++++ 4 files changed, 177 insertions(+), 154 deletions(-) create mode 100644 music_kraken/download/components.py diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index b21b265..9bc0b37 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -1,11 +1,13 @@ import random import re from pathlib import Path -from typing import Dict, List, Set, Type +from typing import Dict, Generator, List, Set, Type from .. import console -from ..download import Downloader, Page -from ..download.results import GoToResults, Option, PageResults, Results +from ..download import Downloader, Page, components +from ..download.results import GoToResults +from ..download.results import Option as ResultOption +from ..download.results import PageResults, Results from ..objects import Album, Artist, DatabaseObject, Song from ..utils import BColors, output from ..utils.config import main_settings, write_config @@ -17,7 +19,7 @@ from ..utils.string_processing import fit_to_file_system from ..utils.support_classes.download_result import DownloadResult from ..utils.support_classes.query import Query from .options.first_config import initial_config -from .utils import cli_function +from .utils import ask_for_bool, cli_function EXIT_COMMANDS = {"q", "quit", "exit", "abort"} ALPHABET = "abcdefghijklmnopqrstuvwxyz" @@ -25,50 +27,19 @@ PAGE_NAME_FILL = "-" MAX_PAGE_LEN = 21 -def get_existing_genre() -> List[str]: - """ - gets the name of all subdirectories of shared.MUSIC_DIR, - but filters out all directories, where the name matches with any patern - from shared.NOT_A_GENRE_REGEX. - """ - existing_genres: List[str] = [] - - # get all subdirectories of MUSIC_DIR, not the files in the dir. - existing_subdirectories: List[Path] = [f for f in main_settings["music_directory"].iterdir() if f.is_dir()] - - for subdirectory in existing_subdirectories: - name: str = subdirectory.name - - if not any(re.match(regex_pattern, name) for regex_pattern in main_settings["not_a_genre_regex"]): - existing_genres.append(name) - - existing_genres.sort() - - return existing_genres - - def get_genre(): - existing_genres = get_existing_genre() - for i, genre_option in enumerate(existing_genres): - print(f"{i + 1:0>2}: {genre_option}") + select_genre = components.GenreSelect() + select_genre._ask_for_creating_option = lambda key: ask_for_bool(f"Create the genre \"{key}\"") - while True: - genre = input("Id or new genre: ") + genre: Optional[components.Option] = None - if genre.isdigit(): - genre_id = int(genre) - 1 - if genre_id >= len(existing_genres): - print(f"No genre under the id {genre_id + 1}.") - continue + while genre is None: + for genre in select_genre: + print(genre) - return existing_genres[genre_id] + genre = select_genre.choose(input("Id or new genre: ")) - new_genre = fit_to_file_system(genre) - - agree_inputs = {"y", "yes", "ok"} - verification = input(f"create new genre \"{new_genre}\"? (Y/N): ").lower() - if verification in agree_inputs: - return new_genre + return genre.value def help_message(): @@ -111,7 +82,7 @@ class CliDownloader: page_count = 0 for option in self.current_results.formatted_generator(): - if isinstance(option, Option): + if isinstance(option, ResultOption): r = f"{BColors.GREY.value}{option.index:0{self.option_digits}}{BColors.ENDC.value} {option.music_object.option_string}" print(r) else: diff --git a/music_kraken/cli/utils.py b/music_kraken/cli/utils.py index e2f9bab..cfcaadc 100644 --- a/music_kraken/cli/utils.py +++ b/music_kraken/cli/utils.py @@ -39,4 +39,8 @@ def print_cute_message(): print(message) +AGREE_INPUTS = {"y", "yes", "ok"} +def ask_for_bool(msg: str) -> bool: + i = input(msg + " (Y/N):").lower() + return i in AGREE_INPUTS \ No newline at end of file diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 0d806b3..1a4aa6d 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -453,113 +453,3 @@ class Page: def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult: return DownloadResult() - - -class Option: - """ - This could represent a data object, a string or a page. - """ - - def __init__( - self, - value: Any, - text: Optional[str] = None, - keys: List[Any] = None, - hidden: bool = False, - parse_key: Callable[[Any], Any] = lambda x: x, - ): - self._parse_key: Callable[[Any], Any] = parse_key - - self.value = value - self.text = text or str(value) - self.hidden = hidden - - self._raw_keys = set(keys or []) - self._raw_keys.add(self.text) - self.keys = set(self.parse_key(key) for key in self._raw_keys) - - def register_key(self, key: Any): - self._raw_keys.add(key) - self.keys.add(self._parse_key(key)) - - @property - def parse_key(self) -> Callable[[Any], Any]: - return self._parse_key - - @parse_key.setter - def parse_key(self, value: Callable[[Any], Any]): - self._parse_key = value - self.keys = set(self._parse_key(key) for key in self._raw_keys) - - def __str__(self): - return self.text - - -class Select: - def __init__( - self, - options: List[Option] = None, - option_factory: Callable[[Any], Option] = None, - raw_options: List[Any] = None, - parse_option_key: Callable[[Any], Any] = lambda x: x, - ask_for_creating_option: Callable[[Option], bool] = lambda x: True, - **kwargs - ): - self._parse_option_key: Callable[[Any], Any] = parse_option_key - self._ask_for_creating_option: Callable[[Option], bool] = ask_for_creating_option - - self._key_to_option: Dict[Any, Option] = dict() - self._options: List[Option] = [] - - options = options or [] - self.option_factory: Optional[Callable[[Any], Option]] = option_factory - if self.can_create_options: - for raw_option in raw_options or []: - self.append(self.option_factory(raw_option)) - elif raw_options is not None: - raise MKComposeException("Cannot create options without a factory.") - - self.extend(options) - - @property - def can_create_options(self) -> bool: - return self.option_factory is not None - - def append(self, option: Option): - option.parse_key = self._parse_option_key - self._options.append(option) - for key in option.keys: - self._key_to_option[key] = option - - def extend(self, options: List[Option]): - for option in options: - self.append(option) - - def __iter__(self) -> Generator[Option, None, None]: - for option in self._options: - if option.hidden: - continue - - yield option - - def __contains__(self, key: Any) -> bool: - return key in self._key_to_option - - def __getitem__(self, key: Any) -> Option: - return self._key_to_option[key] - - def create_option(self, key: Any, **kwargs) -> Option: - if not self.can_create_options: - raise MKComposeException("Cannot create options without a factory.") - - option = self.option_factory(key, **kwargs) - self.append(option) - return option - - def choose(self, key: Any) -> Optional[Option]: - if key not in self: - if self.can_create_options and self._ask_for_creating_option(key): - return self.create_option(key) - return None - - return self[key] diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py new file mode 100644 index 0000000..e30bcb5 --- /dev/null +++ b/music_kraken/download/components.py @@ -0,0 +1,158 @@ +import re +from pathlib import Path +from typing import Any, Callable, Dict, Generator, List, Optional + +from ..utils.config import main_settings +from ..utils.exception import MKComposeException +from ..utils.string_processing import unify + + +class Option: + """ + This could represent a data object, a string or a page. + """ + + def __init__( + self, + value: Any, + text: Optional[str] = None, + keys: List[Any] = None, + hidden: bool = False, + parse_key: Callable[[Any], Any] = lambda x: x, + ): + self._parse_key: Callable[[Any], Any] = parse_key + + self.value = value + self.text = text or str(value) + self.hidden = hidden + + self._raw_keys = set(keys or []) + self._raw_keys.add(self.text) + self.keys = set(self.parse_key(key) for key in self._raw_keys) + + def register_key(self, key: Any): + self._raw_keys.add(key) + self.keys.add(self._parse_key(key)) + + @property + def parse_key(self) -> Callable[[Any], Any]: + return self._parse_key + + @parse_key.setter + def parse_key(self, value: Callable[[Any], Any]): + self._parse_key = value + self.keys = set(self._parse_key(key) for key in self._raw_keys) + + def __str__(self): + return self.text + + +class Select: + def __init__( + self, + options: Generator[Option, None, None] = None, + option_factory: Callable[[Any], Option] = None, + raw_options: List[Any] = None, + parse_option_key: Callable[[Any], Any] = lambda x: x, + ask_for_creating_option: Callable[[Option], bool] = lambda x: True, + sort: bool = False, + **kwargs + ): + self._parse_option_key: Callable[[Any], Any] = parse_option_key + self._ask_for_creating_option: Callable[[Option], bool] = ask_for_creating_option + + self._key_to_option: Dict[Any, Option] = dict() + self._options: List[Option] = [] + + options = options or [] + self.option_factory: Optional[Callable[[Any], Option]] = option_factory + if self.can_create_options: + _raw_options = raw_options or [] + if sort: + _raw_options = sorted(_raw_options) + + for raw_option in _raw_options: + self.append(self.option_factory(raw_option)) + elif raw_options is not None: + raise MKComposeException("Cannot create options without a factory.") + + self.extend(options) + + @property + def can_create_options(self) -> bool: + return self.option_factory is not None + + def append(self, option: Option): + option.parse_key = self._parse_option_key + self._options.append(option) + for key in option.keys: + self._key_to_option[key] = option + + def extend(self, options: List[Option]): + for option in options: + self.append(option) + + def __iter__(self) -> Generator[Option, None, None]: + for option in self._options: + if option.hidden: + continue + + yield option + + def __contains__(self, key: Any) -> bool: + return key in self._key_to_option + + def __getitem__(self, key: Any) -> Option: + return self._key_to_option[key] + + def create_option(self, key: Any, **kwargs) -> Option: + if not self.can_create_options: + raise MKComposeException("Cannot create options without a factory.") + + option = self.option_factory(key, **kwargs) + self.append(option) + return option + + def choose(self, key: Any) -> Optional[Option]: + if key not in self: + if self.can_create_options and self._ask_for_creating_option(key): + return self.create_option(key) + return None + + return self[key] + + + +class StringSelect(Select): + def __init__(self, **kwargs): + self._current_index = 0 + kwargs["option_factory"] = self.next_option + kwargs["parse_option_key"] = lambda x: unify(str(x)) + + super().__init__(**kwargs) + + def next_option(self, value: Any) -> Optional[Option]: + o = Option(value=value, keys=[self._current_index], text=f"{self._current_index:0>2}: {value}") + self._current_index += 1 + return o + + +class GenreSelect(StringSelect): + @staticmethod + def is_valid_genre(genre: Path) -> bool: + """ + gets the name of all subdirectories of shared.MUSIC_DIR, + but filters out all directories, where the name matches with any Patern + from shared.NOT_A_GENRE_REGEX. + """ + if not genre.is_dir(): + return False + + if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): + return False + + return True + + def __init__(self): + super().__init__(sort=True, raw_options=(genre.name for genre in filter(self.is_valid_genre, main_settings["music_directory"].iterdir()))) + -- 2.45.2 From 72190484227dfef83bd05b95a45c0e15b517cfc1 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 13:55:18 +0200 Subject: [PATCH 14/35] feat: colored asking for bool --- music_kraken/cli/main_downloader.py | 3 ++- music_kraken/cli/utils.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 9bc0b37..ddd1ffc 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -14,7 +14,7 @@ from ..utils.config import main_settings, write_config from ..utils.enums.colors import BColors from ..utils.exception import MKInvalidInputException from ..utils.exception.download import UrlNotFoundException -from ..utils.shared import URL_PATTERN +from ..utils.shared import HELP_MESSAGE, URL_PATTERN from ..utils.string_processing import fit_to_file_system from ..utils.support_classes.download_result import DownloadResult from ..utils.support_classes.query import Query @@ -43,6 +43,7 @@ def get_genre(): def help_message(): + print(HELP_MESSAGE) print() print(random.choice(main_settings["happy_messages"])) print() diff --git a/music_kraken/cli/utils.py b/music_kraken/cli/utils.py index cfcaadc..1acfe33 100644 --- a/music_kraken/cli/utils.py +++ b/music_kraken/cli/utils.py @@ -1,3 +1,4 @@ +from ..utils import BColors from ..utils.shared import get_random_message @@ -41,6 +42,6 @@ def print_cute_message(): AGREE_INPUTS = {"y", "yes", "ok"} def ask_for_bool(msg: str) -> bool: - i = input(msg + " (Y/N):").lower() + i = input(f"{msg} ({BColors.OKGREEN.value}Y{BColors.ENDC.value}/{BColors.FAIL.value}N{BColors.ENDC.value})? ").lower() return i in AGREE_INPUTS \ No newline at end of file -- 2.45.2 From 850c68f3e5652fccb85758d9b80be6573b2d1ad0 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 14:00:01 +0200 Subject: [PATCH 15/35] feat: implemented functionality to type out the choice --- music_kraken/download/components.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index e30bcb5..f11d41c 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -28,6 +28,8 @@ class Option: self._raw_keys = set(keys or []) self._raw_keys.add(self.text) + self._raw_keys.add(self.value) + self._raw_keys.add(str(self.value)) self.keys = set(self.parse_key(key) for key in self._raw_keys) def register_key(self, key: Any): @@ -100,10 +102,10 @@ class Select: yield option def __contains__(self, key: Any) -> bool: - return key in self._key_to_option + return self._parse_option_key(key) in self._key_to_option def __getitem__(self, key: Any) -> Option: - return self._key_to_option[key] + return self._key_to_option[self._parse_option_key(key)] def create_option(self, key: Any, **kwargs) -> Option: if not self.can_create_options: -- 2.45.2 From 71ec30995310d18cb4e6e1e0649e3a3f3b0c41b6 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 14:13:18 +0200 Subject: [PATCH 16/35] feat: reworked the genre select --- music_kraken/cli/main_downloader.py | 19 ++++++++++++++---- music_kraken/download/components.py | 30 ++++++++++++++++++++++++----- 2 files changed, 40 insertions(+), 9 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index ddd1ffc..d2c989a 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -27,17 +27,28 @@ PAGE_NAME_FILL = "-" MAX_PAGE_LEN = 21 +class GenreIO(components.HumanIO): + @staticmethod + def ask_to_create(option: components.Option) -> bool: + output() + return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") + + @staticmethod + def not_found(key: str) -> None: + output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) + + def get_genre(): select_genre = components.GenreSelect() - select_genre._ask_for_creating_option = lambda key: ask_for_bool(f"Create the genre \"{key}\"") + select_genre.human_io = GenreIO genre: Optional[components.Option] = None while genre is None: - for genre in select_genre: - print(genre) + print(select_genre.pprint()) + print() - genre = select_genre.choose(input("Id or new genre: ")) + genre = select_genre.choose(input("> ")) return genre.value diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index f11d41c..ef07565 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -1,12 +1,25 @@ +from __future__ import annotations + import re from pathlib import Path from typing import Any, Callable, Dict, Generator, List, Optional +from ..utils import BColors from ..utils.config import main_settings from ..utils.exception import MKComposeException from ..utils.string_processing import unify +class HumanIO: + @staticmethod + def ask_to_create(option: Option) -> bool: + return True + + @staticmethod + def not_found(key: Any) -> None: + return None + + class Option: """ This could represent a data object, a string or a page. @@ -56,12 +69,12 @@ class Select: option_factory: Callable[[Any], Option] = None, raw_options: List[Any] = None, parse_option_key: Callable[[Any], Any] = lambda x: x, - ask_for_creating_option: Callable[[Option], bool] = lambda x: True, + human_io: HumanIO = HumanIO, sort: bool = False, **kwargs ): self._parse_option_key: Callable[[Any], Any] = parse_option_key - self._ask_for_creating_option: Callable[[Option], bool] = ask_for_creating_option + self.human_io: HumanIO = human_io self._key_to_option: Dict[Any, Option] = dict() self._options: List[Option] = [] @@ -117,12 +130,19 @@ class Select: def choose(self, key: Any) -> Optional[Option]: if key not in self: - if self.can_create_options and self._ask_for_creating_option(key): - return self.create_option(key) + if self.can_create_options: + c = self.create_option(key) + if self.human_io.ask_to_create(c): + return c + + self.human_io.not_found(key) return None return self[key] + def pprint(self) -> str: + return "\n".join(str(option) for option in self) + class StringSelect(Select): @@ -134,7 +154,7 @@ class StringSelect(Select): super().__init__(**kwargs) def next_option(self, value: Any) -> Optional[Option]: - o = Option(value=value, keys=[self._current_index], text=f"{self._current_index:0>2}: {value}") + o = Option(value=value, keys=[self._current_index], text=f"{BColors.BOLD.value}{self._current_index: >2}{BColors.ENDC.value}: {value}") self._current_index += 1 return o -- 2.45.2 From 5cdd4fb6a9df0fc05053d19de453e506fb525d74 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 14:16:00 +0200 Subject: [PATCH 17/35] feat: renamed pages --- music_kraken/cli/main_downloader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index d2c989a..6b834ef 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -70,7 +70,7 @@ class CliDownloader: genre: str = None, process_metadata_anyway: bool = False, ) -> None: - self.pages: Downloader = Downloader(exclude_pages=exclude_pages, exclude_shady=exclude_shady) + self.downloader: Downloader = Downloader(exclude_pages=exclude_pages, exclude_shady=exclude_shady) self.page_dict: Dict[str, Type[Page]] = dict() @@ -159,7 +159,7 @@ class CliDownloader: def search(self, query: str): if re.match(URL_PATTERN, query) is not None: try: - page, data_object = self.pages.fetch_url(query) + data_object = self.downloader.fetch_url(query) except UrlNotFoundException as e: print(f"{e.url} could not be attributed/parsed to any yet implemented site.\n" f"PR appreciated if the site isn't implemented.\n" @@ -213,13 +213,13 @@ class CliDownloader: parsed_query: Query = self._process_parsed(key_text, query) - self.set_current_options(self.pages.search(parsed_query)) + self.set_current_options(self.downloader.search(parsed_query)) self.print_current_options() def goto(self, data_object: DatabaseObject): page: Type[Page] - self.pages.fetch_details(data_object, stop_at_level=1) + self.downloader.fetch_details(data_object, stop_at_level=1) self.set_current_options(GoToResults(data_object.options, max_items_per_page=self.max_displayed_options)) @@ -233,7 +233,7 @@ class CliDownloader: _result_map: Dict[DatabaseObject, DownloadResult] = dict() for database_object in data_objects: - r = self.pages.download( + r = self.downloader.download( data_object=database_object, genre=self.genre, **kwargs @@ -320,7 +320,7 @@ class CliDownloader: if do_fetch: for data_object in selected_objects: - self.pages.fetch_details(data_object) + self.downloader.fetch_details(data_object) self.print_current_options() return False -- 2.45.2 From a0e42fc6eeb2ba33cb17f16033fdab10ceb5b199 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 15:50:04 +0200 Subject: [PATCH 18/35] draft: outline of better select --- music_kraken/cli/main_downloader.py | 7 +++- music_kraken/download/__init__.py | 13 ++----- music_kraken/download/components.py | 60 ++++++++++++++++++++++++++++- music_kraken/utils/shared.py | 11 ++++-- 4 files changed, 75 insertions(+), 16 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 6b834ef..97ac247 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -91,7 +91,9 @@ class CliDownloader: self.page_dict = dict() print() + print(self.current_results.pprint()) + """ page_count = 0 for option in self.current_results.formatted_generator(): if isinstance(option, ResultOption): @@ -106,10 +108,13 @@ class CliDownloader: self.page_dict[option.__name__] = option page_count += 1 + """ print() - def set_current_options(self, current_options: Results): + def set_current_options(self, current_options: Generator[DatabaseObject, None, None]): + current_options = components.DataObjectSelect(current_options) + if main_settings["result_history"]: self._result_history.append(current_options) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 1a4aa6d..7484eef 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -117,21 +117,14 @@ class Downloader: def get_pages(self, *page_types: List[Type[Page]]) -> Generator[Page, None, None]: if len(page_types) == 0: - page_types = _registered_pages.keys() + page_types = self._registered_pages.keys() for page_type in page_types: yield from self._registered_pages[page_type] - def search(self, query: Query) -> SearchResults: - result = SearchResults() - + def search(self, query: Query) -> Generator[DataObject, None, None]: for page in self.get_pages(): - result.add( - page=type(page), - search_result=page.search(query=query) - ) - - return result + yield from page.search(query=query) def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: source: Source diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index ef07565..c3763e6 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -1,12 +1,16 @@ from __future__ import annotations import re +from collections import defaultdict from pathlib import Path from typing import Any, Callable, Dict, Generator, List, Optional +from ..objects import OuterProxy as DataObject from ..utils import BColors from ..utils.config import main_settings +from ..utils.enums import SourceType from ..utils.exception import MKComposeException +from ..utils.shared import ALPHABET from ..utils.string_processing import unify @@ -41,7 +45,10 @@ class Option: self._raw_keys = set(keys or []) self._raw_keys.add(self.text) - self._raw_keys.add(self.value) + try: + self._raw_keys.add(self.value) + except TypeError: + pass self._raw_keys.add(str(self.value)) self.keys = set(self.parse_key(key) for key in self._raw_keys) @@ -178,3 +185,54 @@ class GenreSelect(StringSelect): def __init__(self): super().__init__(sort=True, raw_options=(genre.name for genre in filter(self.is_valid_genre, main_settings["music_directory"].iterdir()))) + +class DataObjectSelect(Select): + def __init__(self, data_objects: Generator[DataObject]): + self._source_type_to_data_objects: Dict[SourceType, List[Option]] = defaultdict(list) + + self._data_object_index: int = 0 + self._source_type_index: int = 0 + + super().__init__() + + self.extend(data_objects) + + def option_from_data_object(self, data_object: DataObject) -> Option: + index = self._data_object_index + self._data_object_index += 1 + return Option( + value=data_object, + keys=[index, data_object.option_string, data_object.title_string], + text=f"{BColors.BOLD.value}{index: >2}{BColors.ENDC.value}: {data_object.option_string}", + ) + + def option_from_source_type(self, source_type: SourceType) -> Option: + index = ALPHABET[self._source_type_index % len(ALPHABET)] + self._source_type_index += 1 + + return Option( + value=self._source_type_to_data_objects[source_type], + keys=[index, source_type], + text=f"{BColors.HEADER.value}({index}) --------------------------------{source_type.name:{'-'}<{21}}--------------------{BColors.ENDC.value}", + ) + + def append(self, option: Union[Option, DataObject]): + if isinstance(option, DataObject): + data_object = option + option = self.option_from_data_object(data_object) + else: + data_object = option.value + + for source_type in data_object.source_collection.source_types(only_with_page=True): + if source_type not in self._source_type_to_data_objects: + st_option = self.option_from_source_type(source_type) + self._source_type_to_data_objects[source_type].append(st_option) + super().append(st_option) + + self._source_type_to_data_objects[source_type].append(option) + + super().append(option) + + def __iter__(self): + for options in self._source_type_to_data_objects.values(): + yield from options \ No newline at end of file diff --git a/music_kraken/utils/shared.py b/music_kraken/utils/shared.py index b75cf7f..52613af 100644 --- a/music_kraken/utils/shared.py +++ b/music_kraken/utils/shared.py @@ -1,11 +1,11 @@ -import random -from dotenv import load_dotenv -from pathlib import Path import os +import random +from pathlib import Path +from dotenv import load_dotenv -from .path_manager import LOCATIONS from .config import main_settings +from .path_manager import LOCATIONS if not load_dotenv(Path(__file__).parent.parent.parent / ".env"): load_dotenv(Path(__file__).parent.parent.parent / ".env.example") @@ -51,3 +51,6 @@ have fun :3""".strip() URL_PATTERN = r"https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+" INT_PATTERN = r"^\d*$" FLOAT_PATTERN = r"^[\d|\,|\.]*$" + + +ALPHABET = "abcdefghijklmnopqrstuvwxyz" -- 2.45.2 From 999299c32a65f4cf70094a6d2bc21592ee5ade24 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 16:37:55 +0200 Subject: [PATCH 19/35] draft: improved searching one page only --- music_kraken/cli/main_downloader.py | 37 +++++++++------------ music_kraken/download/components.py | 50 ++++++++++++++++++++++------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 97ac247..6d11fa1 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -1,7 +1,7 @@ import random import re from pathlib import Path -from typing import Dict, Generator, List, Set, Type +from typing import Dict, Generator, List, Set, Type, Union from .. import console from ..download import Downloader, Page, components @@ -112,8 +112,8 @@ class CliDownloader: print() - def set_current_options(self, current_options: Generator[DatabaseObject, None, None]): - current_options = components.DataObjectSelect(current_options) + def set_current_options(self, current_options: Union[Generator[DatabaseObject, None, None], components.Select]): + current_options = current_options if isinstance(current_options, components.Select) else components.DataObjectSelect(current_options) if main_settings["result_history"]: self._result_history.append(current_options) @@ -221,12 +221,14 @@ class CliDownloader: self.set_current_options(self.downloader.search(parsed_query)) self.print_current_options() - def goto(self, data_object: DatabaseObject): + def goto(self, data_object: Union[DatabaseObject, components.Select]): page: Type[Page] - self.downloader.fetch_details(data_object, stop_at_level=1) - - self.set_current_options(GoToResults(data_object.options, max_items_per_page=self.max_displayed_options)) + if isinstance(data_object, components.Select): + self.set_current_options(data_object) + else: + self.downloader.fetch_details(data_object, stop_at_level=1) + self.set_current_options(data_object.options) self.print_current_options() @@ -293,24 +295,15 @@ class CliDownloader: indices = [] for possible_index in q.split(","): - possible_index = possible_index.strip() if possible_index == "": continue + + if possible_index not in self.current_results: + raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not in the current options.") - i = 0 - try: - i = int(possible_index) - except ValueError: - raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not a number.") + yield self.current_results[possible_index] - if i < 0 or i >= len(self.current_results): - raise MKInvalidInputException(message=f"The index \"{i}\" is not within the bounds of 0-{len(self.current_results) - 1}.") - - indices.append(i) - - return [self.current_results[i] for i in indices] - - selected_objects = get_selected_objects(query) + selected_objects = list(get_selected_objects(query)) if do_merge: old_selected_objects = selected_objects @@ -337,7 +330,7 @@ class CliDownloader: if len(selected_objects) != 1: raise MKInvalidInputException(message="You can only go to one object at a time without merging.") - self.goto(selected_objects[0]) + self.goto(selected_objects[0].value) return False except MKInvalidInputException as e: output("\n" + e.message + "\n", color=BColors.FAIL) diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index c3763e6..e170e60 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -125,7 +125,11 @@ class Select: return self._parse_option_key(key) in self._key_to_option def __getitem__(self, key: Any) -> Option: - return self._key_to_option[self._parse_option_key(key)] + r = self._key_to_option[self._parse_option_key(key)] + # check if is callable + if callable(r.value): + r.value = r.value() + return r def create_option(self, key: Any, **kwargs) -> Option: if not self.can_create_options: @@ -186,20 +190,36 @@ class GenreSelect(StringSelect): super().__init__(sort=True, raw_options=(genre.name for genre in filter(self.is_valid_genre, main_settings["music_directory"].iterdir()))) + +class SourceTypeToOption(dict): + def __init__(self, callback): + super().__init__() + + self.callback = callback + + def __missing__(self, key): + self[key] = self.callback(key) + return self[key] + + class DataObjectSelect(Select): def __init__(self, data_objects: Generator[DataObject]): self._source_type_to_data_objects: Dict[SourceType, List[Option]] = defaultdict(list) + self._source_type_to_option: Dict[SourceType, Option] = SourceTypeToOption(self.option_from_source_type) self._data_object_index: int = 0 self._source_type_index: int = 0 - super().__init__() + super().__init__( + parse_option_key=lambda x: unify(str(x)), + ) self.extend(data_objects) def option_from_data_object(self, data_object: DataObject) -> Option: index = self._data_object_index self._data_object_index += 1 + return Option( value=data_object, keys=[index, data_object.option_string, data_object.title_string], @@ -210,12 +230,16 @@ class DataObjectSelect(Select): index = ALPHABET[self._source_type_index % len(ALPHABET)] self._source_type_index += 1 - return Option( - value=self._source_type_to_data_objects[source_type], + o = Option( + value=lambda: DataObjectSelect(self._source_type_to_data_objects[source_type]), keys=[index, source_type], text=f"{BColors.HEADER.value}({index}) --------------------------------{source_type.name:{'-'}<{21}}--------------------{BColors.ENDC.value}", ) + super().append(o) + + return o + def append(self, option: Union[Option, DataObject]): if isinstance(option, DataObject): data_object = option @@ -224,15 +248,19 @@ class DataObjectSelect(Select): data_object = option.value for source_type in data_object.source_collection.source_types(only_with_page=True): - if source_type not in self._source_type_to_data_objects: - st_option = self.option_from_source_type(source_type) - self._source_type_to_data_objects[source_type].append(st_option) - super().append(st_option) - self._source_type_to_data_objects[source_type].append(option) super().append(option) def __iter__(self): - for options in self._source_type_to_data_objects.values(): - yield from options \ No newline at end of file + source_types = list(sorted(self._source_type_to_data_objects.keys(), key=lambda x: x.name)) + has_limit = len(source_types) > 1 + + for st in source_types: + if has_limit: + yield self._source_type_to_option[st] + + limit = min(15, len(self._source_type_to_data_objects[st])) if has_limit else len(self._source_type_to_data_objects[st]) + + for i in range(limit): + yield self._source_type_to_data_objects[st][i] \ No newline at end of file -- 2.45.2 From 413d422e2fdc4630c06d162ecbcb6cd7e414deff Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 16:38:03 +0200 Subject: [PATCH 20/35] draft: improved searching one page only --- music_kraken/download/components.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index e170e60..5aefdab 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -254,13 +254,13 @@ class DataObjectSelect(Select): def __iter__(self): source_types = list(sorted(self._source_type_to_data_objects.keys(), key=lambda x: x.name)) - has_limit = len(source_types) > 1 + single_source = len(source_types) > 1 for st in source_types: - if has_limit: + if single_source: yield self._source_type_to_option[st] - limit = min(15, len(self._source_type_to_data_objects[st])) if has_limit else len(self._source_type_to_data_objects[st]) + limit = min(15, len(self._source_type_to_data_objects[st])) if single_source else len(self._source_type_to_data_objects[st]) for i in range(limit): yield self._source_type_to_data_objects[st][i] \ No newline at end of file -- 2.45.2 From d4fe99ffc77edb92789a729361c37faf44c6e0a1 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 27 May 2024 16:59:51 +0200 Subject: [PATCH 21/35] feat: dynamic indices --- music_kraken/download/components.py | 45 ++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py index 5aefdab..413f794 100644 --- a/music_kraken/download/components.py +++ b/music_kraken/download/components.py @@ -36,11 +36,13 @@ class Option: keys: List[Any] = None, hidden: bool = False, parse_key: Callable[[Any], Any] = lambda x: x, + index: int = None, ): self._parse_key: Callable[[Any], Any] = parse_key + self._index = index self.value = value - self.text = text or str(value) + self._text = text or str(value) self.hidden = hidden self._raw_keys = set(keys or []) @@ -50,7 +52,28 @@ class Option: except TypeError: pass self._raw_keys.add(str(self.value)) + self._raw_keys.add(self._index) self.keys = set(self.parse_key(key) for key in self._raw_keys) + + @property + def text(self) -> str: + return self._text.replace("{index}", str(self.index)) + + @text.setter + def text(self, value: str): + self._text = value + + @property + def index(self) -> int: + return self._index + + @index.setter + def index(self, value: int): + p = self._parse_key(self._index) + if p in self.keys: + self.keys.remove(p) + self._index = value + self.keys.add(p) def register_key(self, key: Any): self._raw_keys.add(key) @@ -110,6 +133,12 @@ class Select: for key in option.keys: self._key_to_option[key] = option + def _remap(self): + self._key_to_option = dict() + for option in self._options: + for key in option.keys: + self._key_to_option[key] = option + def extend(self, options: List[Option]): for option in options: self.append(option) @@ -126,7 +155,8 @@ class Select: def __getitem__(self, key: Any) -> Option: r = self._key_to_option[self._parse_option_key(key)] - # check if is callable + if callable(r): + r = r() if callable(r.value): r.value = r.value() return r @@ -223,7 +253,8 @@ class DataObjectSelect(Select): return Option( value=data_object, keys=[index, data_object.option_string, data_object.title_string], - text=f"{BColors.BOLD.value}{index: >2}{BColors.ENDC.value}: {data_object.option_string}", + text=f"{BColors.BOLD.value}{{index}}{BColors.ENDC.value}: {data_object.option_string}", + index=index, ) def option_from_source_type(self, source_type: SourceType) -> Option: @@ -256,6 +287,7 @@ class DataObjectSelect(Select): source_types = list(sorted(self._source_type_to_data_objects.keys(), key=lambda x: x.name)) single_source = len(source_types) > 1 + j = 0 for st in source_types: if single_source: yield self._source_type_to_option[st] @@ -263,4 +295,9 @@ class DataObjectSelect(Select): limit = min(15, len(self._source_type_to_data_objects[st])) if single_source else len(self._source_type_to_data_objects[st]) for i in range(limit): - yield self._source_type_to_data_objects[st][i] \ No newline at end of file + o = self._source_type_to_data_objects[st][i] + o.index = j + yield o + j += 1 + + self._remap() \ No newline at end of file -- 2.45.2 From 4b2dd4a36a60dc9e0aa06f2d84df06cd6c91a4e1 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 28 May 2024 13:45:08 +0200 Subject: [PATCH 22/35] feat: launch programm --- .vscode/launch.json | 2 +- music_kraken/cli/main_downloader.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index f864249..b245b1d 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -22,7 +22,7 @@ "name": "Python Debugger: Music Kraken", "type": "debugpy", "request": "launch", // run the module - "module": "music_kraken", + "program": "${workspaceFolder}/.vscode/run_script.py", } ] } \ No newline at end of file diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 6d11fa1..12d076e 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -88,12 +88,13 @@ class CliDownloader: output() def print_current_options(self): - self.page_dict = dict() print() print(self.current_results.pprint()) """ + self.page_dict = dict() + page_count = 0 for option in self.current_results.formatted_generator(): if isinstance(option, ResultOption): @@ -324,7 +325,7 @@ class CliDownloader: return False if do_download: - self.download(selected_objects) + self.download(list(o.value for o in selected_objects)) return False if len(selected_objects) != 1: -- 2.45.2 From ead4f834560ef06f792414fefdf060c3cc26255e Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 28 May 2024 13:45:20 +0200 Subject: [PATCH 23/35] feat: launch programm --- .vscode/run_script.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/run_script.py diff --git a/.vscode/run_script.py b/.vscode/run_script.py new file mode 100644 index 0000000..1f7a1b9 --- /dev/null +++ b/.vscode/run_script.py @@ -0,0 +1,3 @@ +from music_kraken.__main__ import cli + +cli() -- 2.45.2 From df1743c695a5786c04483fe770c7439d1c45fe9d Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 3 Jun 2024 15:04:47 +0200 Subject: [PATCH 24/35] feat: implemented better components --- music_kraken/__init__.py | 6 +- music_kraken/cli/genre.py | 45 +++ music_kraken/cli/main_downloader.py | 24 +- music_kraken/download/components.py | 303 ------------------- music_kraken/download/components/__init__.py | 158 ++++++++++ music_kraken/download/components/generic.py | 1 + music_kraken/objects/song.py | 41 ++- 7 files changed, 227 insertions(+), 351 deletions(-) create mode 100644 music_kraken/cli/genre.py delete mode 100644 music_kraken/download/components.py create mode 100644 music_kraken/download/components/__init__.py create mode 100644 music_kraken/download/components/generic.py diff --git a/music_kraken/__init__.py b/music_kraken/__init__.py index 5176b38..e9dbc41 100644 --- a/music_kraken/__init__.py +++ b/music_kraken/__init__.py @@ -1,13 +1,13 @@ -import logging import gc +import logging import sys from pathlib import Path -from rich.logging import RichHandler from rich.console import Console +from rich.logging import RichHandler -from .utils.shared import DEBUG, DEBUG_LOGGING from .utils.config import logging_settings, main_settings, read_config +from .utils.shared import DEBUG, DEBUG_LOGGING read_config() diff --git a/music_kraken/cli/genre.py b/music_kraken/cli/genre.py new file mode 100644 index 0000000..78b67ef --- /dev/null +++ b/music_kraken/cli/genre.py @@ -0,0 +1,45 @@ +class GenreIO(components.HumanIO): + @staticmethod + def ask_to_create(option: components.Option) -> bool: + output() + return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") + + @staticmethod + def not_found(key: str) -> None: + output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) + + +def _genre_generator() -> Generator[str, None, None]: + def is_valid_genre(genre: Path) -> bool: + """ + gets the name of all subdirectories of shared.MUSIC_DIR, + but filters out all directories, where the name matches with any Patern + from shared.NOT_A_GENRE_REGEX. + """ + if not genre.is_dir(): + return False + + if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): + return False + + return True + + for genre in filter(is_valid_genre, main_settings["music_directory"].iterdir()): + yield genre.name + +def get_genre() -> str: + select_genre = components.Select( + human_io=GenreIO, + can_create_options=True, + data=_genre_generator(), + ) + genre: Optional[components.Option[str]] = None + + while genre is None: + print(select_genre.pprint()) + print() + + genre = select_genre.choose(input("> ")) + + return genre.value + \ No newline at end of file diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py index 12d076e..eb01c1a 100644 --- a/music_kraken/cli/main_downloader.py +++ b/music_kraken/cli/main_downloader.py @@ -18,6 +18,7 @@ from ..utils.shared import HELP_MESSAGE, URL_PATTERN from ..utils.string_processing import fit_to_file_system from ..utils.support_classes.download_result import DownloadResult from ..utils.support_classes.query import Query +from .genre import get_genre from .options.first_config import initial_config from .utils import ask_for_bool, cli_function @@ -27,30 +28,7 @@ PAGE_NAME_FILL = "-" MAX_PAGE_LEN = 21 -class GenreIO(components.HumanIO): - @staticmethod - def ask_to_create(option: components.Option) -> bool: - output() - return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") - @staticmethod - def not_found(key: str) -> None: - output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) - - -def get_genre(): - select_genre = components.GenreSelect() - select_genre.human_io = GenreIO - - genre: Optional[components.Option] = None - - while genre is None: - print(select_genre.pprint()) - print() - - genre = select_genre.choose(input("> ")) - - return genre.value def help_message(): diff --git a/music_kraken/download/components.py b/music_kraken/download/components.py deleted file mode 100644 index 413f794..0000000 --- a/music_kraken/download/components.py +++ /dev/null @@ -1,303 +0,0 @@ -from __future__ import annotations - -import re -from collections import defaultdict -from pathlib import Path -from typing import Any, Callable, Dict, Generator, List, Optional - -from ..objects import OuterProxy as DataObject -from ..utils import BColors -from ..utils.config import main_settings -from ..utils.enums import SourceType -from ..utils.exception import MKComposeException -from ..utils.shared import ALPHABET -from ..utils.string_processing import unify - - -class HumanIO: - @staticmethod - def ask_to_create(option: Option) -> bool: - return True - - @staticmethod - def not_found(key: Any) -> None: - return None - - -class Option: - """ - This could represent a data object, a string or a page. - """ - - def __init__( - self, - value: Any, - text: Optional[str] = None, - keys: List[Any] = None, - hidden: bool = False, - parse_key: Callable[[Any], Any] = lambda x: x, - index: int = None, - ): - self._parse_key: Callable[[Any], Any] = parse_key - - self._index = index - self.value = value - self._text = text or str(value) - self.hidden = hidden - - self._raw_keys = set(keys or []) - self._raw_keys.add(self.text) - try: - self._raw_keys.add(self.value) - except TypeError: - pass - self._raw_keys.add(str(self.value)) - self._raw_keys.add(self._index) - self.keys = set(self.parse_key(key) for key in self._raw_keys) - - @property - def text(self) -> str: - return self._text.replace("{index}", str(self.index)) - - @text.setter - def text(self, value: str): - self._text = value - - @property - def index(self) -> int: - return self._index - - @index.setter - def index(self, value: int): - p = self._parse_key(self._index) - if p in self.keys: - self.keys.remove(p) - self._index = value - self.keys.add(p) - - def register_key(self, key: Any): - self._raw_keys.add(key) - self.keys.add(self._parse_key(key)) - - @property - def parse_key(self) -> Callable[[Any], Any]: - return self._parse_key - - @parse_key.setter - def parse_key(self, value: Callable[[Any], Any]): - self._parse_key = value - self.keys = set(self._parse_key(key) for key in self._raw_keys) - - def __str__(self): - return self.text - - -class Select: - def __init__( - self, - options: Generator[Option, None, None] = None, - option_factory: Callable[[Any], Option] = None, - raw_options: List[Any] = None, - parse_option_key: Callable[[Any], Any] = lambda x: x, - human_io: HumanIO = HumanIO, - sort: bool = False, - **kwargs - ): - self._parse_option_key: Callable[[Any], Any] = parse_option_key - self.human_io: HumanIO = human_io - - self._key_to_option: Dict[Any, Option] = dict() - self._options: List[Option] = [] - - options = options or [] - self.option_factory: Optional[Callable[[Any], Option]] = option_factory - if self.can_create_options: - _raw_options = raw_options or [] - if sort: - _raw_options = sorted(_raw_options) - - for raw_option in _raw_options: - self.append(self.option_factory(raw_option)) - elif raw_options is not None: - raise MKComposeException("Cannot create options without a factory.") - - self.extend(options) - - @property - def can_create_options(self) -> bool: - return self.option_factory is not None - - def append(self, option: Option): - option.parse_key = self._parse_option_key - self._options.append(option) - for key in option.keys: - self._key_to_option[key] = option - - def _remap(self): - self._key_to_option = dict() - for option in self._options: - for key in option.keys: - self._key_to_option[key] = option - - def extend(self, options: List[Option]): - for option in options: - self.append(option) - - def __iter__(self) -> Generator[Option, None, None]: - for option in self._options: - if option.hidden: - continue - - yield option - - def __contains__(self, key: Any) -> bool: - return self._parse_option_key(key) in self._key_to_option - - def __getitem__(self, key: Any) -> Option: - r = self._key_to_option[self._parse_option_key(key)] - if callable(r): - r = r() - if callable(r.value): - r.value = r.value() - return r - - def create_option(self, key: Any, **kwargs) -> Option: - if not self.can_create_options: - raise MKComposeException("Cannot create options without a factory.") - - option = self.option_factory(key, **kwargs) - self.append(option) - return option - - def choose(self, key: Any) -> Optional[Option]: - if key not in self: - if self.can_create_options: - c = self.create_option(key) - if self.human_io.ask_to_create(c): - return c - - self.human_io.not_found(key) - return None - - return self[key] - - def pprint(self) -> str: - return "\n".join(str(option) for option in self) - - - -class StringSelect(Select): - def __init__(self, **kwargs): - self._current_index = 0 - kwargs["option_factory"] = self.next_option - kwargs["parse_option_key"] = lambda x: unify(str(x)) - - super().__init__(**kwargs) - - def next_option(self, value: Any) -> Optional[Option]: - o = Option(value=value, keys=[self._current_index], text=f"{BColors.BOLD.value}{self._current_index: >2}{BColors.ENDC.value}: {value}") - self._current_index += 1 - return o - - -class GenreSelect(StringSelect): - @staticmethod - def is_valid_genre(genre: Path) -> bool: - """ - gets the name of all subdirectories of shared.MUSIC_DIR, - but filters out all directories, where the name matches with any Patern - from shared.NOT_A_GENRE_REGEX. - """ - if not genre.is_dir(): - return False - - if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): - return False - - return True - - def __init__(self): - super().__init__(sort=True, raw_options=(genre.name for genre in filter(self.is_valid_genre, main_settings["music_directory"].iterdir()))) - - - -class SourceTypeToOption(dict): - def __init__(self, callback): - super().__init__() - - self.callback = callback - - def __missing__(self, key): - self[key] = self.callback(key) - return self[key] - - -class DataObjectSelect(Select): - def __init__(self, data_objects: Generator[DataObject]): - self._source_type_to_data_objects: Dict[SourceType, List[Option]] = defaultdict(list) - self._source_type_to_option: Dict[SourceType, Option] = SourceTypeToOption(self.option_from_source_type) - - self._data_object_index: int = 0 - self._source_type_index: int = 0 - - super().__init__( - parse_option_key=lambda x: unify(str(x)), - ) - - self.extend(data_objects) - - def option_from_data_object(self, data_object: DataObject) -> Option: - index = self._data_object_index - self._data_object_index += 1 - - return Option( - value=data_object, - keys=[index, data_object.option_string, data_object.title_string], - text=f"{BColors.BOLD.value}{{index}}{BColors.ENDC.value}: {data_object.option_string}", - index=index, - ) - - def option_from_source_type(self, source_type: SourceType) -> Option: - index = ALPHABET[self._source_type_index % len(ALPHABET)] - self._source_type_index += 1 - - o = Option( - value=lambda: DataObjectSelect(self._source_type_to_data_objects[source_type]), - keys=[index, source_type], - text=f"{BColors.HEADER.value}({index}) --------------------------------{source_type.name:{'-'}<{21}}--------------------{BColors.ENDC.value}", - ) - - super().append(o) - - return o - - def append(self, option: Union[Option, DataObject]): - if isinstance(option, DataObject): - data_object = option - option = self.option_from_data_object(data_object) - else: - data_object = option.value - - for source_type in data_object.source_collection.source_types(only_with_page=True): - self._source_type_to_data_objects[source_type].append(option) - - super().append(option) - - def __iter__(self): - source_types = list(sorted(self._source_type_to_data_objects.keys(), key=lambda x: x.name)) - single_source = len(source_types) > 1 - - j = 0 - for st in source_types: - if single_source: - yield self._source_type_to_option[st] - - limit = min(15, len(self._source_type_to_data_objects[st])) if single_source else len(self._source_type_to_data_objects[st]) - - for i in range(limit): - o = self._source_type_to_data_objects[st][i] - o.index = j - yield o - j += 1 - - self._remap() \ No newline at end of file diff --git a/music_kraken/download/components/__init__.py b/music_kraken/download/components/__init__.py new file mode 100644 index 0000000..19d5cda --- /dev/null +++ b/music_kraken/download/components/__init__.py @@ -0,0 +1,158 @@ +from __future__ import annotations + +import re +from collections import defaultdict +from pathlib import Path +from typing import (Any, Callable, Dict, Generator, Generic, Hashable, List, + Optional, Tuple, TypeVar, Union) + +from ...objects import OuterProxy as DataObject +from ...utils import BColors +from ...utils.config import main_settings +from ...utils.enums import SourceType +from ...utils.exception import MKComposeException +from ...utils.shared import ALPHABET +from ...utils.string_processing import unify + +P = TypeVar('P') + + +class HumanIO: + @staticmethod + def ask_to_create(option: Option) -> bool: + return True + + @staticmethod + def not_found(key: Any) -> None: + return None + + +class Option(Generic[P]): + """ + This could represent a data object, a string or a page. + """ + TEXT_TEMPLATE: str = f"{BColors.BOLD.value}{{index}}{BColors.ENDC.value}: {{value}}" + ATTRIBUTES_FORMATTING: Tuple[str, ...] = ("index", "value") + ATTRIBUTES_KEY: Tuple[str, ...] = ("index", ) + + def __init__( + self, + value: P, + hidden: bool = False, + additional_keys: List[Hashable] = None, + **kwargs + ): + self.value = value + self.hidden = hidden + self._additional_keys = set(self._to_hash(key) for key in additional_keys or []) + + for key, value in kwargs.items(): + setattr(self, key, value) + + def _to_option_string(self, value: Any) -> str: + if hasattr(value, "option_string"): + return value.option_string + + return str(value) + + @property + def text(self) -> str: + text = self.TEXT_TEMPLATE + + for attribute_key in self.ATTRIBUTES_FORMATTING: + text = text.replace(f"{{{attribute_key}}}", self._to_option_string(getattr(self, attribute_key))) + + return text + + def _to_hash(self, key: Any) -> int: + try: + key = int(key) + except ValueError: + pass + + if isinstance(key, str): + return hash(unify(key)) + + return hash(key) + + @property + def keys(self) -> set: + keys = self._additional_keys.copy() + + for key in self.ATTRIBUTES_KEY: + keys.add(self._to_hash(getattr(self, key))) + + def __contains__(self, key: Any) -> bool: + return self._to_hash(key) in self.keys + + def __str__(self): + return self.text + + +class Select(Generic[P]): + OPTION: Type[Option[P]] = Option + HUMAN_IO: Type[HumanIO] = HumanIO + CAN_CREATE_OPTIONS: bool = False + + def __init__( + self, + data: Generator[P, None, None], + **kwargs + ): + self.option: Type[Option[P]] = kwargs.get("option", self.OPTION) + self.human_io: Type[HumanIO] = kwargs.get("human_io", self.HUMAN_IO) + self.can_create_options: bool = kwargs.get("can_create_options", self.CAN_CREATE_OPTIONS) + + self._options: List[Option[P]] = [] + self.extend(data) + + def append(self, value: P) -> Option[P]: + option = self.option(value) + self._options.append(option) + return option + + def extend(self, values: Generator[P, None, None]): + for value in values: + self.append(value) + + @property + def _options_to_show(self) -> Generator[Option[P], None, None]: + for option in self._options: + if option.hidden: + continue + + yield option + + def __iter__(self) -> Generator[Option, None, None]: + _index = 0 + + for i, option in enumerate(self._options_to_show): + option.index = _index + yield option + _index += 1 + + def __contains__(self, key: Any) -> bool: + for option in self._options: + if key in option: + return True + + return False + + def __getitem__(self, key: Any) -> Option[P]: + for option in self._options: + if key in option: + return option + + raise KeyError(key) + + def choose(self, key: Any) -> Optional[Option[P]]: + try: + return self[key] + except KeyError: + if self.can_create_options: + return self.append(key) + + self.human_io.not_found(key) + + def pprint(self) -> str: + return "\n".join(str(option) for option in self) diff --git a/music_kraken/download/components/generic.py b/music_kraken/download/components/generic.py new file mode 100644 index 0000000..be88017 --- /dev/null +++ b/music_kraken/download/components/generic.py @@ -0,0 +1 @@ +from . import Option, Select diff --git a/music_kraken/objects/song.py b/music_kraken/objects/song.py index 980bc08..a25252e 100644 --- a/music_kraken/objects/song.py +++ b/music_kraken/objects/song.py @@ -1,35 +1,32 @@ from __future__ import annotations +import copy import random from collections import defaultdict -from typing import List, Optional, Dict, Tuple, Type, Union -import copy +from typing import Dict, List, Optional, Tuple, Type, Union import pycountry -from ..utils.enums.album import AlbumType, AlbumStatus -from .collection import Collection -from .formatted_text import FormattedText -from .lyrics import Lyrics -from .contact import Contact -from .artwork import Artwork -from .metadata import ( - Mapping as id3Mapping, - ID3Timestamp, - Metadata -) -from .option import Options -from .parents import OuterProxy, P -from .source import Source, SourceCollection -from .target import Target -from .country import Language, Country +from ..utils.config import main_settings +from ..utils.enums.album import AlbumStatus, AlbumType +from ..utils.enums.colors import BColors from ..utils.shared import DEBUG_PRINT_ID from ..utils.string_processing import unify - +from .artwork import Artwork +from .collection import Collection +from .contact import Contact +from .country import Country, Language +from .formatted_text import FormattedText +from .lyrics import Lyrics +from .metadata import ID3Timestamp +from .metadata import Mapping as id3Mapping +from .metadata import Metadata +from .option import Options +from .parents import OuterProxy from .parents import OuterProxy as Base - -from ..utils.config import main_settings -from ..utils.enums.colors import BColors +from .parents import P +from .source import Source, SourceCollection +from .target import Target """ All Objects dependent -- 2.45.2 From 636645e86295470e5cd72f29ef151587174147e1 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Tue, 4 Jun 2024 11:00:12 +0200 Subject: [PATCH 25/35] feat: implemented genre to external file --- music_kraken/cli/genre.py | 10 ++++++- music_kraken/download/components/__init__.py | 31 +++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/music_kraken/cli/genre.py b/music_kraken/cli/genre.py index 78b67ef..3580495 100644 --- a/music_kraken/cli/genre.py +++ b/music_kraken/cli/genre.py @@ -1,3 +1,12 @@ +import re +from pathlib import Path +from typing import Dict, Generator, List, Set, Type, Union + +from ..download import Downloader, Page, components +from ..utils.config import main_settings +from .utils import ask_for_bool, cli_function + + class GenreIO(components.HumanIO): @staticmethod def ask_to_create(option: components.Option) -> bool: @@ -42,4 +51,3 @@ def get_genre() -> str: genre = select_genre.choose(input("> ")) return genre.value - \ No newline at end of file diff --git a/music_kraken/download/components/__init__.py b/music_kraken/download/components/__init__.py index 19d5cda..bd93939 100644 --- a/music_kraken/download/components/__init__.py +++ b/music_kraken/download/components/__init__.py @@ -49,6 +49,8 @@ class Option(Generic[P]): for key, value in kwargs.items(): setattr(self, key, value) + super(Option, self).__init__() + def _to_option_string(self, value: Any) -> str: if hasattr(value, "option_string"): return value.option_string @@ -88,6 +90,9 @@ class Option(Generic[P]): def __str__(self): return self.text + def __iter__(self) -> Generator[Option[P], None, None]: + yield self + class Select(Generic[P]): OPTION: Type[Option[P]] = Option @@ -106,6 +111,8 @@ class Select(Generic[P]): self._options: List[Option[P]] = [] self.extend(data) + super(Select, self).__init__(**kwargs) + def append(self, value: P) -> Option[P]: option = self.option(value) self._options.append(option) @@ -126,10 +133,11 @@ class Select(Generic[P]): def __iter__(self) -> Generator[Option, None, None]: _index = 0 - for i, option in enumerate(self._options_to_show): - option.index = _index - yield option - _index += 1 + for i, o in enumerate(self._options_to_show): + for option in o: + option.index = _index + yield option + _index += 1 def __contains__(self, key: Any) -> bool: for option in self._options: @@ -156,3 +164,18 @@ class Select(Generic[P]): def pprint(self) -> str: return "\n".join(str(option) for option in self) + + +class OptionGroup(Option[P], Select[P]): + ALPHABET: str = "abcdefghijklmnopqrstuvwxyz" + ATTRIBUTES_FORMATTING: Tuple[str, ...] = ("alphabetic_index", "value") + + TEXT_TEMPLATE: str = f"{BColors.HEADER.value}{{alphabetic_index}}) {{value}}{BColors.ENDC.value}" + + @property + def alphabetic_index(self) -> str: + return self.ALPHABET[self.index % len(self.ALPHABET)] + + def __init__(self, value: P, data: Generator[P, None, None] **kwargs): + super(OptionGroup, self).__init__(value=value, data=data, **kwargs) + -- 2.45.2 From 130f5edcfe6b5d23b1624836edbba65793808b9c Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 5 Jun 2024 12:02:12 +0200 Subject: [PATCH 26/35] draft: rewrite of interface --- music_kraken/audio/metadata.py | 15 ++--- music_kraken/download/components/__init__.py | 64 +++++++++++++++++--- music_kraken/utils/__init__.py | 13 ++-- music_kraken/utils/string_processing.py | 22 +++++-- 4 files changed, 85 insertions(+), 29 deletions(-) diff --git a/music_kraken/audio/metadata.py b/music_kraken/audio/metadata.py index bceb775..1ec09f5 100644 --- a/music_kraken/audio/metadata.py +++ b/music_kraken/audio/metadata.py @@ -1,14 +1,15 @@ -import mutagen -from mutagen.id3 import ID3, Frame, APIC, USLT +import logging from pathlib import Path from typing import List -import logging + +import mutagen +from mutagen.id3 import APIC, ID3, USLT, Frame from PIL import Image -from ..utils.config import logging_settings, main_settings -from ..objects import Song, Target, Metadata -from ..objects.metadata import Mapping from ..connection import Connection +from ..objects import Metadata, Song, Target +from ..objects.metadata import Mapping +from ..utils.config import logging_settings, main_settings LOGGER = logging_settings["tagging_logger"] @@ -105,7 +106,7 @@ def write_metadata_to_target(metadata: Metadata, target: Target, song: Song): APIC( encoding=0, mime="image/jpeg", - type=3, + type=mutagen.id3.PictureType.COVER_FRONT, desc=u"Cover", data=converted_target.read_bytes(), ) diff --git a/music_kraken/download/components/__init__.py b/music_kraken/download/components/__init__.py index bd93939..ab79e04 100644 --- a/music_kraken/download/components/__init__.py +++ b/music_kraken/download/components/__init__.py @@ -1,7 +1,9 @@ from __future__ import annotations +import copy import re from collections import defaultdict +from dataclasses import dataclass, field from pathlib import Path from typing import (Any, Callable, Dict, Generator, Generic, Hashable, List, Optional, Tuple, TypeVar, Union) @@ -26,7 +28,6 @@ class HumanIO: def not_found(key: Any) -> None: return None - class Option(Generic[P]): """ This could represent a data object, a string or a page. @@ -166,16 +167,59 @@ class Select(Generic[P]): return "\n".join(str(option) for option in self) -class OptionGroup(Option[P], Select[P]): - ALPHABET: str = "abcdefghijklmnopqrstuvwxyz" - ATTRIBUTES_FORMATTING: Tuple[str, ...] = ("alphabetic_index", "value") +class Node(Generator[P]): + def __init__( + self, + value: Optional[P] = None, + children: List[Node[P]] = None, + parent: Node[P] = None, + **kwargs + ): + self.value = value + self.depth = 0 + self.same_level_index: int = 0 - TEXT_TEMPLATE: str = f"{BColors.HEADER.value}{{alphabetic_index}}) {{value}}{BColors.ENDC.value}" + + self.children: List[Node[P]] = kwargs.get("children", []) + self.parent: Optional[Node[P]] = kwargs.get("parent", None) + + super(Node, self).__init__(**kwargs) + + def hash_key(self, key: Any) -> int: + try: + key = int(key) + except ValueError: + pass + + if isinstance(key, str): + return hash(unify(key)) + + return hash(key) @property - def alphabetic_index(self) -> str: - return self.ALPHABET[self.index % len(self.ALPHABET)] + def is_root(self) -> bool: + return self.parent is None - def __init__(self, value: P, data: Generator[P, None, None] **kwargs): - super(OptionGroup, self).__init__(value=value, data=data, **kwargs) - + @property + def is_leaf(self) -> bool: + return not self.children + + def __iter__(self, **kwargs) -> Generator[Node[P], None, None]: + _level_index_map: Dict[int, int] = kwargs.get("level_index_map", defaultdict(lambda: 0)) + + self.same_level_index = _level_index_map[self.depth] + yield self + _level_index_map[self.depth] += 1 + + for child in self.children: + child.depth = self.depth + 1 + + for node in child.__iter__(level_index_map=_level_index_map): + yield node + + def __getitem__(self, key: Any) -> Option[P]: + pass + + def __contains__(self, key: Any) -> bool: + if key in self.option: + return True diff --git a/music_kraken/utils/__init__.py b/music_kraken/utils/__init__.py index a8d658b..8e96fce 100644 --- a/music_kraken/utils/__init__.py +++ b/music_kraken/utils/__init__.py @@ -1,15 +1,16 @@ -from datetime import datetime -from pathlib import Path +import inspect import json import logging -import inspect +from datetime import datetime +from pathlib import Path from typing import List, Union -from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP, DEBUG_TRACE, DEBUG_OBJECT_TRACE, DEBUG_OBJECT_TRACE_CALLSTACK from .config import config, read_config, write_config from .enums.colors import BColors -from .path_manager import LOCATIONS from .hacking import merge_args +from .path_manager import LOCATIONS +from .shared import (DEBUG, DEBUG_DUMP, DEBUG_LOGGING, DEBUG_OBJECT_TRACE, + DEBUG_OBJECT_TRACE_CALLSTACK, DEBUG_TRACE) """ IO functions @@ -125,4 +126,4 @@ def get_current_millis() -> int: def get_unix_time() -> int: - return int(datetime.now().timestamp()) \ No newline at end of file + return int(datetime.now().timestamp()) diff --git a/music_kraken/utils/string_processing.py b/music_kraken/utils/string_processing.py index b76e3fc..2a7ed23 100644 --- a/music_kraken/utils/string_processing.py +++ b/music_kraken/utils/string_processing.py @@ -1,13 +1,12 @@ -from typing import Tuple, Union, Optional -from pathlib import Path import string from functools import lru_cache +from pathlib import Path +from typing import Optional, Tuple, Union +from urllib.parse import ParseResult, parse_qs, urlparse -from transliterate.exceptions import LanguageDetectionError -from transliterate import translit from pathvalidate import sanitize_filename -from urllib.parse import urlparse, ParseResult, parse_qs - +from transliterate import translit +from transliterate.exceptions import LanguageDetectionError COMMON_TITLE_APPENDIX_LIST: Tuple[str, ...] = ( "(official video)", @@ -180,6 +179,17 @@ def hash_url(url: Union[str, ParseResult]) -> str: r = r.lower().strip() return r +def hash(self, key: Any) -> int: + try: + key = int(key) + except ValueError: + pass + + if isinstance(key, str): + return hash(unify(key)) + + return hash(key) + def remove_feature_part_from_track(title: str) -> str: if ")" != title[-1]: -- 2.45.2 From b30824baf92b9d0ad6b3047ab29b0efa81637c5e Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 10 Jun 2024 11:56:26 +0200 Subject: [PATCH 27/35] fix: removed --- music_kraken/download/components/__init__.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/music_kraken/download/components/__init__.py b/music_kraken/download/components/__init__.py index ab79e04..030d48f 100644 --- a/music_kraken/download/components/__init__.py +++ b/music_kraken/download/components/__init__.py @@ -185,17 +185,6 @@ class Node(Generator[P]): super(Node, self).__init__(**kwargs) - def hash_key(self, key: Any) -> int: - try: - key = int(key) - except ValueError: - pass - - if isinstance(key, str): - return hash(unify(key)) - - return hash(key) - @property def is_root(self) -> bool: return self.parent is None -- 2.45.2 From 684c90a7b48ffb93aabe59ad1d15914c7639be4f Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Mon, 10 Jun 2024 13:46:40 +0200 Subject: [PATCH 28/35] feat: renamed --- music_kraken/download/components/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/music_kraken/download/components/__init__.py b/music_kraken/download/components/__init__.py index 030d48f..effc09f 100644 --- a/music_kraken/download/components/__init__.py +++ b/music_kraken/download/components/__init__.py @@ -177,7 +177,7 @@ class Node(Generator[P]): ): self.value = value self.depth = 0 - self.same_level_index: int = 0 + self.index: int = 0 self.children: List[Node[P]] = kwargs.get("children", []) @@ -196,7 +196,7 @@ class Node(Generator[P]): def __iter__(self, **kwargs) -> Generator[Node[P], None, None]: _level_index_map: Dict[int, int] = kwargs.get("level_index_map", defaultdict(lambda: 0)) - self.same_level_index = _level_index_map[self.depth] + self.index = _level_index_map[self.depth] yield self _level_index_map[self.depth] += 1 -- 2.45.2 From c306da793494475b006dd0fa26c652387a634e80 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 12 Jun 2024 14:18:52 +0200 Subject: [PATCH 29/35] feat: improved and documented the search functions --- music_kraken/download/__init__.py | 42 +++++++++++++++++++++++++++++-- music_kraken/utils/__init__.py | 6 +++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 7484eef..6ec70dd 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -19,7 +19,7 @@ from ..connection import Connection from ..objects import Album, Artist, Collection from ..objects import DatabaseObject as DataObject from ..objects import Label, Options, Song, Source, Target -from ..utils import BColors, output, trace +from ..utils import BColors, limit_generator, output, trace from ..utils.config import main_settings, youtube_settings from ..utils.enums import ALL_SOURCE_TYPES, SourceType from ..utils.enums.album import AlbumType @@ -74,6 +74,8 @@ class Downloader: if auto_register_pages: self.scan_for_pages(**kwargs) + # manage which pages to use + def register_page(self, page_type: Type[Page], **kwargs): if page_type in self._registered_pages: return @@ -122,10 +124,46 @@ class Downloader: for page_type in page_types: yield from self._registered_pages[page_type] + # fetching/downloading data + def search(self, query: Query) -> Generator[DataObject, None, None]: + """Yields all data objects that were found by the query. + Other than `Downloader.search_yield_pages`, this function just yields all data objects. + This looses the data, where th objects were searched originally, so this might not be the best choice. + + Args: + query (Query): The query to search for. + + Yields: + Generator[DataObject, None, None]: A generator that yields all found data objects. + """ + for page in self.get_pages(): yield from page.search(query=query) - + + def search_yield_pages(self, query: Query, results_per_page: Optional[int] = None) -> Generator[Tuple[Page, Generator[DataObject, None, None]], None, None]: + """Yields all data objects that were found by the query, grouped by the page they were found on. + every yield is a tuple of the page and a generator that yields the data objects. + So this could be how it is used: + + ```python + for page, data_objects in downloader.search_yield_pages(query): + print(f"Found on {page}:") + for data_object in data_objects: + print(data_object) + ``` + + Args: + query (Query): The query to search for. + results_per_page (Optional[int], optional): If this is set, the generators only yield this amount of data objects per page. + + Yields: + Generator[Tuple[Page, Generator[DataObject, None, None]], None, None]: yields the page and a generator that yields the data objects. + """ + + for page in self.get_pages(): + yield page, limit_generator(page.search(query=query), limit=results_per_page) + def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: source: Source for source in data_object.source_collection.get_sources(source_type_sorting={ diff --git a/music_kraken/utils/__init__.py b/music_kraken/utils/__init__.py index 8e96fce..bc386a9 100644 --- a/music_kraken/utils/__init__.py +++ b/music_kraken/utils/__init__.py @@ -2,6 +2,7 @@ import inspect import json import logging from datetime import datetime +from itertools import takewhile from pathlib import Path from typing import List, Union @@ -127,3 +128,8 @@ def get_current_millis() -> int: def get_unix_time() -> int: return int(datetime.now().timestamp()) + + +def limit_generator(generator, limit: Optional[int] = None): + return takewhile(lambda x: x < limit, generator) if limit is not None else generator + \ No newline at end of file -- 2.45.2 From f839cdf906e3cc79413629b926a390b6d3e5d185 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 12 Jun 2024 14:25:37 +0200 Subject: [PATCH 30/35] feat: commented fetch functions --- music_kraken/download/__init__.py | 31 +++++++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 6ec70dd..0eadffc 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -164,18 +164,36 @@ class Downloader: for page in self.get_pages(): yield page, limit_generator(page.search(query=query), limit=results_per_page) - def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject: + def fetch_details(self, data_object: DataObject, **kwargs) -> DataObject: + """Fetches more details for the given data object. + This uses every source contained in data_object.source_collection that has a page. + + Args: + data_object (DataObject): The data object to fetch details for. + + Returns: + DataObject: The same data object, but with more details. + """ + source: Source for source in data_object.source_collection.get_sources(source_type_sorting={ "only_with_page": True, }): - new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level) + new_data_object = self.fetch_from_source(source=source, **kwargs) if new_data_object is not None: data_object.merge(new_data_object) return data_object def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]: + """Gets a data object from the given source. + + Args: + source (Source): The source to get the data object from. + + Returns: + Optional[DataObject]: If a data object can be retrieved, it is returned. Otherwise, None is returned. + """ if not source.has_page: return None @@ -192,6 +210,15 @@ class Downloader: return data_object def fetch_from_url(self, url: str) -> Optional[DataObject]: + """This function tries to detect the source of the given url and fetches the data object from it. + + Args: + url (str): The url to fetch the data object from. + + Returns: + Optional[DataObject]: The fetched data object, or None if no source could be detected or if no object could be retrieved. + """ + source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL) if source is None: return None -- 2.45.2 From 097211b3cda5ae18ced5a91603f70e2d8acf621f Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 12 Jun 2024 14:42:08 +0200 Subject: [PATCH 31/35] feat: commented the download function --- music_kraken/download/__init__.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 0eadffc..99ff41d 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -226,6 +226,14 @@ class Downloader: return self.fetch_from_source(source=source) def _skip_object(self, data_object: DataObject) -> bool: + """Determine if the given data object should be downloaded or not. + + Args: + data_object (DataObject): The data object in question. + + Returns: + bool: Returns True if the given data object should be skipped. + """ if isinstance(data_object, Album): if not self.download_options.download_all and data_object.album_type in self.download_options.album_type_blacklist: return True @@ -233,6 +241,18 @@ class Downloader: return False def download(self, data_object: DataObject, genre: str, **kwargs) -> DownloadResult: + """Downloads the given data object. + It will recursively fetch all available details for this and every related object. + Then it will create the folder structure and download the audio file. + In the end the metadata will be written to the file. + + Args: + data_object (DataObject): The data object to download. If it is a artist, it will download the whole discography, if it is an Album it will download the whole Tracklist. + + Returns: + DownloadResult: Relevant information about the download success. + """ + # fetch the given object self.fetch_details(data_object) output(f"\nDownloading {data_object.option_string}...", color=BColors.BOLD) -- 2.45.2 From b2362913786f1ced20eb0ae83ad13dfcb007c5b9 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 19 Jun 2024 10:04:41 +0200 Subject: [PATCH 32/35] feat: implemented downloader --- music_kraken/cli/__init__.py | 5 - music_kraken/cli/genre.py | 53 ---- music_kraken/cli/informations/__init__.py | 1 - music_kraken/cli/informations/paths.py | 22 -- music_kraken/cli/main_downloader.py | 354 ---------------------- music_kraken/cli/options/__init__.py | 0 music_kraken/cli/options/cache.py | 26 -- music_kraken/cli/options/first_config.py | 6 - music_kraken/cli/options/frontend.py | 196 ------------ music_kraken/cli/options/settings.py | 71 ----- music_kraken/cli/utils.py | 47 --- music_kraken/download/__init__.py | 25 ++ 12 files changed, 25 insertions(+), 781 deletions(-) delete mode 100644 music_kraken/cli/__init__.py delete mode 100644 music_kraken/cli/genre.py delete mode 100644 music_kraken/cli/informations/__init__.py delete mode 100644 music_kraken/cli/informations/paths.py delete mode 100644 music_kraken/cli/main_downloader.py delete mode 100644 music_kraken/cli/options/__init__.py delete mode 100644 music_kraken/cli/options/cache.py delete mode 100644 music_kraken/cli/options/first_config.py delete mode 100644 music_kraken/cli/options/frontend.py delete mode 100644 music_kraken/cli/options/settings.py delete mode 100644 music_kraken/cli/utils.py diff --git a/music_kraken/cli/__init__.py b/music_kraken/cli/__init__.py deleted file mode 100644 index 1dbf213..0000000 --- a/music_kraken/cli/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .informations import print_paths -from .main_downloader import download -from .options.settings import settings -from .options.frontend import set_frontend - diff --git a/music_kraken/cli/genre.py b/music_kraken/cli/genre.py deleted file mode 100644 index 3580495..0000000 --- a/music_kraken/cli/genre.py +++ /dev/null @@ -1,53 +0,0 @@ -import re -from pathlib import Path -from typing import Dict, Generator, List, Set, Type, Union - -from ..download import Downloader, Page, components -from ..utils.config import main_settings -from .utils import ask_for_bool, cli_function - - -class GenreIO(components.HumanIO): - @staticmethod - def ask_to_create(option: components.Option) -> bool: - output() - return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") - - @staticmethod - def not_found(key: str) -> None: - output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) - - -def _genre_generator() -> Generator[str, None, None]: - def is_valid_genre(genre: Path) -> bool: - """ - gets the name of all subdirectories of shared.MUSIC_DIR, - but filters out all directories, where the name matches with any Patern - from shared.NOT_A_GENRE_REGEX. - """ - if not genre.is_dir(): - return False - - if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): - return False - - return True - - for genre in filter(is_valid_genre, main_settings["music_directory"].iterdir()): - yield genre.name - -def get_genre() -> str: - select_genre = components.Select( - human_io=GenreIO, - can_create_options=True, - data=_genre_generator(), - ) - genre: Optional[components.Option[str]] = None - - while genre is None: - print(select_genre.pprint()) - print() - - genre = select_genre.choose(input("> ")) - - return genre.value diff --git a/music_kraken/cli/informations/__init__.py b/music_kraken/cli/informations/__init__.py deleted file mode 100644 index be110bf..0000000 --- a/music_kraken/cli/informations/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .paths import print_paths \ No newline at end of file diff --git a/music_kraken/cli/informations/paths.py b/music_kraken/cli/informations/paths.py deleted file mode 100644 index 327b351..0000000 --- a/music_kraken/cli/informations/paths.py +++ /dev/null @@ -1,22 +0,0 @@ -from ..utils import cli_function - -from ...utils.path_manager import LOCATIONS -from ...utils.config import main_settings - - -def all_paths(): - return { - "Temp dir": main_settings["temp_directory"], - "Music dir": main_settings["music_directory"], - "Conf dir": LOCATIONS.CONFIG_DIRECTORY, - "Conf file": LOCATIONS.CONFIG_FILE, - "logging file": main_settings["log_file"], - "FFMPEG bin": main_settings["ffmpeg_binary"], - "Cache Dir": main_settings["cache_directory"], - } - - -@cli_function -def print_paths(): - for name, path in all_paths().items(): - print(f"{name}:\t{path}") \ No newline at end of file diff --git a/music_kraken/cli/main_downloader.py b/music_kraken/cli/main_downloader.py deleted file mode 100644 index eb01c1a..0000000 --- a/music_kraken/cli/main_downloader.py +++ /dev/null @@ -1,354 +0,0 @@ -import random -import re -from pathlib import Path -from typing import Dict, Generator, List, Set, Type, Union - -from .. import console -from ..download import Downloader, Page, components -from ..download.results import GoToResults -from ..download.results import Option as ResultOption -from ..download.results import PageResults, Results -from ..objects import Album, Artist, DatabaseObject, Song -from ..utils import BColors, output -from ..utils.config import main_settings, write_config -from ..utils.enums.colors import BColors -from ..utils.exception import MKInvalidInputException -from ..utils.exception.download import UrlNotFoundException -from ..utils.shared import HELP_MESSAGE, URL_PATTERN -from ..utils.string_processing import fit_to_file_system -from ..utils.support_classes.download_result import DownloadResult -from ..utils.support_classes.query import Query -from .genre import get_genre -from .options.first_config import initial_config -from .utils import ask_for_bool, cli_function - -EXIT_COMMANDS = {"q", "quit", "exit", "abort"} -ALPHABET = "abcdefghijklmnopqrstuvwxyz" -PAGE_NAME_FILL = "-" -MAX_PAGE_LEN = 21 - - - - - -def help_message(): - print(HELP_MESSAGE) - print() - print(random.choice(main_settings["happy_messages"])) - print() - - -class CliDownloader: - def __init__( - self, - exclude_pages: Set[Type[Page]] = None, - exclude_shady: bool = False, - max_displayed_options: int = 10, - option_digits: int = 3, - genre: str = None, - process_metadata_anyway: bool = False, - ) -> None: - self.downloader: Downloader = Downloader(exclude_pages=exclude_pages, exclude_shady=exclude_shady) - - self.page_dict: Dict[str, Type[Page]] = dict() - - self.max_displayed_options = max_displayed_options - self.option_digits: int = option_digits - - self.current_results: Results = None - self._result_history: List[Results] = [] - - self.genre = genre or get_genre() - self.process_metadata_anyway = process_metadata_anyway - - output() - output(f"Downloading to: \"{self.genre}\"", color=BColors.HEADER) - output() - - def print_current_options(self): - - print() - print(self.current_results.pprint()) - - """ - self.page_dict = dict() - - page_count = 0 - for option in self.current_results.formatted_generator(): - if isinstance(option, ResultOption): - r = f"{BColors.GREY.value}{option.index:0{self.option_digits}}{BColors.ENDC.value} {option.music_object.option_string}" - print(r) - else: - prefix = ALPHABET[page_count % len(ALPHABET)] - print( - f"{BColors.HEADER.value}({prefix}) --------------------------------{option.__name__:{PAGE_NAME_FILL}<{MAX_PAGE_LEN}}--------------------{BColors.ENDC.value}") - - self.page_dict[prefix] = option - self.page_dict[option.__name__] = option - - page_count += 1 - """ - - print() - - def set_current_options(self, current_options: Union[Generator[DatabaseObject, None, None], components.Select]): - current_options = current_options if isinstance(current_options, components.Select) else components.DataObjectSelect(current_options) - - if main_settings["result_history"]: - self._result_history.append(current_options) - - if main_settings["history_length"] != -1: - if len(self._result_history) > main_settings["history_length"]: - self._result_history.pop(0) - - self.current_results = current_options - - def previous_option(self) -> bool: - if not main_settings["result_history"]: - print("History is turned of.\nGo to main_settings, and change the value at 'result_history' to 'true'.") - return False - - if len(self._result_history) <= 1: - print(f"No results in history.") - return False - self._result_history.pop() - self.current_results = self._result_history[-1] - return True - - def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: - # strip all the values in key_text - key_text = {key: value.strip() for key, value in key_text.items()} - - song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) - album = None if not "r" in key_text else Album(title=key_text["r"], dynamic=True) - artist = None if not "a" in key_text else Artist(name=key_text["a"], dynamic=True) - - if song is not None: - if album is not None: - song.album_collection.append(album) - if artist is not None: - song.artist_collection.append(artist) - return Query(raw_query=query, music_object=song) - - if album is not None: - if artist is not None: - album.artist_collection.append(artist) - return Query(raw_query=query, music_object=album) - - if artist is not None: - return Query(raw_query=query, music_object=artist) - - return Query(raw_query=query) - - def search(self, query: str): - if re.match(URL_PATTERN, query) is not None: - try: - data_object = self.downloader.fetch_url(query) - except UrlNotFoundException as e: - print(f"{e.url} could not be attributed/parsed to any yet implemented site.\n" - f"PR appreciated if the site isn't implemented.\n" - f"Recommendations and suggestions on sites to implement appreciated.\n" - f"But don't be a bitch if I don't end up implementing it.") - return - self.set_current_options(PageResults(page, data_object.options, max_items_per_page=self.max_displayed_options)) - self.print_current_options() - return - - special_characters = "#\\" - query = query + " " - - key_text = {} - - skip_next = False - escape_next = False - new_text = "" - latest_key: str = None - for i in range(len(query) - 1): - current_char = query[i] - next_char = query[i + 1] - - if skip_next: - skip_next = False - continue - - if escape_next: - new_text += current_char - escape_next = False - - # escaping - if current_char == "\\": - if next_char in special_characters: - escape_next = True - continue - - if current_char == "#": - if latest_key is not None: - key_text[latest_key] = new_text - new_text = "" - - latest_key = next_char - skip_next = True - continue - - new_text += current_char - - if latest_key is not None: - key_text[latest_key] = new_text - - parsed_query: Query = self._process_parsed(key_text, query) - - self.set_current_options(self.downloader.search(parsed_query)) - self.print_current_options() - - def goto(self, data_object: Union[DatabaseObject, components.Select]): - page: Type[Page] - - if isinstance(data_object, components.Select): - self.set_current_options(data_object) - else: - self.downloader.fetch_details(data_object, stop_at_level=1) - self.set_current_options(data_object.options) - - self.print_current_options() - - def download(self, data_objects: List[DatabaseObject], **kwargs) -> bool: - output() - if len(data_objects) > 1: - output(f"Downloading {len(data_objects)} objects...", *("- " + o.option_string for o in data_objects), color=BColors.BOLD, sep="\n") - - _result_map: Dict[DatabaseObject, DownloadResult] = dict() - - for database_object in data_objects: - r = self.downloader.download( - data_object=database_object, - genre=self.genre, - **kwargs - ) - _result_map[database_object] = r - - for music_object, result in _result_map.items(): - output() - output(music_object.option_string) - output(result) - - return True - - def process_input(self, input_str: str) -> bool: - try: - input_str = input_str.strip() - processed_input: str = input_str.lower() - - if processed_input in EXIT_COMMANDS: - return True - - if processed_input == ".": - self.print_current_options() - return False - - if processed_input == "..": - if self.previous_option(): - self.print_current_options() - return False - - command = "" - query = processed_input - if ":" in processed_input: - _ = processed_input.split(":") - command, query = _[0], ":".join(_[1:]) - - do_search = "s" in command - do_fetch = "f" in command - do_download = "d" in command - do_merge = "m" in command - - if do_search and (do_download or do_fetch or do_merge): - raise MKInvalidInputException(message="You can't search and do another operation at the same time.") - - if do_search: - self.search(":".join(input_str.split(":")[1:])) - return False - - def get_selected_objects(q: str): - if q.strip().lower() == "all": - return list(self.current_results) - - indices = [] - for possible_index in q.split(","): - if possible_index == "": - continue - - if possible_index not in self.current_results: - raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not in the current options.") - - yield self.current_results[possible_index] - - selected_objects = list(get_selected_objects(query)) - - if do_merge: - old_selected_objects = selected_objects - - a = old_selected_objects[0] - for b in old_selected_objects[1:]: - if type(a) != type(b): - raise MKInvalidInputException(message="You can't merge different types of objects.") - a.merge(b) - - selected_objects = [a] - - if do_fetch: - for data_object in selected_objects: - self.downloader.fetch_details(data_object) - - self.print_current_options() - return False - - if do_download: - self.download(list(o.value for o in selected_objects)) - return False - - if len(selected_objects) != 1: - raise MKInvalidInputException(message="You can only go to one object at a time without merging.") - - self.goto(selected_objects[0].value) - return False - except MKInvalidInputException as e: - output("\n" + e.message + "\n", color=BColors.FAIL) - help_message() - - return False - - def mainloop(self): - while True: - if self.process_input(input("> ")): - return - - -@cli_function -def download( - genre: str = None, - download_all: bool = False, - direct_download_url: str = None, - command_list: List[str] = None, - process_metadata_anyway: bool = False, -): - if main_settings["hasnt_yet_started"]: - code = initial_config() - if code == 0: - main_settings["hasnt_yet_started"] = False - write_config() - print(f"{BColors.OKGREEN.value}Restart the programm to use it.{BColors.ENDC.value}") - else: - print(f"{BColors.FAIL.value}Something went wrong configuring.{BColors.ENDC.value}") - - shell = CliDownloader(genre=genre, process_metadata_anyway=process_metadata_anyway) - - if command_list is not None: - for command in command_list: - shell.process_input(command) - return - - if direct_download_url is not None: - if shell.download(direct_download_url, download_all=download_all): - return - - shell.mainloop() diff --git a/music_kraken/cli/options/__init__.py b/music_kraken/cli/options/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/music_kraken/cli/options/cache.py b/music_kraken/cli/options/cache.py deleted file mode 100644 index 42cb76b..0000000 --- a/music_kraken/cli/options/cache.py +++ /dev/null @@ -1,26 +0,0 @@ -from logging import getLogger - -from ..utils import cli_function -from ...connection.cache import Cache - - -@cli_function -def clear_cache(): - """ - Deletes the cache. - :return: - """ - - Cache("main", getLogger("cache")).clear() - print("Cleared cache") - - -@cli_function -def clean_cache(): - """ - Deletes the outdated cache. (all expired cached files, and not indexed files) - :return: - """ - - Cache("main", getLogger("cache")).clean() - print("Cleaned cache") diff --git a/music_kraken/cli/options/first_config.py b/music_kraken/cli/options/first_config.py deleted file mode 100644 index 6160265..0000000 --- a/music_kraken/cli/options/first_config.py +++ /dev/null @@ -1,6 +0,0 @@ -from .frontend import set_frontend - - -def initial_config(): - code = set_frontend(no_cli=True) - return code diff --git a/music_kraken/cli/options/frontend.py b/music_kraken/cli/options/frontend.py deleted file mode 100644 index 64838c4..0000000 --- a/music_kraken/cli/options/frontend.py +++ /dev/null @@ -1,196 +0,0 @@ -from typing import Dict, List -from dataclasses import dataclass -from collections import defaultdict -from urllib.parse import urlparse - -from ..utils import cli_function - -from ...objects import Country -from ...utils import config, write_config -from ...utils.config import youtube_settings -from ...connection import Connection - - -@dataclass -class Instance: - """ - Attributes which influence the quality of an instance: - - - users - """ - name: str - uri: str - regions: List[Country] - users: int = 0 - - def __str__(self) -> str: - return f"{self.name} with {self.users} users." - - -class FrontendInstance: - SETTING_NAME = "placeholder" - - def __init__(self) -> None: - self.region_instances: Dict[Country, List[Instance]] = defaultdict(list) - self.all_instances: List[Instance] = [] - - def add_instance(self, instance: Instance): - self.all_instances.append(instance) - - youtube_lists = youtube_settings["youtube_url"] - existing_netlocs = set(tuple(url.netloc for url in youtube_lists)) - - parsed_instance = urlparse(instance.uri) - instance_netloc = parsed_instance.netloc - - if instance_netloc not in existing_netlocs: - youtube_lists.append(parsed_instance) - youtube_settings.__setitem__("youtube_url", youtube_lists, is_parsed=True) - - for region in instance.regions: - self.region_instances[region].append(instance) - - def fetch(self, silent: bool = False): - if not silent: - print(f"Downloading {type(self).__name__} instances...") - - def set_instance(self, instance: Instance): - youtube_settings.__setitem__(self.SETTING_NAME, instance.uri) - - def _choose_country(self) -> List[Instance]: - print("Input the country code, an example would be \"US\"") - print('\n'.join(f'{region.name} ({region.alpha_2})' for region in self.region_instances)) - print() - - - available_instances = set(i.alpha_2 for i in self.region_instances) - - chosen_region = "" - - while chosen_region not in available_instances: - chosen_region = input("nearest country: ").strip().upper() - - return self.region_instances[Country.by_alpha_2(chosen_region)] - - def choose(self, silent: bool = False): - instances = self.all_instances if silent else self._choose_country() - instances.sort(key=lambda x: x.users, reverse=True) - - if silent: - self.set_instance(instances[0]) - return - - # output the options - print("Choose your instance (input needs to be a digit):") - for i, instance in enumerate(instances): - print(f"{i}) {instance}") - - print() - - # ask for index - index = "" - while not index.isdigit() or int(index) >= len(instances): - index = input("> ").strip() - - instance = instances[int(index)] - print() - print(f"Setting the instance to {instance}") - - self.set_instance(instance) - - -class Invidious(FrontendInstance): - SETTING_NAME = "invidious_instance" - - def __init__(self) -> None: - self.connection = Connection(host="https://api.invidious.io/") - self.endpoint = "https://api.invidious.io/instances.json" - - super().__init__() - - - def _process_instance(self, all_instance_data: dict): - instance_data = all_instance_data[1] - stats = instance_data["stats"] - - if not instance_data["api"]: - return - if instance_data["type"] != "https": - return - - region = instance_data["region"] - - instance = Instance( - name=all_instance_data[0], - uri=instance_data["uri"], - regions=[Country.by_alpha_2(region)], - users=stats["usage"]["users"]["total"] - ) - - self.add_instance(instance) - - def fetch(self, silent: bool): - r = self.connection.get(self.endpoint) - if r is None: - return - - for instance in r.json(): - self._process_instance(all_instance_data=instance) - - -class Piped(FrontendInstance): - SETTING_NAME = "piped_instance" - - def __init__(self) -> None: - self.connection = Connection(host="https://raw.githubusercontent.com") - - super().__init__() - - def process_instance(self, instance_data: str): - cells = instance_data.split(" | ") - - instance = Instance( - name=cells[0].strip(), - uri=cells[1].strip(), - regions=[Country.by_emoji(flag) for flag in cells[2].split(", ")] - ) - - self.add_instance(instance) - - def fetch(self, silent: bool = False): - r = self.connection.get("https://raw.githubusercontent.com/wiki/TeamPiped/Piped-Frontend/Instances.md") - if r is None: - return - - process = False - - for line in r.content.decode("utf-8").split("\n"): - line = line.strip() - - if line != "" and process: - self.process_instance(line) - - if line.startswith("---"): - process = True - - -class FrontendSelection: - def __init__(self): - self.invidious = Invidious() - self.piped = Piped() - - def choose(self, silent: bool = False): - self.invidious.fetch(silent) - self.invidious.choose(silent) - - self.piped.fetch(silent) - self.piped.choose(silent) - - -@cli_function -def set_frontend(silent: bool = False): - shell = FrontendSelection() - shell.choose(silent=silent) - - return 0 - \ No newline at end of file diff --git a/music_kraken/cli/options/settings.py b/music_kraken/cli/options/settings.py deleted file mode 100644 index 3ba0ade..0000000 --- a/music_kraken/cli/options/settings.py +++ /dev/null @@ -1,71 +0,0 @@ -from ..utils import cli_function - -from ...utils.config import config, write_config -from ...utils import exception - - -def modify_setting(_name: str, _value: str, invalid_ok: bool = True) -> bool: - try: - config.set_name_to_value(_name, _value) - except exception.config.SettingException as e: - if invalid_ok: - print(e) - return False - else: - raise e - - write_config() - return True - - -def print_settings(): - for i, attribute in enumerate(config): - print(f"{i:0>2}: {attribute.name}={attribute.value}") - - - def modify_setting_by_index(index: int) -> bool: - attribute = list(config)[index] - - print() - print(attribute) - - input__ = input(f"{attribute.name}=") - if not modify_setting(attribute.name, input__.strip()): - return modify_setting_by_index(index) - - return True - - -def modify_setting_by_index(index: int) -> bool: - attribute = list(config)[index] - - print() - print(attribute) - - input__ = input(f"{attribute.name}=") - if not modify_setting(attribute.name, input__.strip()): - return modify_setting_by_index(index) - - return True - - -@cli_function -def settings( - name: str = None, - value: str = None, -): - if name is not None and value is not None: - modify_setting(name, value, invalid_ok=True) - return - - while True: - print_settings() - - input_ = input("Id of setting to modify: ") - print() - if input_.isdigit() and int(input_) < len(config): - if modify_setting_by_index(int(input_)): - return - else: - print("Please input a valid ID.") - print() \ No newline at end of file diff --git a/music_kraken/cli/utils.py b/music_kraken/cli/utils.py deleted file mode 100644 index 1acfe33..0000000 --- a/music_kraken/cli/utils.py +++ /dev/null @@ -1,47 +0,0 @@ -from ..utils import BColors -from ..utils.shared import get_random_message - - -def cli_function(function): - def wrapper(*args, **kwargs): - silent = kwargs.get("no_cli", False) - if "no_cli" in kwargs: - del kwargs["no_cli"] - - if silent: - return function(*args, **kwargs) - return - - code = 0 - - print_cute_message() - print() - try: - code = function(*args, **kwargs) - except KeyboardInterrupt: - print("\n\nRaise an issue if I fucked up:\nhttps://github.com/HeIIow2/music-downloader/issues") - - finally: - print() - print_cute_message() - print("See you soon! :3") - - exit() - - return wrapper - - -def print_cute_message(): - message = get_random_message() - try: - print(message) - except UnicodeEncodeError: - message = str(c for c in message if 0 < ord(c) < 127) - print(message) - - -AGREE_INPUTS = {"y", "yes", "ok"} -def ask_for_bool(msg: str) -> bool: - i = input(f"{msg} ({BColors.OKGREEN.value}Y{BColors.ENDC.value}/{BColors.FAIL.value}N{BColors.ENDC.value})? ").lower() - return i in AGREE_INPUTS - \ No newline at end of file diff --git a/music_kraken/download/__init__.py b/music_kraken/download/__init__.py index 99ff41d..21567f8 100644 --- a/music_kraken/download/__init__.py +++ b/music_kraken/download/__init__.py @@ -415,6 +415,31 @@ class Downloader: return source.page.fetch_object_from_source(source=source, **kwargs) + # misc function + + def get_existing_genres(self) -> Generator[str, None, None]: + """Yields every existing genre, for the user to select from. + + Yields: + Generator[str, None, None]: a generator that yields every existing genre. + """ + + def is_valid_genre(genre: Path) -> bool: + """ + gets the name of all subdirectories of shared.MUSIC_DIR, + but filters out all directories, where the name matches with any Patern + from shared.NOT_A_GENRE_REGEX. + """ + if not genre.is_dir(): + return False + + if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): + return False + + return True + + for genre in filter(is_valid_genre, main_settings["music_directory"].iterdir()): + yield genre.name class Page: REGISTER = True -- 2.45.2 From ed2eeabd6a964f7ede8eb4f93562267ed228a756 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 19 Jun 2024 11:02:58 +0200 Subject: [PATCH 33/35] draft: new cli --- music_kraken/__meta__.py | 3 + music_kraken/development_cli/__init__.py | 77 ++++ music_kraken/development_cli/genre.py | 53 +++ .../development_cli/informations/__init__.py | 1 + .../development_cli/informations/paths.py | 22 ++ .../development_cli/main_downloader.py | 354 ++++++++++++++++++ .../development_cli/options/__init__.py | 0 music_kraken/development_cli/options/cache.py | 26 ++ .../development_cli/options/first_config.py | 6 + .../development_cli/options/frontend.py | 196 ++++++++++ .../development_cli/options/settings.py | 71 ++++ music_kraken/development_cli/utils.py | 51 +++ 12 files changed, 860 insertions(+) create mode 100644 music_kraken/__meta__.py create mode 100644 music_kraken/development_cli/__init__.py create mode 100644 music_kraken/development_cli/genre.py create mode 100644 music_kraken/development_cli/informations/__init__.py create mode 100644 music_kraken/development_cli/informations/paths.py create mode 100644 music_kraken/development_cli/main_downloader.py create mode 100644 music_kraken/development_cli/options/__init__.py create mode 100644 music_kraken/development_cli/options/cache.py create mode 100644 music_kraken/development_cli/options/first_config.py create mode 100644 music_kraken/development_cli/options/frontend.py create mode 100644 music_kraken/development_cli/options/settings.py create mode 100644 music_kraken/development_cli/utils.py diff --git a/music_kraken/__meta__.py b/music_kraken/__meta__.py new file mode 100644 index 0000000..9a47f73 --- /dev/null +++ b/music_kraken/__meta__.py @@ -0,0 +1,3 @@ +PROGRAMM: str = "music-kraken" +DESCRIPTION: str = """This program will first get the metadata of various songs from metadata providers like musicbrainz, and then search for download links on pages like bandcamp. +Then it will download the song and edit the metadata accordingly.""" diff --git a/music_kraken/development_cli/__init__.py b/music_kraken/development_cli/__init__.py new file mode 100644 index 0000000..7cc3b15 --- /dev/null +++ b/music_kraken/development_cli/__init__.py @@ -0,0 +1,77 @@ +import argparse +from functools import cached_property + +from ..__meta__ import DESCRIPTION, PROGRAMM +from ..download import Downloader +from ..utils import BColors +from ..utils.string_processing import unify +from .utils import ask_for_bool, ask_for_create + + +class Selection: + def __init__(self, options: list): + self.options = options + + def pprint(self): + return "\n".join(f"{i}: {option}" for i, option in enumerate(self.options)) + + def choose(self, input_: str): + try: + return self.options[int(input_)] + except (ValueError, IndexError): + return None + + +class DevelopmentCli: + def __init__(self, args: argparse.Namespace): + self.args = args + + if args.genre: + self.genre = args.genre + + self.downloader: Downloader = Downloader() + + @cached_property + def genre(self) -> str: + """This is a cached property, which means if it isn't set in the constructor or before it is accessed, + the program will be thrown in a shell + + Returns: + str: the genre that should be used + """ + option_string = f"{BColors.HEADER}Genres{BColors.ENDC}" + genre_map = {} + + _string_list = [] + for i, genre in enumerate(self.downloader.get_existing_genres()): + option_string += f"\n{BColors.BOLD}{i}{BColors.ENDC}: {genre}" + + genre_map[str(i)] = genre + genre_map[unify(genre)] = genre + + genre = None + while genre is None: + print(option_string) + print() + + i = input("> ") + u = unify(i) + if u in genre_map: + genre = genre_map[u] + break + + if ask_for_create("genre", i): + genre = i + break + + return genre + + +def main(): + parser = argparse.ArgumentParser( + prog=PROGRAMM, + description=DESCRIPTION, + epilog='This is just a development cli. The real frontend is coming soon.' + ) + parser.add_argument('--genre', '-g', action='store_const') + args = parser.parse_args() diff --git a/music_kraken/development_cli/genre.py b/music_kraken/development_cli/genre.py new file mode 100644 index 0000000..3580495 --- /dev/null +++ b/music_kraken/development_cli/genre.py @@ -0,0 +1,53 @@ +import re +from pathlib import Path +from typing import Dict, Generator, List, Set, Type, Union + +from ..download import Downloader, Page, components +from ..utils.config import main_settings +from .utils import ask_for_bool, cli_function + + +class GenreIO(components.HumanIO): + @staticmethod + def ask_to_create(option: components.Option) -> bool: + output() + return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") + + @staticmethod + def not_found(key: str) -> None: + output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) + + +def _genre_generator() -> Generator[str, None, None]: + def is_valid_genre(genre: Path) -> bool: + """ + gets the name of all subdirectories of shared.MUSIC_DIR, + but filters out all directories, where the name matches with any Patern + from shared.NOT_A_GENRE_REGEX. + """ + if not genre.is_dir(): + return False + + if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): + return False + + return True + + for genre in filter(is_valid_genre, main_settings["music_directory"].iterdir()): + yield genre.name + +def get_genre() -> str: + select_genre = components.Select( + human_io=GenreIO, + can_create_options=True, + data=_genre_generator(), + ) + genre: Optional[components.Option[str]] = None + + while genre is None: + print(select_genre.pprint()) + print() + + genre = select_genre.choose(input("> ")) + + return genre.value diff --git a/music_kraken/development_cli/informations/__init__.py b/music_kraken/development_cli/informations/__init__.py new file mode 100644 index 0000000..be110bf --- /dev/null +++ b/music_kraken/development_cli/informations/__init__.py @@ -0,0 +1 @@ +from .paths import print_paths \ No newline at end of file diff --git a/music_kraken/development_cli/informations/paths.py b/music_kraken/development_cli/informations/paths.py new file mode 100644 index 0000000..327b351 --- /dev/null +++ b/music_kraken/development_cli/informations/paths.py @@ -0,0 +1,22 @@ +from ..utils import cli_function + +from ...utils.path_manager import LOCATIONS +from ...utils.config import main_settings + + +def all_paths(): + return { + "Temp dir": main_settings["temp_directory"], + "Music dir": main_settings["music_directory"], + "Conf dir": LOCATIONS.CONFIG_DIRECTORY, + "Conf file": LOCATIONS.CONFIG_FILE, + "logging file": main_settings["log_file"], + "FFMPEG bin": main_settings["ffmpeg_binary"], + "Cache Dir": main_settings["cache_directory"], + } + + +@cli_function +def print_paths(): + for name, path in all_paths().items(): + print(f"{name}:\t{path}") \ No newline at end of file diff --git a/music_kraken/development_cli/main_downloader.py b/music_kraken/development_cli/main_downloader.py new file mode 100644 index 0000000..eb01c1a --- /dev/null +++ b/music_kraken/development_cli/main_downloader.py @@ -0,0 +1,354 @@ +import random +import re +from pathlib import Path +from typing import Dict, Generator, List, Set, Type, Union + +from .. import console +from ..download import Downloader, Page, components +from ..download.results import GoToResults +from ..download.results import Option as ResultOption +from ..download.results import PageResults, Results +from ..objects import Album, Artist, DatabaseObject, Song +from ..utils import BColors, output +from ..utils.config import main_settings, write_config +from ..utils.enums.colors import BColors +from ..utils.exception import MKInvalidInputException +from ..utils.exception.download import UrlNotFoundException +from ..utils.shared import HELP_MESSAGE, URL_PATTERN +from ..utils.string_processing import fit_to_file_system +from ..utils.support_classes.download_result import DownloadResult +from ..utils.support_classes.query import Query +from .genre import get_genre +from .options.first_config import initial_config +from .utils import ask_for_bool, cli_function + +EXIT_COMMANDS = {"q", "quit", "exit", "abort"} +ALPHABET = "abcdefghijklmnopqrstuvwxyz" +PAGE_NAME_FILL = "-" +MAX_PAGE_LEN = 21 + + + + + +def help_message(): + print(HELP_MESSAGE) + print() + print(random.choice(main_settings["happy_messages"])) + print() + + +class CliDownloader: + def __init__( + self, + exclude_pages: Set[Type[Page]] = None, + exclude_shady: bool = False, + max_displayed_options: int = 10, + option_digits: int = 3, + genre: str = None, + process_metadata_anyway: bool = False, + ) -> None: + self.downloader: Downloader = Downloader(exclude_pages=exclude_pages, exclude_shady=exclude_shady) + + self.page_dict: Dict[str, Type[Page]] = dict() + + self.max_displayed_options = max_displayed_options + self.option_digits: int = option_digits + + self.current_results: Results = None + self._result_history: List[Results] = [] + + self.genre = genre or get_genre() + self.process_metadata_anyway = process_metadata_anyway + + output() + output(f"Downloading to: \"{self.genre}\"", color=BColors.HEADER) + output() + + def print_current_options(self): + + print() + print(self.current_results.pprint()) + + """ + self.page_dict = dict() + + page_count = 0 + for option in self.current_results.formatted_generator(): + if isinstance(option, ResultOption): + r = f"{BColors.GREY.value}{option.index:0{self.option_digits}}{BColors.ENDC.value} {option.music_object.option_string}" + print(r) + else: + prefix = ALPHABET[page_count % len(ALPHABET)] + print( + f"{BColors.HEADER.value}({prefix}) --------------------------------{option.__name__:{PAGE_NAME_FILL}<{MAX_PAGE_LEN}}--------------------{BColors.ENDC.value}") + + self.page_dict[prefix] = option + self.page_dict[option.__name__] = option + + page_count += 1 + """ + + print() + + def set_current_options(self, current_options: Union[Generator[DatabaseObject, None, None], components.Select]): + current_options = current_options if isinstance(current_options, components.Select) else components.DataObjectSelect(current_options) + + if main_settings["result_history"]: + self._result_history.append(current_options) + + if main_settings["history_length"] != -1: + if len(self._result_history) > main_settings["history_length"]: + self._result_history.pop(0) + + self.current_results = current_options + + def previous_option(self) -> bool: + if not main_settings["result_history"]: + print("History is turned of.\nGo to main_settings, and change the value at 'result_history' to 'true'.") + return False + + if len(self._result_history) <= 1: + print(f"No results in history.") + return False + self._result_history.pop() + self.current_results = self._result_history[-1] + return True + + def _process_parsed(self, key_text: Dict[str, str], query: str) -> Query: + # strip all the values in key_text + key_text = {key: value.strip() for key, value in key_text.items()} + + song = None if not "t" in key_text else Song(title=key_text["t"], dynamic=True) + album = None if not "r" in key_text else Album(title=key_text["r"], dynamic=True) + artist = None if not "a" in key_text else Artist(name=key_text["a"], dynamic=True) + + if song is not None: + if album is not None: + song.album_collection.append(album) + if artist is not None: + song.artist_collection.append(artist) + return Query(raw_query=query, music_object=song) + + if album is not None: + if artist is not None: + album.artist_collection.append(artist) + return Query(raw_query=query, music_object=album) + + if artist is not None: + return Query(raw_query=query, music_object=artist) + + return Query(raw_query=query) + + def search(self, query: str): + if re.match(URL_PATTERN, query) is not None: + try: + data_object = self.downloader.fetch_url(query) + except UrlNotFoundException as e: + print(f"{e.url} could not be attributed/parsed to any yet implemented site.\n" + f"PR appreciated if the site isn't implemented.\n" + f"Recommendations and suggestions on sites to implement appreciated.\n" + f"But don't be a bitch if I don't end up implementing it.") + return + self.set_current_options(PageResults(page, data_object.options, max_items_per_page=self.max_displayed_options)) + self.print_current_options() + return + + special_characters = "#\\" + query = query + " " + + key_text = {} + + skip_next = False + escape_next = False + new_text = "" + latest_key: str = None + for i in range(len(query) - 1): + current_char = query[i] + next_char = query[i + 1] + + if skip_next: + skip_next = False + continue + + if escape_next: + new_text += current_char + escape_next = False + + # escaping + if current_char == "\\": + if next_char in special_characters: + escape_next = True + continue + + if current_char == "#": + if latest_key is not None: + key_text[latest_key] = new_text + new_text = "" + + latest_key = next_char + skip_next = True + continue + + new_text += current_char + + if latest_key is not None: + key_text[latest_key] = new_text + + parsed_query: Query = self._process_parsed(key_text, query) + + self.set_current_options(self.downloader.search(parsed_query)) + self.print_current_options() + + def goto(self, data_object: Union[DatabaseObject, components.Select]): + page: Type[Page] + + if isinstance(data_object, components.Select): + self.set_current_options(data_object) + else: + self.downloader.fetch_details(data_object, stop_at_level=1) + self.set_current_options(data_object.options) + + self.print_current_options() + + def download(self, data_objects: List[DatabaseObject], **kwargs) -> bool: + output() + if len(data_objects) > 1: + output(f"Downloading {len(data_objects)} objects...", *("- " + o.option_string for o in data_objects), color=BColors.BOLD, sep="\n") + + _result_map: Dict[DatabaseObject, DownloadResult] = dict() + + for database_object in data_objects: + r = self.downloader.download( + data_object=database_object, + genre=self.genre, + **kwargs + ) + _result_map[database_object] = r + + for music_object, result in _result_map.items(): + output() + output(music_object.option_string) + output(result) + + return True + + def process_input(self, input_str: str) -> bool: + try: + input_str = input_str.strip() + processed_input: str = input_str.lower() + + if processed_input in EXIT_COMMANDS: + return True + + if processed_input == ".": + self.print_current_options() + return False + + if processed_input == "..": + if self.previous_option(): + self.print_current_options() + return False + + command = "" + query = processed_input + if ":" in processed_input: + _ = processed_input.split(":") + command, query = _[0], ":".join(_[1:]) + + do_search = "s" in command + do_fetch = "f" in command + do_download = "d" in command + do_merge = "m" in command + + if do_search and (do_download or do_fetch or do_merge): + raise MKInvalidInputException(message="You can't search and do another operation at the same time.") + + if do_search: + self.search(":".join(input_str.split(":")[1:])) + return False + + def get_selected_objects(q: str): + if q.strip().lower() == "all": + return list(self.current_results) + + indices = [] + for possible_index in q.split(","): + if possible_index == "": + continue + + if possible_index not in self.current_results: + raise MKInvalidInputException(message=f"The index \"{possible_index}\" is not in the current options.") + + yield self.current_results[possible_index] + + selected_objects = list(get_selected_objects(query)) + + if do_merge: + old_selected_objects = selected_objects + + a = old_selected_objects[0] + for b in old_selected_objects[1:]: + if type(a) != type(b): + raise MKInvalidInputException(message="You can't merge different types of objects.") + a.merge(b) + + selected_objects = [a] + + if do_fetch: + for data_object in selected_objects: + self.downloader.fetch_details(data_object) + + self.print_current_options() + return False + + if do_download: + self.download(list(o.value for o in selected_objects)) + return False + + if len(selected_objects) != 1: + raise MKInvalidInputException(message="You can only go to one object at a time without merging.") + + self.goto(selected_objects[0].value) + return False + except MKInvalidInputException as e: + output("\n" + e.message + "\n", color=BColors.FAIL) + help_message() + + return False + + def mainloop(self): + while True: + if self.process_input(input("> ")): + return + + +@cli_function +def download( + genre: str = None, + download_all: bool = False, + direct_download_url: str = None, + command_list: List[str] = None, + process_metadata_anyway: bool = False, +): + if main_settings["hasnt_yet_started"]: + code = initial_config() + if code == 0: + main_settings["hasnt_yet_started"] = False + write_config() + print(f"{BColors.OKGREEN.value}Restart the programm to use it.{BColors.ENDC.value}") + else: + print(f"{BColors.FAIL.value}Something went wrong configuring.{BColors.ENDC.value}") + + shell = CliDownloader(genre=genre, process_metadata_anyway=process_metadata_anyway) + + if command_list is not None: + for command in command_list: + shell.process_input(command) + return + + if direct_download_url is not None: + if shell.download(direct_download_url, download_all=download_all): + return + + shell.mainloop() diff --git a/music_kraken/development_cli/options/__init__.py b/music_kraken/development_cli/options/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/music_kraken/development_cli/options/cache.py b/music_kraken/development_cli/options/cache.py new file mode 100644 index 0000000..42cb76b --- /dev/null +++ b/music_kraken/development_cli/options/cache.py @@ -0,0 +1,26 @@ +from logging import getLogger + +from ..utils import cli_function +from ...connection.cache import Cache + + +@cli_function +def clear_cache(): + """ + Deletes the cache. + :return: + """ + + Cache("main", getLogger("cache")).clear() + print("Cleared cache") + + +@cli_function +def clean_cache(): + """ + Deletes the outdated cache. (all expired cached files, and not indexed files) + :return: + """ + + Cache("main", getLogger("cache")).clean() + print("Cleaned cache") diff --git a/music_kraken/development_cli/options/first_config.py b/music_kraken/development_cli/options/first_config.py new file mode 100644 index 0000000..6160265 --- /dev/null +++ b/music_kraken/development_cli/options/first_config.py @@ -0,0 +1,6 @@ +from .frontend import set_frontend + + +def initial_config(): + code = set_frontend(no_cli=True) + return code diff --git a/music_kraken/development_cli/options/frontend.py b/music_kraken/development_cli/options/frontend.py new file mode 100644 index 0000000..64838c4 --- /dev/null +++ b/music_kraken/development_cli/options/frontend.py @@ -0,0 +1,196 @@ +from typing import Dict, List +from dataclasses import dataclass +from collections import defaultdict +from urllib.parse import urlparse + +from ..utils import cli_function + +from ...objects import Country +from ...utils import config, write_config +from ...utils.config import youtube_settings +from ...connection import Connection + + +@dataclass +class Instance: + """ + Attributes which influence the quality of an instance: + + - users + """ + name: str + uri: str + regions: List[Country] + users: int = 0 + + def __str__(self) -> str: + return f"{self.name} with {self.users} users." + + +class FrontendInstance: + SETTING_NAME = "placeholder" + + def __init__(self) -> None: + self.region_instances: Dict[Country, List[Instance]] = defaultdict(list) + self.all_instances: List[Instance] = [] + + def add_instance(self, instance: Instance): + self.all_instances.append(instance) + + youtube_lists = youtube_settings["youtube_url"] + existing_netlocs = set(tuple(url.netloc for url in youtube_lists)) + + parsed_instance = urlparse(instance.uri) + instance_netloc = parsed_instance.netloc + + if instance_netloc not in existing_netlocs: + youtube_lists.append(parsed_instance) + youtube_settings.__setitem__("youtube_url", youtube_lists, is_parsed=True) + + for region in instance.regions: + self.region_instances[region].append(instance) + + def fetch(self, silent: bool = False): + if not silent: + print(f"Downloading {type(self).__name__} instances...") + + def set_instance(self, instance: Instance): + youtube_settings.__setitem__(self.SETTING_NAME, instance.uri) + + def _choose_country(self) -> List[Instance]: + print("Input the country code, an example would be \"US\"") + print('\n'.join(f'{region.name} ({region.alpha_2})' for region in self.region_instances)) + print() + + + available_instances = set(i.alpha_2 for i in self.region_instances) + + chosen_region = "" + + while chosen_region not in available_instances: + chosen_region = input("nearest country: ").strip().upper() + + return self.region_instances[Country.by_alpha_2(chosen_region)] + + def choose(self, silent: bool = False): + instances = self.all_instances if silent else self._choose_country() + instances.sort(key=lambda x: x.users, reverse=True) + + if silent: + self.set_instance(instances[0]) + return + + # output the options + print("Choose your instance (input needs to be a digit):") + for i, instance in enumerate(instances): + print(f"{i}) {instance}") + + print() + + # ask for index + index = "" + while not index.isdigit() or int(index) >= len(instances): + index = input("> ").strip() + + instance = instances[int(index)] + print() + print(f"Setting the instance to {instance}") + + self.set_instance(instance) + + +class Invidious(FrontendInstance): + SETTING_NAME = "invidious_instance" + + def __init__(self) -> None: + self.connection = Connection(host="https://api.invidious.io/") + self.endpoint = "https://api.invidious.io/instances.json" + + super().__init__() + + + def _process_instance(self, all_instance_data: dict): + instance_data = all_instance_data[1] + stats = instance_data["stats"] + + if not instance_data["api"]: + return + if instance_data["type"] != "https": + return + + region = instance_data["region"] + + instance = Instance( + name=all_instance_data[0], + uri=instance_data["uri"], + regions=[Country.by_alpha_2(region)], + users=stats["usage"]["users"]["total"] + ) + + self.add_instance(instance) + + def fetch(self, silent: bool): + r = self.connection.get(self.endpoint) + if r is None: + return + + for instance in r.json(): + self._process_instance(all_instance_data=instance) + + +class Piped(FrontendInstance): + SETTING_NAME = "piped_instance" + + def __init__(self) -> None: + self.connection = Connection(host="https://raw.githubusercontent.com") + + super().__init__() + + def process_instance(self, instance_data: str): + cells = instance_data.split(" | ") + + instance = Instance( + name=cells[0].strip(), + uri=cells[1].strip(), + regions=[Country.by_emoji(flag) for flag in cells[2].split(", ")] + ) + + self.add_instance(instance) + + def fetch(self, silent: bool = False): + r = self.connection.get("https://raw.githubusercontent.com/wiki/TeamPiped/Piped-Frontend/Instances.md") + if r is None: + return + + process = False + + for line in r.content.decode("utf-8").split("\n"): + line = line.strip() + + if line != "" and process: + self.process_instance(line) + + if line.startswith("---"): + process = True + + +class FrontendSelection: + def __init__(self): + self.invidious = Invidious() + self.piped = Piped() + + def choose(self, silent: bool = False): + self.invidious.fetch(silent) + self.invidious.choose(silent) + + self.piped.fetch(silent) + self.piped.choose(silent) + + +@cli_function +def set_frontend(silent: bool = False): + shell = FrontendSelection() + shell.choose(silent=silent) + + return 0 + \ No newline at end of file diff --git a/music_kraken/development_cli/options/settings.py b/music_kraken/development_cli/options/settings.py new file mode 100644 index 0000000..3ba0ade --- /dev/null +++ b/music_kraken/development_cli/options/settings.py @@ -0,0 +1,71 @@ +from ..utils import cli_function + +from ...utils.config import config, write_config +from ...utils import exception + + +def modify_setting(_name: str, _value: str, invalid_ok: bool = True) -> bool: + try: + config.set_name_to_value(_name, _value) + except exception.config.SettingException as e: + if invalid_ok: + print(e) + return False + else: + raise e + + write_config() + return True + + +def print_settings(): + for i, attribute in enumerate(config): + print(f"{i:0>2}: {attribute.name}={attribute.value}") + + + def modify_setting_by_index(index: int) -> bool: + attribute = list(config)[index] + + print() + print(attribute) + + input__ = input(f"{attribute.name}=") + if not modify_setting(attribute.name, input__.strip()): + return modify_setting_by_index(index) + + return True + + +def modify_setting_by_index(index: int) -> bool: + attribute = list(config)[index] + + print() + print(attribute) + + input__ = input(f"{attribute.name}=") + if not modify_setting(attribute.name, input__.strip()): + return modify_setting_by_index(index) + + return True + + +@cli_function +def settings( + name: str = None, + value: str = None, +): + if name is not None and value is not None: + modify_setting(name, value, invalid_ok=True) + return + + while True: + print_settings() + + input_ = input("Id of setting to modify: ") + print() + if input_.isdigit() and int(input_) < len(config): + if modify_setting_by_index(int(input_)): + return + else: + print("Please input a valid ID.") + print() \ No newline at end of file diff --git a/music_kraken/development_cli/utils.py b/music_kraken/development_cli/utils.py new file mode 100644 index 0000000..b0fb1b8 --- /dev/null +++ b/music_kraken/development_cli/utils.py @@ -0,0 +1,51 @@ +from ..utils import BColors +from ..utils.shared import get_random_message + + +def cli_function(function): + def wrapper(*args, **kwargs): + silent = kwargs.get("no_cli", False) + if "no_cli" in kwargs: + del kwargs["no_cli"] + + if silent: + return function(*args, **kwargs) + return + + code = 0 + + print_cute_message() + print() + try: + code = function(*args, **kwargs) + except KeyboardInterrupt: + print("\n\nRaise an issue if I fucked up:\nhttps://github.com/HeIIow2/music-downloader/issues") + + finally: + print() + print_cute_message() + print("See you soon! :3") + + exit() + + return wrapper + + +def print_cute_message(): + message = get_random_message() + try: + print(message) + except UnicodeEncodeError: + message = str(c for c in message if 0 < ord(c) < 127) + print(message) + + +AGREE_INPUTS = {"y", "yes", "ok"} +def ask_for_bool(msg: str) -> bool: + i = input(f"{msg} ({BColors.OKGREEN.value}Y{BColors.ENDC.value}/{BColors.FAIL.value}N{BColors.ENDC.value})? ").lower() + return i in AGREE_INPUTS + + +def ask_for_create(name: str, value: str) -> bool: + return ask_for_bool(f"Do you want to create the {name} {BColors.OKBLUE}{value}{BColors.ENDC}?") + \ No newline at end of file -- 2.45.2 From 240bd105f03491daf46f5e45a8f341e6b17cd773 Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 19 Jun 2024 11:05:52 +0200 Subject: [PATCH 34/35] draft: genre is optional --- music_kraken/development_cli/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/music_kraken/development_cli/__init__.py b/music_kraken/development_cli/__init__.py index 7cc3b15..664f2ae 100644 --- a/music_kraken/development_cli/__init__.py +++ b/music_kraken/development_cli/__init__.py @@ -73,5 +73,5 @@ def main(): description=DESCRIPTION, epilog='This is just a development cli. The real frontend is coming soon.' ) - parser.add_argument('--genre', '-g', action='store_const') + parser.add_argument('--genre', '-g', action='store_const', required=False, help="choose a genre to download from") args = parser.parse_args() -- 2.45.2 From e53e50b5d23d46d3f55723ff09487dae9c39687f Mon Sep 17 00:00:00 2001 From: Lars Noack Date: Wed, 19 Jun 2024 11:23:17 +0200 Subject: [PATCH 35/35] draft: proper help message --- music_kraken/development_cli/__init__.py | 35 +++++++++------- music_kraken/development_cli/genre.py | 53 ------------------------ music_kraken/development_cli/utils.py | 38 ++++++++++++++++- music_kraken/utils/enums/colors.py | 2 +- 4 files changed, 57 insertions(+), 71 deletions(-) delete mode 100644 music_kraken/development_cli/genre.py diff --git a/music_kraken/development_cli/__init__.py b/music_kraken/development_cli/__init__.py index 664f2ae..7c86dcd 100644 --- a/music_kraken/development_cli/__init__.py +++ b/music_kraken/development_cli/__init__.py @@ -5,21 +5,7 @@ from ..__meta__ import DESCRIPTION, PROGRAMM from ..download import Downloader from ..utils import BColors from ..utils.string_processing import unify -from .utils import ask_for_bool, ask_for_create - - -class Selection: - def __init__(self, options: list): - self.options = options - - def pprint(self): - return "\n".join(f"{i}: {option}" for i, option in enumerate(self.options)) - - def choose(self, input_: str): - try: - return self.options[int(input_)] - except (ValueError, IndexError): - return None +from .utils import HELP_MESSAGE, ask_for_bool, ask_for_create class DevelopmentCli: @@ -67,6 +53,25 @@ class DevelopmentCli: return genre + def help_screen(self) -> None: + print(HELP_MESSAGE) + + def shell(self) -> None: + print(f"Welcome to the {PROGRAMM} shell!") + print(f"Type '{BColors.OKBLUE}help{BColors.ENDC}' for a list of commands.") + print("") + + while True: + i = input("> ") + if i == "help": + self.help_screen() + elif i == "genre": + self.genre + elif i == "exit": + break + else: + print("Unknown command. Type 'help' for a list of commands.") + def main(): parser = argparse.ArgumentParser( prog=PROGRAMM, diff --git a/music_kraken/development_cli/genre.py b/music_kraken/development_cli/genre.py deleted file mode 100644 index 3580495..0000000 --- a/music_kraken/development_cli/genre.py +++ /dev/null @@ -1,53 +0,0 @@ -import re -from pathlib import Path -from typing import Dict, Generator, List, Set, Type, Union - -from ..download import Downloader, Page, components -from ..utils.config import main_settings -from .utils import ask_for_bool, cli_function - - -class GenreIO(components.HumanIO): - @staticmethod - def ask_to_create(option: components.Option) -> bool: - output() - return ask_for_bool(f"create the genre {BColors.OKBLUE.value}{option.value}{BColors.ENDC.value}") - - @staticmethod - def not_found(key: str) -> None: - output(f"\ngenre {BColors.BOLD.value}{key}{BColors.ENDC.value} not found\n", color=BColors.FAIL) - - -def _genre_generator() -> Generator[str, None, None]: - def is_valid_genre(genre: Path) -> bool: - """ - gets the name of all subdirectories of shared.MUSIC_DIR, - but filters out all directories, where the name matches with any Patern - from shared.NOT_A_GENRE_REGEX. - """ - if not genre.is_dir(): - return False - - if any(re.match(regex_pattern, genre.name) for regex_pattern in main_settings["not_a_genre_regex"]): - return False - - return True - - for genre in filter(is_valid_genre, main_settings["music_directory"].iterdir()): - yield genre.name - -def get_genre() -> str: - select_genre = components.Select( - human_io=GenreIO, - can_create_options=True, - data=_genre_generator(), - ) - genre: Optional[components.Option[str]] = None - - while genre is None: - print(select_genre.pprint()) - print() - - genre = select_genre.choose(input("> ")) - - return genre.value diff --git a/music_kraken/development_cli/utils.py b/music_kraken/development_cli/utils.py index b0fb1b8..830bec8 100644 --- a/music_kraken/development_cli/utils.py +++ b/music_kraken/development_cli/utils.py @@ -40,10 +40,44 @@ def print_cute_message(): print(message) -AGREE_INPUTS = {"y", "yes", "ok"} +def highlight_placeholder(text: str) -> str: + return text.replace("<", f"{BColors.BOLD}<").replace(">", f">{BColors.ENDC}") + + +HELP_MESSAGE = highlight_placeholder(f"""{BColors.HEADER}To search:{BColors.ENDC} +> s: +> s: https://musify.club/release/some-random-release-183028492 +> s: #a #r #t + +If you found the same object twice from different sources you can merge those objects. +Then it will use those sources. To do so, use the {BColors.BOLD}m{BColors.ENDC} command. + +{BColors.HEADER}To download:{BColors.ENDC} +> d: +> dm: 0, 3, 4 # merge all objects into one and download this object +> d: 1 +> d: https://musify.club/release/some-random-release-183028492 + +{BColors.HEADER}To inspect an object:{BColors.ENDC} +If you inspect an object, you see its adjacent object. This means for example the releases of an artist, or the tracks of a release. +You can also merge objects with the {BColors.BOLD}m{BColors.ENDC} command here. + +> g: +> gm: 0, 3, 4 # merge all objects into one and inspect this object +> g: 1 +> g: https://musify.club/release/some-random-release-183028492""") + + +class COMMANDS: + AGREE = {"y", "yes", "ok"} + DISAGREE = {"n", "no"} + EXIT = {"exit"} + HELP = {"help", "h"} + + def ask_for_bool(msg: str) -> bool: i = input(f"{msg} ({BColors.OKGREEN.value}Y{BColors.ENDC.value}/{BColors.FAIL.value}N{BColors.ENDC.value})? ").lower() - return i in AGREE_INPUTS + return i in COMMANDS.AGREE def ask_for_create(name: str, value: str) -> bool: diff --git a/music_kraken/utils/enums/colors.py b/music_kraken/utils/enums/colors.py index a9fac51..44f79da 100644 --- a/music_kraken/utils/enums/colors.py +++ b/music_kraken/utils/enums/colors.py @@ -1,7 +1,7 @@ from enum import Enum -class BColors(Enum): +class BColors: # https://stackoverflow.com/a/287944 HEADER = "\033[95m" OKBLUE = "\033[94m" -- 2.45.2