draft:

music_kraken/pages/__init__.py

@@ -1,55 +1,9 @@
-from typing import Type, Generator, Set, Dict, List
 from collections import defaultdict
+from typing import Dict, Generator, List, Set, Type
 
+from ._bandcamp import Bandcamp
 from ._encyclopaedia_metallum import EncyclopaediaMetallum
+from ._genius import Genius
 from ._musify import Musify
 from ._youtube import YouTube
 from ._youtube_music import YoutubeMusic
-from ._bandcamp import Bandcamp
-from ._genius import Genius
-from ._abstract import Page, INDEPENDENT_DB_OBJECTS
-
-
-_registered_pages: Dict[Type[Page], Set[Page]] = defaultdict(set)
-
-
-def get_pages(*page_types: List[Type[Page]]) -> Generator[Page, None, None]:
-    if len(page_types) == 0:
-        page_types = _registered_pages.keys()
-
-    for page_type in page_types:
-        yield from _registered_pages[page_type]
-
-
-def register_page(page_type: Type[Page], **kwargs):
-    if page_type in _registered_pages:
-        return
-
-    _registered_pages[page_type].add(page_type(**kwargs))
-
-
-def deregister_page(page_type: Type[Page]):
-    if page_type not in _registered_pages:
-        return
-
-    for p in _registered_pages[page_type]:
-        p.__del__()
-    del _registered_pages[page_type]
-
-def scan_for_pages(**kwargs):
-    # assuming the wanted pages are the leaf classes of the interface
-    leaf_classes = []
-
-    _class_list = [Page]
-    while len(_class_list):
-        _class = _class_list.pop()
-        _class_subclasses = _class.__subclasses__()
-
-        if len(_class_subclasses) == 0:
-            if _class.REGISTER:
-                leaf_classes.append(_class)
-        else:
-            _class_list.extend(_class_subclasses)
-
-    for leaf_class in leaf_classes:
-        register_page(leaf_class, **kwargs)

music_kraken/pages/_abstract.py

@@ -1,152 +0,0 @@
-from __future__ import annotations
-
-import logging
-import random
-import re
-from copy import copy
-from pathlib import Path
-from typing import Optional, Union, Type, Dict, Set, List, Tuple, TypedDict, TYPE_CHECKING
-from string import Formatter
-from dataclasses import dataclass, field
-
-import requests
-from bs4 import BeautifulSoup
-
-if TYPE_CHECKING:
-    from ..download.page_attributes import DownloadOptions, FetchOptions
-from ..connection import Connection
-from ..objects import (
-    Song,
-    Source,
-    Album,
-    Artist,
-    Target,
-    DatabaseObject,
-    Options,
-    Collection,
-    Label,
-)
-from ..utils.enums import SourceType
-from ..utils.enums.album import AlbumType
-from ..audio import write_metadata_to_target, correct_codec
-from ..utils.config import main_settings
-from ..utils.support_classes.query import Query
-from ..utils.support_classes.download_result import DownloadResult
-from ..utils.string_processing import fit_to_file_system
-from ..utils import trace, output, BColors
-
-INDEPENDENT_DB_OBJECTS = Union[Label, Album, Artist, Song]
-INDEPENDENT_DB_TYPES = Union[Type[Song], Type[Album], Type[Artist], Type[Label]]
-
-
-class Page:
-    REGISTER = True
-    SOURCE_TYPE: SourceType
-    LOGGER: logging.Logger
-
-    def __new__(cls, *args, **kwargs):
-        cls.LOGGER = logging.getLogger(cls.__name__)
-        return super().__new__(cls)
-
-    def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs):
-        self.SOURCE_TYPE.register_page(self)
-        
-        self.download_options: DownloadOptions = download_options or DownloadOptions()
-        self.fetch_options: FetchOptions = fetch_options or FetchOptions()
-
-    def __del__(self):
-        self.SOURCE_TYPE.deregister_page()
-
-    def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None):
-        """
-        Perform a regex search on the given string, using a single or a list of
-        patterns returning the first matching group.
-        In case of failure return a default value or raise a WARNING or a
-        RegexNotFoundError, depending on fatal, specifying the field name.
-        """
-
-        if isinstance(pattern, str):
-            mobj = re.search(pattern, string, flags)
-        else:
-            for p in pattern:
-                mobj = re.search(p, string, flags)
-                if mobj:
-                    break
-
-        if mobj:
-            if group is None:
-                # return the first matching group
-                return next(g for g in mobj.groups() if g is not None)
-            elif isinstance(group, (list, tuple)):
-                return tuple(mobj.group(g) for g in group)
-            else:
-                return mobj.group(group)
-
-        return default
-
-    def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]:
-        return None
-
-    def get_soup_from_response(self, r: requests.Response) -> BeautifulSoup:
-        return BeautifulSoup(r.content, "html.parser")
-
-    # to search stuff
-    def search(self, query: Query) -> List[DatabaseObject]:
-        music_object = query.music_object
-
-        search_functions = {
-            Song: self.song_search,
-            Album: self.album_search,
-            Artist: self.artist_search,
-            Label: self.label_search
-        }
-
-        if type(music_object) in search_functions:
-            r = search_functions[type(music_object)](music_object)
-            if r is not None and len(r) > 0:
-                return r
-
-        r = []
-        for default_query in query.default_search:
-            for single_option in self.general_search(default_query):
-                r.append(single_option)
-
-        return r
-
-    def general_search(self, search_query: str) -> List[DatabaseObject]:
-        return []
-
-    def label_search(self, label: Label) -> List[Label]:
-        return []
-
-    def artist_search(self, artist: Artist) -> List[Artist]:
-        return []
-
-    def album_search(self, album: Album) -> List[Album]:
-        return []
-
-    def song_search(self, song: Song) -> List[Song]:
-        return []
-
-    # to fetch stuff
-    def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song:
-        return Song()
-
-    def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
-        return Album()
-
-    def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist:
-        return Artist()
-
-    def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label:
-        return Label()
-
-    # to download stuff
-    def get_skip_intervals(self, song: Song, source: Source) -> List[Tuple[float, float]]:
-        return []
-
-    def post_process_hook(self, song: Song, temp_target: Target, **kwargs):
-        pass
-
-    def download_song_to_target(self, source: Source, target: Target, desc: str = None) -> DownloadResult:
-        return DownloadResult()

music_kraken/pages/_bandcamp.py

@@ -1,33 +1,22 @@
-from typing import List, Optional, Type
-from urllib.parse import urlparse, urlunparse
 import json
 from enum import Enum
-from bs4 import BeautifulSoup
-import pycountry
+from typing import List, Optional, Type
+from urllib.parse import urlparse, urlunparse
+
+import pycountry
+from bs4 import BeautifulSoup
 
-from ..objects import Source, DatabaseObject
-from ._abstract import Page
-from ..objects import (
-    Artist,
-    Source,
-    SourceType,
-    Song,
-    Album,
-    Label,
-    Target,
-    Contact,
-    ID3Timestamp,
-    Lyrics,
-    FormattedText,
-    Artwork,
-)
 from ..connection import Connection
+from ..download import Page
+from ..objects import (Album, Artist, Artwork, Contact, DatabaseObject,
+                       FormattedText, ID3Timestamp, Label, Lyrics, Song,
+                       Source, SourceType, Target)
 from ..utils import dump_to_file
-from ..utils.enums import SourceType, ALL_SOURCE_TYPES
-from ..utils.support_classes.download_result import DownloadResult
-from ..utils.string_processing import clean_song_title
-from ..utils.config import main_settings, logging_settings
+from ..utils.config import logging_settings, main_settings
+from ..utils.enums import ALL_SOURCE_TYPES, SourceType
 from ..utils.shared import DEBUG
+from ..utils.string_processing import clean_song_title
+from ..utils.support_classes.download_result import DownloadResult
 
 if DEBUG:
     from ..utils import dump_to_file

music_kraken/pages/_encyclopaedia_metallum.py

@@ -1,31 +1,20 @@
 from collections import defaultdict
-from typing import List, Optional, Dict, Type, Union
-from bs4 import BeautifulSoup
+from typing import Dict, List, Optional, Type, Union
+from urllib.parse import urlencode, urlparse
+
 import pycountry
-from urllib.parse import urlparse, urlencode
+from bs4 import BeautifulSoup
 
 from ..connection import Connection
-from ..utils.config import logging_settings
-from ._abstract import Page
-from ..utils.enums import SourceType, ALL_SOURCE_TYPES
-from ..utils.enums.album import AlbumType
-from ..utils.support_classes.query import Query
-from ..objects import (
-    Lyrics,
-    Artist,
-    Source,
-    Song,
-    Album,
-    ID3Timestamp,
-    FormattedText,
-    Label,
-    Options,
-    DatabaseObject
-)
-from ..utils.shared import DEBUG
+from ..download import Page
+from ..objects import (Album, Artist, DatabaseObject, FormattedText,
+                       ID3Timestamp, Label, Lyrics, Options, Song, Source)
 from ..utils import dump_to_file
-
-
+from ..utils.config import logging_settings
+from ..utils.enums import ALL_SOURCE_TYPES, SourceType
+from ..utils.enums.album import AlbumType
+from ..utils.shared import DEBUG
+from ..utils.support_classes.query import Query
 
 ALBUM_TYPE_MAP: Dict[str, AlbumType] = defaultdict(lambda: AlbumType.OTHER, {
     "Full-length": AlbumType.STUDIO_ALBUM,

music_kraken/pages/_genius.py

@@ -1,33 +1,22 @@
-from typing import List, Optional, Type
-from urllib.parse import urlparse, urlunparse, urlencode
 import json
 from enum import Enum
-from bs4 import BeautifulSoup
-import pycountry
+from typing import List, Optional, Type
+from urllib.parse import urlencode, urlparse, urlunparse
+
+import pycountry
+from bs4 import BeautifulSoup
 
-from ..objects import Source, DatabaseObject
-from ._abstract import Page
-from ..objects import (
-    Artist,
-    Source,
-    SourceType,
-    Song,
-    Album,
-    Label,
-    Target,
-    Contact,
-    ID3Timestamp,
-    Lyrics,
-    FormattedText,
-    Artwork,
-)
 from ..connection import Connection
+from ..download import Page
+from ..objects import (Album, Artist, Artwork, Contact, DatabaseObject,
+                       FormattedText, ID3Timestamp, Label, Lyrics, Song,
+                       Source, SourceType, Target)
 from ..utils import dump_to_file, traverse_json_path
-from ..utils.enums import SourceType, ALL_SOURCE_TYPES
-from ..utils.support_classes.download_result import DownloadResult
-from ..utils.string_processing import clean_song_title
-from ..utils.config import main_settings, logging_settings
+from ..utils.config import logging_settings, main_settings
+from ..utils.enums import ALL_SOURCE_TYPES, SourceType
 from ..utils.shared import DEBUG
+from ..utils.string_processing import clean_song_title
+from ..utils.support_classes.download_result import DownloadResult
 
 if DEBUG:
     from ..utils import dump_to_file

music_kraken/pages/_musify.py

@@ -1,34 +1,23 @@
 from collections import defaultdict
 from dataclasses import dataclass
 from enum import Enum
-from typing import List, Optional, Type, Union, Generator, Dict, Any
+from typing import Any, Dict, Generator, List, Optional, Type, Union
 from urllib.parse import urlparse
 
 import pycountry
 from bs4 import BeautifulSoup
 
 from ..connection import Connection
-from ._abstract import Page
-from ..utils.enums import SourceType, ALL_SOURCE_TYPES
-from ..utils.enums.album import AlbumType, AlbumStatus
-from ..objects import (
-    Artist,
-    Source,
-    Song,
-    Album,
-    ID3Timestamp,
-    FormattedText,
-    Label,
-    Target,
-    DatabaseObject,
-    Lyrics,
-    Artwork
-)
+from ..download import Page
+from ..objects import (Album, Artist, Artwork, DatabaseObject, FormattedText,
+                       ID3Timestamp, Label, Lyrics, Song, Source, Target)
+from ..utils import shared, string_processing
 from ..utils.config import logging_settings, main_settings
-from ..utils import string_processing, shared
+from ..utils.enums import ALL_SOURCE_TYPES, SourceType
+from ..utils.enums.album import AlbumStatus, AlbumType
 from ..utils.string_processing import clean_song_title
-from ..utils.support_classes.query import Query
 from ..utils.support_classes.download_result import DownloadResult
+from ..utils.support_classes.query import Query
 
 """
 https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent

music_kraken/pages/_youtube.py

@@ -1,29 +1,19 @@
-from typing import List, Optional, Type, Tuple
-from urllib.parse import urlparse, urlunparse, parse_qs
 from enum import Enum
+from typing import List, Optional, Tuple, Type
+from urllib.parse import parse_qs, urlparse, urlunparse
 
 import python_sponsorblock
 
-from ..objects import Source, DatabaseObject, Song, Target
-from ._abstract import Page
-from ..objects import (
-    Artist,
-    Source,
-    Song,
-    Album,
-    Label,
-    Target,
-    FormattedText,
-    ID3Timestamp
-)
 from ..connection import Connection
+from ..download import Page
+from ..objects import (Album, Artist, DatabaseObject, FormattedText,
+                       ID3Timestamp, Label, Song, Source, Target)
+from ..utils.config import logging_settings, main_settings, youtube_settings
+from ..utils.enums import ALL_SOURCE_TYPES, SourceType
 from ..utils.string_processing import clean_song_title
-from ..utils.enums import SourceType, ALL_SOURCE_TYPES
 from ..utils.support_classes.download_result import DownloadResult
-from ..utils.config import youtube_settings, main_settings, logging_settings
-
-from ._youtube_music.super_youtube import SuperYouTube, YouTubeUrl, get_invidious_url, YouTubeUrlType
-
+from ._youtube_music.super_youtube import (SuperYouTube, YouTubeUrl,
+                                           YouTubeUrlType, get_invidious_url)
 
 """
 - https://yt.artemislena.eu/api/v1/search?q=Zombiez+-+Topic&page=1&date=none&type=channel&duration=none&sort=relevance

music_kraken/pages/_youtube_music/youtube_music.py

@@ -1,46 +1,33 @@
-from __future__ import unicode_literals, annotations
+from __future__ import annotations, unicode_literals
 
-from typing import Dict, List, Optional, Set, Type
-from urllib.parse import urlparse, urlunparse, quote, parse_qs, urlencode
+import json
 import logging
 import random
-import json
-from dataclasses import dataclass
 import re
-from functools import lru_cache
 from collections import defaultdict
+from dataclasses import dataclass
+from functools import lru_cache
+from typing import Dict, List, Optional, Set, Type
+from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse
 
 import youtube_dl
 from youtube_dl.extractor.youtube import YoutubeIE
 from youtube_dl.utils import DownloadError
 
+from ...connection import Connection
+from ...download import Page
+from ...objects import Album, Artist, Artwork
+from ...objects import DatabaseObject as DataObject
+from ...objects import (FormattedText, ID3Timestamp, Label, Lyrics, Song,
+                        Source, Target)
+from ...utils import dump_to_file, get_current_millis, traverse_json_path
+from ...utils.config import logging_settings, main_settings, youtube_settings
+from ...utils.enums import ALL_SOURCE_TYPES, SourceType
+from ...utils.enums.album import AlbumType
 from ...utils.exception.config import SettingValueError
-from ...utils.config import main_settings, youtube_settings, logging_settings
 from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING
 from ...utils.string_processing import clean_song_title
-from ...utils import get_current_millis, traverse_json_path
-
-from ...utils import dump_to_file
-
-from .._abstract import Page
-from ...objects import (
-    DatabaseObject as DataObject,
-    Source,
-    FormattedText,
-    ID3Timestamp,
-    Artwork,
-    Artist,
-    Song,
-    Album,
-    Label,
-    Target,
-    Lyrics,
-)
-from ...connection import Connection
-from ...utils.enums import SourceType, ALL_SOURCE_TYPES
-from ...utils.enums.album import AlbumType
 from ...utils.support_classes.download_result import DownloadResult
-
 from ._list_render import parse_renderer
 from ._music_object_render import parse_run_element
 from .super_youtube import SuperYouTube