Compare commits

...

3 Commits

Author SHA1 Message Date
0f2229b0f2 feat: completely dynamified the datasource import
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
2024-05-24 17:00:39 +02:00
5af95f1b03 feat: auto import pages in page module 2024-05-24 15:46:42 +02:00
c24cf701c1 fix: pages were not in the subclasses because the module was never importet 2024-05-24 15:28:47 +02:00
2 changed files with 54 additions and 4 deletions

View File

@ -94,6 +94,8 @@ class Downloader:
def scan_for_pages(self, **kwargs): def scan_for_pages(self, **kwargs):
# assuming the wanted pages are the leaf classes of the interface # assuming the wanted pages are the leaf classes of the interface
from .. import pages
leaf_classes = [] leaf_classes = []
class_list = [Page] class_list = [Page]
@ -107,6 +109,9 @@ class Downloader:
else: else:
class_list.extend(class_subclasses) class_list.extend(class_subclasses)
if Page in leaf_classes:
self.LOGGER.warn("couldn't find any data source")
return
for leaf_class in leaf_classes: for leaf_class in leaf_classes:
self.register_page(leaf_class, **kwargs) self.register_page(leaf_class, **kwargs)
@ -342,16 +347,18 @@ class Page:
cls.LOGGER = logging.getLogger(cls.__name__) cls.LOGGER = logging.getLogger(cls.__name__)
return super().__new__(cls) return super().__new__(cls)
@classmethod
def is_leaf_page(cls) -> bool:
return len(cls.__subclasses__()) == 0
def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs): def __init__(self, download_options: DownloadOptions = None, fetch_options: FetchOptions = None, **kwargs):
if self.SOURCE_TYPE is not None: self.SOURCE_TYPE.register_page(self)
self.SOURCE_TYPE.register_page(self)
self.download_options: DownloadOptions = download_options or DownloadOptions() self.download_options: DownloadOptions = download_options or DownloadOptions()
self.fetch_options: FetchOptions = fetch_options or FetchOptions() self.fetch_options: FetchOptions = fetch_options or FetchOptions()
def __del__(self): def __del__(self):
if self.SOURCE_TYPE is not None: self.SOURCE_TYPE.deregister_page()
self.SOURCE_TYPE.deregister_page()
def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None): def _search_regex(self, pattern, string, default=None, fatal=True, flags=0, group=None):
""" """

View File

@ -1,4 +1,11 @@
import importlib
import inspect
import logging
import pkgutil
import sys
from collections import defaultdict from collections import defaultdict
from copy import copy
from pathlib import Path
from typing import Dict, Generator, List, Set, Type from typing import Dict, Generator, List, Set, Type
from ._bandcamp import Bandcamp from ._bandcamp import Bandcamp
@ -7,3 +14,39 @@ from ._genius import Genius
from ._musify import Musify from ._musify import Musify
from ._youtube import YouTube from ._youtube import YouTube
from ._youtube_music import YoutubeMusic from ._youtube_music import YoutubeMusic
def import_children():
_page_directory = Path(__file__).parent
_stem_blacklist = set(["__pycache__", "__init__"])
for _file in _page_directory.iterdir():
if _file.stem in _stem_blacklist:
continue
logging.debug(f"importing {_file.absolute()}")
exec(f"from . import {_file.stem}")
# module_blacklist = set(sys.modules.keys())
import_children()
"""
classes = set()
print(__name__)
for module_name, module in sys.modules.items():
if module_name in module_blacklist or not module_name.startswith(__name__):
continue
print("scanning module", module_name)
for name, obj in inspect.getmembers(module, predicate=inspect.isclass):
_module = obj.__module__
if _module.startswith(__name__) and hasattr(obj, "SOURCE_TYPE"):
print("checking object", name, obj.__module__)
classes.add(obj)
print()
print(*(c.__name__ for c in classes), sep=",\t")
__all__ = [c.__name__ for c in classes]
"""