feat: completely dynamified the datasource import
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
Hazel 2024-05-24 17:00:39 +02:00
parent 5af95f1b03
commit 0f2229b0f2
2 changed files with 42 additions and 11 deletions

View File

@ -78,8 +78,6 @@ class Downloader:
if page_type in self._registered_pages:
return
print(page_type)
self._registered_pages[page_type].add(page_type(
download_options=self.download_options,
fetch_options=self.fetch_options,
@ -97,6 +95,7 @@ class Downloader:
def scan_for_pages(self, **kwargs):
# assuming the wanted pages are the leaf classes of the interface
from .. import pages
leaf_classes = []
class_list = [Page]
@ -110,6 +109,9 @@ class Downloader:
else:
class_list.extend(class_subclasses)
if Page in leaf_classes:
self.LOGGER.warn("couldn't find any data source")
return
for leaf_class in leaf_classes:
self.register_page(leaf_class, **kwargs)

View File

@ -1,23 +1,52 @@
import importlib
import inspect
import logging
import pkgutil
import sys
from collections import defaultdict
from copy import copy
from pathlib import Path
from typing import Dict, Generator, List, Set, Type
"""
from ._bandcamp import Bandcamp
from ._encyclopaedia_metallum import EncyclopaediaMetallum
from ._genius import Genius
from ._musify import Musify
from ._youtube import YouTube
from ._youtube_music import YoutubeMusic
"""
_page_directory = Path(__file__).parent
_stem_blacklist = set(["__pycache__", "__init__"])
for _file in _page_directory.iterdir():
def import_children():
_page_directory = Path(__file__).parent
_stem_blacklist = set(["__pycache__", "__init__"])
for _file in _page_directory.iterdir():
if _file.stem in _stem_blacklist:
continue
logging.debug(f"importing {_file.absolute()}")
exec(f"from . import {_file.stem}")
# module_blacklist = set(sys.modules.keys())
import_children()
"""
classes = set()
print(__name__)
for module_name, module in sys.modules.items():
if module_name in module_blacklist or not module_name.startswith(__name__):
continue
print("scanning module", module_name)
for name, obj in inspect.getmembers(module, predicate=inspect.isclass):
_module = obj.__module__
if _module.startswith(__name__) and hasattr(obj, "SOURCE_TYPE"):
print("checking object", name, obj.__module__)
classes.add(obj)
print()
print(*(c.__name__ for c in classes), sep=",\t")
__all__ = [c.__name__ for c in classes]
"""