feat: completely dynamified the datasource import
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful

This commit is contained in:
Hazel 2024-05-24 17:00:39 +02:00
parent 5af95f1b03
commit 0f2229b0f2
2 changed files with 42 additions and 11 deletions

View File

@ -78,8 +78,6 @@ class Downloader:
if page_type in self._registered_pages: if page_type in self._registered_pages:
return return
print(page_type)
self._registered_pages[page_type].add(page_type( self._registered_pages[page_type].add(page_type(
download_options=self.download_options, download_options=self.download_options,
fetch_options=self.fetch_options, fetch_options=self.fetch_options,
@ -97,6 +95,7 @@ class Downloader:
def scan_for_pages(self, **kwargs): def scan_for_pages(self, **kwargs):
# assuming the wanted pages are the leaf classes of the interface # assuming the wanted pages are the leaf classes of the interface
from .. import pages from .. import pages
leaf_classes = [] leaf_classes = []
class_list = [Page] class_list = [Page]
@ -110,6 +109,9 @@ class Downloader:
else: else:
class_list.extend(class_subclasses) class_list.extend(class_subclasses)
if Page in leaf_classes:
self.LOGGER.warn("couldn't find any data source")
return
for leaf_class in leaf_classes: for leaf_class in leaf_classes:
self.register_page(leaf_class, **kwargs) self.register_page(leaf_class, **kwargs)

View File

@ -1,17 +1,22 @@
import importlib
import inspect
import logging import logging
import pkgutil
import sys
from collections import defaultdict from collections import defaultdict
from copy import copy
from pathlib import Path from pathlib import Path
from typing import Dict, Generator, List, Set, Type from typing import Dict, Generator, List, Set, Type
"""
from ._bandcamp import Bandcamp from ._bandcamp import Bandcamp
from ._encyclopaedia_metallum import EncyclopaediaMetallum from ._encyclopaedia_metallum import EncyclopaediaMetallum
from ._genius import Genius from ._genius import Genius
from ._musify import Musify from ._musify import Musify
from ._youtube import YouTube from ._youtube import YouTube
from ._youtube_music import YoutubeMusic from ._youtube_music import YoutubeMusic
"""
def import_children():
_page_directory = Path(__file__).parent _page_directory = Path(__file__).parent
_stem_blacklist = set(["__pycache__", "__init__"]) _stem_blacklist = set(["__pycache__", "__init__"])
@ -21,3 +26,27 @@ for _file in _page_directory.iterdir():
logging.debug(f"importing {_file.absolute()}") logging.debug(f"importing {_file.absolute()}")
exec(f"from . import {_file.stem}") exec(f"from . import {_file.stem}")
# module_blacklist = set(sys.modules.keys())
import_children()
"""
classes = set()
print(__name__)
for module_name, module in sys.modules.items():
if module_name in module_blacklist or not module_name.startswith(__name__):
continue
print("scanning module", module_name)
for name, obj in inspect.getmembers(module, predicate=inspect.isclass):
_module = obj.__module__
if _module.startswith(__name__) and hasattr(obj, "SOURCE_TYPE"):
print("checking object", name, obj.__module__)
classes.add(obj)
print()
print(*(c.__name__ for c in classes), sep=",\t")
__all__ = [c.__name__ for c in classes]
"""