WIP: feature/cleanup_programming_interface #40

Draft
Hazel wants to merge 35 commits from feature/cleanup_programming_interface into experimental
Showing only changes of commit f839cdf906 - Show all commits

View File

@ -164,18 +164,36 @@ class Downloader:
for page in self.get_pages():
yield page, limit_generator(page.search(query=query), limit=results_per_page)
def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject:
def fetch_details(self, data_object: DataObject, **kwargs) -> DataObject:
"""Fetches more details for the given data object.
This uses every source contained in data_object.source_collection that has a page.
Args:
data_object (DataObject): The data object to fetch details for.
Returns:
DataObject: The same data object, but with more details.
"""
source: Source
for source in data_object.source_collection.get_sources(source_type_sorting={
"only_with_page": True,
}):
new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level)
new_data_object = self.fetch_from_source(source=source, **kwargs)
if new_data_object is not None:
data_object.merge(new_data_object)
return data_object
def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
"""Gets a data object from the given source.
Args:
source (Source): The source to get the data object from.
Returns:
Optional[DataObject]: If a data object can be retrieved, it is returned. Otherwise, None is returned.
"""
if not source.has_page:
return None
@ -192,6 +210,15 @@ class Downloader:
return data_object
def fetch_from_url(self, url: str) -> Optional[DataObject]:
"""This function tries to detect the source of the given url and fetches the data object from it.
Args:
url (str): The url to fetch the data object from.
Returns:
Optional[DataObject]: The fetched data object, or None if no source could be detected or if no object could be retrieved.
"""
source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
if source is None:
return None