feat: commented fetch functions
This commit is contained in:
parent
c306da7934
commit
f839cdf906
@ -164,18 +164,36 @@ class Downloader:
|
||||
for page in self.get_pages():
|
||||
yield page, limit_generator(page.search(query=query), limit=results_per_page)
|
||||
|
||||
def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject:
|
||||
def fetch_details(self, data_object: DataObject, **kwargs) -> DataObject:
|
||||
"""Fetches more details for the given data object.
|
||||
This uses every source contained in data_object.source_collection that has a page.
|
||||
|
||||
Args:
|
||||
data_object (DataObject): The data object to fetch details for.
|
||||
|
||||
Returns:
|
||||
DataObject: The same data object, but with more details.
|
||||
"""
|
||||
|
||||
source: Source
|
||||
for source in data_object.source_collection.get_sources(source_type_sorting={
|
||||
"only_with_page": True,
|
||||
}):
|
||||
new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level)
|
||||
new_data_object = self.fetch_from_source(source=source, **kwargs)
|
||||
if new_data_object is not None:
|
||||
data_object.merge(new_data_object)
|
||||
|
||||
return data_object
|
||||
|
||||
def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
|
||||
"""Gets a data object from the given source.
|
||||
|
||||
Args:
|
||||
source (Source): The source to get the data object from.
|
||||
|
||||
Returns:
|
||||
Optional[DataObject]: If a data object can be retrieved, it is returned. Otherwise, None is returned.
|
||||
"""
|
||||
if not source.has_page:
|
||||
return None
|
||||
|
||||
@ -192,6 +210,15 @@ class Downloader:
|
||||
return data_object
|
||||
|
||||
def fetch_from_url(self, url: str) -> Optional[DataObject]:
|
||||
"""This function tries to detect the source of the given url and fetches the data object from it.
|
||||
|
||||
Args:
|
||||
url (str): The url to fetch the data object from.
|
||||
|
||||
Returns:
|
||||
Optional[DataObject]: The fetched data object, or None if no source could be detected or if no object could be retrieved.
|
||||
"""
|
||||
|
||||
source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
|
||||
if source is None:
|
||||
return None
|
||||
|
Loading…
Reference in New Issue
Block a user