feat: commented fetch functions
This commit is contained in:
parent
c306da7934
commit
f839cdf906
@ -164,18 +164,36 @@ class Downloader:
|
|||||||
for page in self.get_pages():
|
for page in self.get_pages():
|
||||||
yield page, limit_generator(page.search(query=query), limit=results_per_page)
|
yield page, limit_generator(page.search(query=query), limit=results_per_page)
|
||||||
|
|
||||||
def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject:
|
def fetch_details(self, data_object: DataObject, **kwargs) -> DataObject:
|
||||||
|
"""Fetches more details for the given data object.
|
||||||
|
This uses every source contained in data_object.source_collection that has a page.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
data_object (DataObject): The data object to fetch details for.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
DataObject: The same data object, but with more details.
|
||||||
|
"""
|
||||||
|
|
||||||
source: Source
|
source: Source
|
||||||
for source in data_object.source_collection.get_sources(source_type_sorting={
|
for source in data_object.source_collection.get_sources(source_type_sorting={
|
||||||
"only_with_page": True,
|
"only_with_page": True,
|
||||||
}):
|
}):
|
||||||
new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level)
|
new_data_object = self.fetch_from_source(source=source, **kwargs)
|
||||||
if new_data_object is not None:
|
if new_data_object is not None:
|
||||||
data_object.merge(new_data_object)
|
data_object.merge(new_data_object)
|
||||||
|
|
||||||
return data_object
|
return data_object
|
||||||
|
|
||||||
def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
|
def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
|
||||||
|
"""Gets a data object from the given source.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
source (Source): The source to get the data object from.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[DataObject]: If a data object can be retrieved, it is returned. Otherwise, None is returned.
|
||||||
|
"""
|
||||||
if not source.has_page:
|
if not source.has_page:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@ -192,6 +210,15 @@ class Downloader:
|
|||||||
return data_object
|
return data_object
|
||||||
|
|
||||||
def fetch_from_url(self, url: str) -> Optional[DataObject]:
|
def fetch_from_url(self, url: str) -> Optional[DataObject]:
|
||||||
|
"""This function tries to detect the source of the given url and fetches the data object from it.
|
||||||
|
|
||||||
|
Args:
|
||||||
|
url (str): The url to fetch the data object from.
|
||||||
|
|
||||||
|
Returns:
|
||||||
|
Optional[DataObject]: The fetched data object, or None if no source could be detected or if no object could be retrieved.
|
||||||
|
"""
|
||||||
|
|
||||||
source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
|
source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
|
||||||
if source is None:
|
if source is None:
|
||||||
return None
|
return None
|
||||||
|
Loading…
Reference in New Issue
Block a user