feat: commented fetch functions
This commit is contained in:
		@@ -164,18 +164,36 @@ class Downloader:
 | 
				
			|||||||
        for page in self.get_pages():
 | 
					        for page in self.get_pages():
 | 
				
			||||||
            yield page, limit_generator(page.search(query=query), limit=results_per_page)
 | 
					            yield page, limit_generator(page.search(query=query), limit=results_per_page)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def fetch_details(self, data_object: DataObject, stop_at_level: int = 1, **kwargs) -> DataObject:
 | 
					    def fetch_details(self, data_object: DataObject, **kwargs) -> DataObject:
 | 
				
			||||||
 | 
					        """Fetches more details for the given data object.
 | 
				
			||||||
 | 
					        This uses every source contained in data_object.source_collection that has a page.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Args:
 | 
				
			||||||
 | 
					            data_object (DataObject): The data object to fetch details for.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Returns:
 | 
				
			||||||
 | 
					            DataObject: The same data object, but with more details.
 | 
				
			||||||
 | 
					        """
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        source: Source
 | 
					        source: Source
 | 
				
			||||||
        for source in data_object.source_collection.get_sources(source_type_sorting={
 | 
					        for source in data_object.source_collection.get_sources(source_type_sorting={
 | 
				
			||||||
            "only_with_page": True,
 | 
					            "only_with_page": True,
 | 
				
			||||||
        }):
 | 
					        }):
 | 
				
			||||||
            new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level)
 | 
					            new_data_object = self.fetch_from_source(source=source, **kwargs)
 | 
				
			||||||
            if new_data_object is not None:
 | 
					            if new_data_object is not None:
 | 
				
			||||||
                data_object.merge(new_data_object)
 | 
					                data_object.merge(new_data_object)
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        return data_object
 | 
					        return data_object
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
 | 
					    def fetch_from_source(self, source: Source, **kwargs) -> Optional[DataObject]:
 | 
				
			||||||
 | 
					        """Gets a data object from the given source.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Args:
 | 
				
			||||||
 | 
					            source (Source): The source to get the data object from.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Returns:
 | 
				
			||||||
 | 
					            Optional[DataObject]: If a data object can be retrieved, it is returned. Otherwise, None is returned.
 | 
				
			||||||
 | 
					        """
 | 
				
			||||||
        if not source.has_page:
 | 
					        if not source.has_page:
 | 
				
			||||||
            return None
 | 
					            return None
 | 
				
			||||||
        
 | 
					        
 | 
				
			||||||
@@ -192,6 +210,15 @@ class Downloader:
 | 
				
			|||||||
        return data_object
 | 
					        return data_object
 | 
				
			||||||
 | 
					
 | 
				
			||||||
    def fetch_from_url(self, url: str) -> Optional[DataObject]:
 | 
					    def fetch_from_url(self, url: str) -> Optional[DataObject]:
 | 
				
			||||||
 | 
					        """This function tries to detect the source of the given url and fetches the data object from it.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Args:
 | 
				
			||||||
 | 
					            url (str): The url to fetch the data object from.
 | 
				
			||||||
 | 
					
 | 
				
			||||||
 | 
					        Returns:
 | 
				
			||||||
 | 
					            Optional[DataObject]: The fetched data object, or None if no source could be detected or if no object could be retrieved.
 | 
				
			||||||
 | 
					        """
 | 
				
			||||||
 | 
					
 | 
				
			||||||
        source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
 | 
					        source = Source.match_url(url, ALL_SOURCE_TYPES.MANUAL)
 | 
				
			||||||
        if source is None:
 | 
					        if source is None:
 | 
				
			||||||
            return None
 | 
					            return None
 | 
				
			||||||
 
 | 
				
			|||||||
		Reference in New Issue
	
	Block a user