layed out the logging config

This commit is contained in:
Hellow 2023-04-13 23:45:50 +02:00
parent 1c5c390f20
commit 421ab5d4fb
4 changed files with 145 additions and 12 deletions

View File

@ -4,9 +4,8 @@ from typing import List, Tuple
from ...utils.shared import SHOW_DOWNLOAD_ERRORS_THRESHOLD, DOWNLOAD_LOGGER as LOGGER from ...utils.shared import SHOW_DOWNLOAD_ERRORS_THRESHOLD, DOWNLOAD_LOGGER as LOGGER
from ...objects import Target from ...objects import Target
UNIT_PREFIXES: List[str] = ["", "k", "m", "g", "t"] UNIT_PREFIXES: List[str] = ["", "k", "m", "g", "t"]
UNIT_DIVISOR=1024 UNIT_DIVISOR = 1024
@dataclass @dataclass
@ -44,20 +43,20 @@ class DownloadResult:
return True return True
return self.failure_percentage > SHOW_DOWNLOAD_ERRORS_THRESHOLD return self.failure_percentage > SHOW_DOWNLOAD_ERRORS_THRESHOLD
def _size_val_unit_pref_ind(self, val: float, ind: int) -> Tuple[float, int]: def _size_val_unit_pref_ind(self, val: float, ind: int) -> Tuple[float, int]:
if val < UNIT_DIVISOR: if val < UNIT_DIVISOR:
return val, ind return val, ind
if ind >= len(UNIT_PREFIXES): if ind >= len(UNIT_PREFIXES):
return val, ind return val, ind
return self._size_val_unit_pref_ind(val=val/UNIT_DIVISOR, ind=ind+1) return self._size_val_unit_pref_ind(val=val / UNIT_DIVISOR, ind=ind + 1)
@property @property
def formated_size(self) -> str: def formated_size(self) -> str:
total_size, prefix_index = self._size_val_unit_pref_ind(self.total_size, 0) total_size, prefix_index = self._size_val_unit_pref_ind(self.total_size, 0)
return f"{total_size:.{2}f} {UNIT_PREFIXES[prefix_index]}B" return f"{total_size:.{2}f} {UNIT_PREFIXES[prefix_index]}B"
def add_target(self, target: Target): def add_target(self, target: Target):
self.total_size += target.size self.total_size += target.size
@ -71,15 +70,15 @@ class DownloadResult:
self.total += other.total self.total += other.total
self.fail += other.fail self.fail += other.fail
self._error_message_list.extend(other._error_message_list) self._error_message_list.extend(other._error_message_list)
self.total_size += other.total_size self.total_size += other.total_size
def __str__(self): def __str__(self):
if self.is_fatal_error: if self.is_fatal_error:
return self.error_message return self.error_message
head = f"{self.fail} from {self.total} downloads failed:\n" \ head = f"{self.fail} from {self.total} downloads failed:\n" \
f"successrate:\t{int(self.success_percentage*100)}%\n" \ f"successrate:\t{int(self.success_percentage * 100)}%\n" \
f"failrate:\t{int(self.failure_percentage*100)}%\n" \ f"failrate:\t{int(self.failure_percentage * 100)}%\n" \
f"total size:\t{self.formated_size}" f"total size:\t{self.formated_size}"
if not self.is_mild_failure: if not self.is_mild_failure:

View File

@ -0,0 +1,34 @@
from dataclasses import dataclass
from typing import Optional, List, Union
@dataclass
class Attribute:
name: str
description: Optional[str]
value: Union[str, List[str]]
@property
def object_from_value(self):
return self.value
class SingleAttribute(Attribute):
value: str
class StringAttribute(Attribute):
@property
def object_from_value(self) -> str:
return self.value
class ListAttribute(Attribute):
value: List[str]
class Section:
"""
A placeholder class
"""
pass

View File

@ -0,0 +1,101 @@
import logging
from typing import Callable
from .config import SingleAttribute, ListAttribute, StringAttribute, Section
LOG_LEVELS = {
"CRITICAL": 50,
"ERROR": 40,
"WARNING": 30,
"INFO": 20,
"DEBUG": 10,
"NOTSET": 0
}
class LoggerAttribute(SingleAttribute):
@property
def object_from_value(self) -> logging.Logger:
return logging.getLogger(self.value)
class LogLevelAttribute(SingleAttribute):
@property
def object_from_value(self) -> int:
"""
gets the numeric value of a log level
:return:
"""
if self.value.isnumeric():
return int(self.value)
v = self.value.strip().upper()
if v not in LOG_LEVELS:
raise ValueError(
f"{self.name} can only been either one of the following levels, or an integer:\n"
f"{';'.join(key for key in LOG_LEVELS)}"
)
return LOG_LEVELS[v]
class LoggingSection(Section):
def __init__(self):
self.FORMAT = StringAttribute(
name="logging_format",
description="Reference for the logging formats: https://docs.python.org/3/library/logging.html#logrecord-attributes",
value=logging.BASIC_FORMAT
)
self.LOG_LEVEL = LogLevelAttribute(
name="log_level",
description=f"can only been either one of the following levels, or an integer:\n"
f"{';'.join(key for key in LOG_LEVELS)}",
value=str(logging.INFO)
)
self.DOWNLOAD_LOGGER = LoggerAttribute(
name="download_logger",
description="The logger for downloading.",
value="download"
)
self.TAGGING_LOGGER = LoggerAttribute(
name="tagging_logger",
description="The logger for tagging id3 containers.",
value="tagging"
)
self.CODEX_LOGGER = LoggerAttribute(
name="codex_logger",
description="The logger for streaming the audio into an uniform codex.",
value="codex"
)
self.OBJECT_LOGGER = LoggerAttribute(
name="object_logger",
description="The logger for creating Data-Objects.",
value="object"
)
self.DATABASE_LOGGER = LoggerAttribute(
name="database_logger",
description="The logger for Database operations.",
value="database"
)
self.MUSIFY_LOGGER = LoggerAttribute(
name="musify_logger",
description="The logger for the musify scraper.",
value="musify"
)
self.YOUTUBE_LOGGER = LoggerAttribute(
name="youtube_logger",
description="The logger for the youtube scraper.",
value="youtube"
)
self.ENCYCLOPAEDIA_METALLUM_LOGGER = LoggerAttribute(
name="metal_archives_logger",
description="The logger for the metal archives scraper.",
value="metal_archives"
)
self.GENIUS_LOGGER = LoggerAttribute(
name="genius_logger",
description="The logger for the genius scraper",
value="genius"
)

View File

@ -47,13 +47,12 @@ logging.basicConfig(
level=logging.INFO, level=logging.INFO,
format=logging.BASIC_FORMAT, format=logging.BASIC_FORMAT,
handlers=[ handlers=[
logging.FileHandler(Path(TEMP_DIR, LOG_PATH)), logging.FileHandler(LOG_PATH),
logging.StreamHandler() logging.StreamHandler()
] ]
) )
OBJECT_LOGGER = logging.getLogger("objects") OBJECT_LOGGER = logging.getLogger("objects")
TARGET_LOGGER = logging.getLogger("target")
DATABASE_LOGGER = logging.getLogger("database") DATABASE_LOGGER = logging.getLogger("database")
YOUTUBE_LOGGER = logging.getLogger("Youtube") YOUTUBE_LOGGER = logging.getLogger("Youtube")