Compare commits
2 Commits
experiment
...
feature/mu
Author | SHA1 | Date | |
---|---|---|---|
265c9f462f | |||
780daac0ef |
1
.vscode/settings.json
vendored
1
.vscode/settings.json
vendored
@ -20,7 +20,6 @@
|
|||||||
"APIC",
|
"APIC",
|
||||||
"Bandcamp",
|
"Bandcamp",
|
||||||
"bitrate",
|
"bitrate",
|
||||||
"CALLSTACK",
|
|
||||||
"DEEZER",
|
"DEEZER",
|
||||||
"dotenv",
|
"dotenv",
|
||||||
"encyclopaedia",
|
"encyclopaedia",
|
||||||
|
@ -1,13 +1,15 @@
|
|||||||
import logging
|
|
||||||
|
|
||||||
import music_kraken
|
import music_kraken
|
||||||
|
|
||||||
|
import logging
|
||||||
print("Setting logging-level to DEBUG")
|
print("Setting logging-level to DEBUG")
|
||||||
logging.getLogger().setLevel(logging.DEBUG)
|
logging.getLogger().setLevel(logging.DEBUG)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
commands = [
|
commands = [
|
||||||
"s: #a Ghost Bath",
|
"s: #a Crystal F",
|
||||||
|
"10",
|
||||||
|
"1",
|
||||||
|
"3",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@ -1,21 +1,21 @@
|
|||||||
import logging
|
import mutagen
|
||||||
|
from mutagen.id3 import ID3, Frame, APIC, USLT
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import List
|
from typing import List
|
||||||
|
import logging
|
||||||
import mutagen
|
|
||||||
from mutagen.id3 import APIC, ID3, USLT, Frame
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
from ..connection import Connection
|
|
||||||
from ..objects import Metadata, Song, Target
|
|
||||||
from ..objects.metadata import Mapping
|
|
||||||
from ..utils.config import logging_settings, main_settings
|
from ..utils.config import logging_settings, main_settings
|
||||||
|
from ..objects import Song, Target, Metadata
|
||||||
|
from ..objects.metadata import Mapping
|
||||||
|
from ..connection import Connection
|
||||||
|
|
||||||
LOGGER = logging_settings["tagging_logger"]
|
LOGGER = logging_settings["tagging_logger"]
|
||||||
|
|
||||||
|
|
||||||
artwork_connection: Connection = Connection()
|
artwork_connection: Connection = Connection()
|
||||||
|
|
||||||
|
|
||||||
class AudioMetadata:
|
class AudioMetadata:
|
||||||
def __init__(self, file_location: str = None) -> None:
|
def __init__(self, file_location: str = None) -> None:
|
||||||
self._file_location = None
|
self._file_location = None
|
||||||
@ -67,14 +67,13 @@ def write_metadata_to_target(metadata: Metadata, target: Target, song: Song):
|
|||||||
id3_object = AudioMetadata(file_location=target.file_path)
|
id3_object = AudioMetadata(file_location=target.file_path)
|
||||||
|
|
||||||
LOGGER.info(str(metadata))
|
LOGGER.info(str(metadata))
|
||||||
## REWRITE COMPLETLY !!!!!!!!!!!!
|
|
||||||
if len(song.artwork._data) != 0:
|
if song.artwork.best_variant is not None:
|
||||||
variants = song.artwork._data.__getitem__(0)
|
best_variant = song.artwork.best_variant
|
||||||
best_variant = variants.variants.__getitem__(0)
|
|
||||||
|
|
||||||
r = artwork_connection.get(
|
r = artwork_connection.get(
|
||||||
url=best_variant.url,
|
url=best_variant["url"],
|
||||||
name=best_variant.url,
|
name=song.artwork.get_variant_name(best_variant),
|
||||||
)
|
)
|
||||||
|
|
||||||
temp_target: Target = Target.temp()
|
temp_target: Target = Target.temp()
|
||||||
@ -108,7 +107,7 @@ def write_metadata_to_target(metadata: Metadata, target: Target, song: Song):
|
|||||||
mime="image/jpeg",
|
mime="image/jpeg",
|
||||||
type=3,
|
type=3,
|
||||||
desc=u"Cover",
|
desc=u"Cover",
|
||||||
data=converted_target.raw_content,
|
data=converted_target.read_bytes(),
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
id3_object.frames.delall("USLT")
|
id3_object.frames.delall("USLT")
|
||||||
|
@ -1,12 +1,12 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import copy
|
|
||||||
import inspect
|
|
||||||
import logging
|
import logging
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from typing import TYPE_CHECKING, Dict, List, Optional, Set
|
from typing import List, Dict, Optional, Set
|
||||||
from urllib.parse import ParseResult, urlparse, urlunsplit
|
from urllib.parse import urlparse, urlunsplit, ParseResult
|
||||||
|
import copy
|
||||||
|
import inspect
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
import responses
|
import responses
|
||||||
@ -14,15 +14,12 @@ from tqdm import tqdm
|
|||||||
|
|
||||||
from .cache import Cache
|
from .cache import Cache
|
||||||
from .rotating import RotatingProxy
|
from .rotating import RotatingProxy
|
||||||
|
from ..objects import Target
|
||||||
if TYPE_CHECKING:
|
|
||||||
from ..objects import Target
|
|
||||||
|
|
||||||
from ..utils import request_trace
|
from ..utils import request_trace
|
||||||
from ..utils.config import main_settings
|
|
||||||
from ..utils.hacking import merge_args
|
|
||||||
from ..utils.string_processing import shorten_display_url
|
from ..utils.string_processing import shorten_display_url
|
||||||
|
from ..utils.config import main_settings
|
||||||
from ..utils.support_classes.download_result import DownloadResult
|
from ..utils.support_classes.download_result import DownloadResult
|
||||||
|
from ..utils.hacking import merge_args
|
||||||
|
|
||||||
|
|
||||||
class Connection:
|
class Connection:
|
||||||
|
@ -3,9 +3,6 @@ from collections import defaultdict
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import logging
|
import logging
|
||||||
import subprocess
|
|
||||||
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
from . import FetchOptions, DownloadOptions
|
from . import FetchOptions, DownloadOptions
|
||||||
from .results import SearchResults
|
from .results import SearchResults
|
||||||
@ -20,7 +17,6 @@ from ..objects import (
|
|||||||
Artist,
|
Artist,
|
||||||
Label,
|
Label,
|
||||||
)
|
)
|
||||||
from ..objects.artwork import ArtworkVariant
|
|
||||||
from ..audio import write_metadata_to_target, correct_codec
|
from ..audio import write_metadata_to_target, correct_codec
|
||||||
from ..utils import output, BColors
|
from ..utils import output, BColors
|
||||||
from ..utils.string_processing import fit_to_file_system
|
from ..utils.string_processing import fit_to_file_system
|
||||||
@ -33,16 +29,17 @@ from ..utils.support_classes.download_result import DownloadResult
|
|||||||
from ..utils.exception import MKMissingNameException
|
from ..utils.exception import MKMissingNameException
|
||||||
from ..utils.exception.download import UrlNotFoundException
|
from ..utils.exception.download import UrlNotFoundException
|
||||||
from ..utils.shared import DEBUG_PAGES
|
from ..utils.shared import DEBUG_PAGES
|
||||||
from ..connection import Connection
|
|
||||||
|
|
||||||
from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, Bandcamp, Genius, INDEPENDENT_DB_OBJECTS
|
from ..pages import Page, EncyclopaediaMetallum, Musify, YouTube, YoutubeMusic, Bandcamp, Musicbrainz, Genius, INDEPENDENT_DB_OBJECTS
|
||||||
|
|
||||||
|
|
||||||
ALL_PAGES: Set[Type[Page]] = {
|
ALL_PAGES: Set[Type[Page]] = {
|
||||||
# EncyclopaediaMetallum,
|
# EncyclopaediaMetallum,
|
||||||
Genius,
|
Genius,
|
||||||
Musify,
|
Musify,
|
||||||
YoutubeMusic,
|
YoutubeMusic,
|
||||||
Bandcamp
|
Bandcamp,
|
||||||
|
Musicbrainz
|
||||||
}
|
}
|
||||||
|
|
||||||
if youtube_settings["use_youtube_alongside_youtube_music"]:
|
if youtube_settings["use_youtube_alongside_youtube_music"]:
|
||||||
@ -91,8 +88,7 @@ class Pages:
|
|||||||
exclude_pages = exclude_pages.union(SHADY_PAGES)
|
exclude_pages = exclude_pages.union(SHADY_PAGES)
|
||||||
|
|
||||||
if not exclude_pages.issubset(ALL_PAGES):
|
if not exclude_pages.issubset(ALL_PAGES):
|
||||||
raise ValueError(
|
raise ValueError(f"The excluded pages have to be a subset of all pages: {exclude_pages} | {ALL_PAGES}")
|
||||||
f"The excluded pages have to be a subset of all pages: {exclude_pages} | {ALL_PAGES}")
|
|
||||||
|
|
||||||
def _set_to_tuple(page_set: Set[Type[Page]]) -> Tuple[Type[Page], ...]:
|
def _set_to_tuple(page_set: Set[Type[Page]]) -> Tuple[Type[Page], ...]:
|
||||||
return tuple(sorted(page_set, key=lambda page: page.__name__))
|
return tuple(sorted(page_set, key=lambda page: page.__name__))
|
||||||
@ -100,14 +96,11 @@ class Pages:
|
|||||||
self._pages_set: Set[Type[Page]] = ALL_PAGES.difference(exclude_pages)
|
self._pages_set: Set[Type[Page]] = ALL_PAGES.difference(exclude_pages)
|
||||||
self.pages: Tuple[Type[Page], ...] = _set_to_tuple(self._pages_set)
|
self.pages: Tuple[Type[Page], ...] = _set_to_tuple(self._pages_set)
|
||||||
|
|
||||||
self._audio_pages_set: Set[Type[Page]
|
self._audio_pages_set: Set[Type[Page]] = self._pages_set.intersection(AUDIO_PAGES)
|
||||||
] = self._pages_set.intersection(AUDIO_PAGES)
|
self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(self._audio_pages_set)
|
||||||
self.audio_pages: Tuple[Type[Page], ...] = _set_to_tuple(
|
|
||||||
self._audio_pages_set)
|
|
||||||
|
|
||||||
for page_type in self.pages:
|
for page_type in self.pages:
|
||||||
self._page_instances[page_type] = page_type(
|
self._page_instances[page_type] = page_type(fetch_options=self.fetch_options, download_options=self.download_options)
|
||||||
fetch_options=self.fetch_options, download_options=self.download_options)
|
|
||||||
self._source_to_page[page_type.SOURCE_TYPE] = page_type
|
self._source_to_page[page_type.SOURCE_TYPE] = page_type
|
||||||
|
|
||||||
def _get_page_from_enum(self, source_page: SourceType) -> Page:
|
def _get_page_from_enum(self, source_page: SourceType) -> Page:
|
||||||
@ -121,8 +114,7 @@ class Pages:
|
|||||||
for page_type in self.pages:
|
for page_type in self.pages:
|
||||||
result.add(
|
result.add(
|
||||||
page=page_type,
|
page=page_type,
|
||||||
search_result=self._page_instances[page_type].search(
|
search_result=self._page_instances[page_type].search(query=query)
|
||||||
query=query)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
@ -135,8 +127,7 @@ class Pages:
|
|||||||
for source in data_object.source_collection.get_sources(source_type_sorting={
|
for source in data_object.source_collection.get_sources(source_type_sorting={
|
||||||
"only_with_page": True,
|
"only_with_page": True,
|
||||||
}):
|
}):
|
||||||
new_data_object = self.fetch_from_source(
|
new_data_object = self.fetch_from_source(source=source, stop_at_level=stop_at_level)
|
||||||
source=source, stop_at_level=stop_at_level)
|
|
||||||
if new_data_object is not None:
|
if new_data_object is not None:
|
||||||
data_object.merge(new_data_object)
|
data_object.merge(new_data_object)
|
||||||
|
|
||||||
@ -172,38 +163,10 @@ class Pages:
|
|||||||
|
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def _fetch_artist_artwork(self, artist: Artist, naming: dict):
|
|
||||||
naming: Dict[str, List[str]] = defaultdict(list, naming)
|
|
||||||
naming["artist"].append(artist.name)
|
|
||||||
naming["label"].extend(
|
|
||||||
[l.title_value for l in artist.label_collection])
|
|
||||||
# removing duplicates from the naming, and process the strings
|
|
||||||
for key, value in naming.items():
|
|
||||||
# https://stackoverflow.com/a/17016257
|
|
||||||
naming[key] = list(dict.fromkeys(value))
|
|
||||||
|
|
||||||
artwork_collection: ArtworkCollection = artist.artwork
|
|
||||||
artwork_collection.compile()
|
|
||||||
for image_number, artwork in enumerate(artwork_collection):
|
|
||||||
for artwork_variant in artwork.variants:
|
|
||||||
naming["image_number"] = [str(image_number)]
|
|
||||||
target = Target(
|
|
||||||
relative_to_music_dir=True,
|
|
||||||
file_path=Path(self._parse_path_template(
|
|
||||||
main_settings["artist_artwork_path"], naming=naming))
|
|
||||||
)
|
|
||||||
if not target.file_path.parent.exists():
|
|
||||||
target.create_path()
|
|
||||||
subprocess.Popen(["gio", "set", target.file_path.parent, "metadata::custom-icon", "file://"+str(target.file_path)])
|
|
||||||
with Image.open(artwork_variant.target.file_path) as img:
|
|
||||||
img.save(target.file_path, main_settings["image_format"])
|
|
||||||
artwork_variant.target = Target
|
|
||||||
|
|
||||||
def download(self, data_object: DataObject, genre: str, **kwargs) -> DownloadResult:
|
def download(self, data_object: DataObject, genre: str, **kwargs) -> DownloadResult:
|
||||||
# fetch the given object
|
# fetch the given object
|
||||||
self.fetch_details(data_object)
|
self.fetch_details(data_object)
|
||||||
output(
|
output(f"\nDownloading {data_object.option_string}...", color=BColors.BOLD)
|
||||||
f"\nDownloading {data_object.option_string}...", color=BColors.BOLD)
|
|
||||||
|
|
||||||
# fetching all parent objects (e.g. if you only download a song)
|
# fetching all parent objects (e.g. if you only download a song)
|
||||||
if not kwargs.get("fetched_upwards", False):
|
if not kwargs.get("fetched_upwards", False):
|
||||||
@ -224,16 +187,6 @@ class Pages:
|
|||||||
|
|
||||||
kwargs["fetched_upwards"] = True
|
kwargs["fetched_upwards"] = True
|
||||||
|
|
||||||
naming = kwargs.get("naming", {
|
|
||||||
"genre": [genre],
|
|
||||||
"audio_format": [main_settings["audio_format"]],
|
|
||||||
"image_format": [main_settings["image_format"]]
|
|
||||||
})
|
|
||||||
|
|
||||||
# download artist artwork
|
|
||||||
if isinstance(data_object, Artist):
|
|
||||||
self._fetch_artist_artwork(artist=data_object, naming=naming)
|
|
||||||
|
|
||||||
# download all children
|
# download all children
|
||||||
download_result: DownloadResult = DownloadResult()
|
download_result: DownloadResult = DownloadResult()
|
||||||
for c in data_object.get_child_collections():
|
for c in data_object.get_child_collections():
|
||||||
@ -251,7 +204,10 @@ class Pages:
|
|||||||
I am able to do that, because duplicate values are removed later on.
|
I am able to do that, because duplicate values are removed later on.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
self._download_song(data_object, naming=naming)
|
self._download_song(data_object, naming={
|
||||||
|
"genre": [genre],
|
||||||
|
"audio_format": [main_settings["audio_format"]],
|
||||||
|
})
|
||||||
|
|
||||||
return download_result
|
return download_result
|
||||||
|
|
||||||
@ -259,15 +215,13 @@ class Pages:
|
|||||||
return set(re.findall(r"{([^}]+)}", path_template))
|
return set(re.findall(r"{([^}]+)}", path_template))
|
||||||
|
|
||||||
def _parse_path_template(self, path_template: str, naming: Dict[str, List[str]]) -> str:
|
def _parse_path_template(self, path_template: str, naming: Dict[str, List[str]]) -> str:
|
||||||
field_names: Set[str] = self._extract_fields_from_template(
|
field_names: Set[str] = self._extract_fields_from_template(path_template)
|
||||||
path_template)
|
|
||||||
|
|
||||||
for field in field_names:
|
for field in field_names:
|
||||||
if len(naming[field]) == 0:
|
if len(naming[field]) == 0:
|
||||||
raise MKMissingNameException(f"Missing field for {field}.")
|
raise MKMissingNameException(f"Missing field for {field}.")
|
||||||
|
|
||||||
path_template = path_template.replace(
|
path_template = path_template.replace(f"{{{field}}}", naming[field][0])
|
||||||
f"{{{field}}}", naming[field][0])
|
|
||||||
|
|
||||||
return path_template
|
return path_template
|
||||||
|
|
||||||
@ -286,8 +240,7 @@ class Pages:
|
|||||||
naming["song"].append(song.title_value)
|
naming["song"].append(song.title_value)
|
||||||
naming["isrc"].append(song.isrc)
|
naming["isrc"].append(song.isrc)
|
||||||
naming["album"].extend(a.title_value for a in song.album_collection)
|
naming["album"].extend(a.title_value for a in song.album_collection)
|
||||||
naming["album_type"].extend(
|
naming["album_type"].extend(a.album_type.value for a in song.album_collection)
|
||||||
a.album_type.value for a in song.album_collection)
|
|
||||||
naming["artist"].extend(a.name for a in song.artist_collection)
|
naming["artist"].extend(a.name for a in song.artist_collection)
|
||||||
naming["artist"].extend(a.name for a in song.feature_artist_collection)
|
naming["artist"].extend(a.name for a in song.feature_artist_collection)
|
||||||
for a in song.album_collection:
|
for a in song.album_collection:
|
||||||
@ -304,16 +257,13 @@ class Pages:
|
|||||||
song.target_collection.append(Target(
|
song.target_collection.append(Target(
|
||||||
relative_to_music_dir=True,
|
relative_to_music_dir=True,
|
||||||
file_path=Path(
|
file_path=Path(
|
||||||
self._parse_path_template(
|
self._parse_path_template(main_settings["download_path"], naming=naming),
|
||||||
main_settings["download_path"], naming=naming),
|
self._parse_path_template(main_settings["download_file"], naming=naming),
|
||||||
self._parse_path_template(
|
|
||||||
main_settings["download_file"], naming=naming),
|
|
||||||
)
|
)
|
||||||
))
|
))
|
||||||
for target in song.target_collection:
|
for target in song.target_collection:
|
||||||
if target.exists:
|
if target.exists:
|
||||||
output(
|
output(f'{target.file_path} {BColors.OKGREEN.value}[already exists]', color=BColors.GREY)
|
||||||
f'{target.file_path} {BColors.OKGREEN.value}[already exists]', color=BColors.GREY)
|
|
||||||
r.found_on_disk += 1
|
r.found_on_disk += 1
|
||||||
|
|
||||||
if not self.download_options.download_again_if_found:
|
if not self.download_options.download_again_if_found:
|
||||||
@ -334,10 +284,8 @@ class Pages:
|
|||||||
break
|
break
|
||||||
|
|
||||||
used_source = source
|
used_source = source
|
||||||
streaming_results = source.page.download_song_to_target(
|
streaming_results = source.page.download_song_to_target(source=source, target=tmp, desc="download")
|
||||||
source=source, target=tmp, desc="download")
|
skip_intervals = source.page.get_skip_intervals(song=song, source=source)
|
||||||
skip_intervals = source.page.get_skip_intervals(
|
|
||||||
song=song, source=source)
|
|
||||||
|
|
||||||
# if something has been downloaded but it somehow failed, delete the file
|
# if something has been downloaded but it somehow failed, delete the file
|
||||||
if streaming_results.is_fatal_error and tmp.exists:
|
if streaming_results.is_fatal_error and tmp.exists:
|
||||||
@ -361,8 +309,7 @@ class Pages:
|
|||||||
used_source.page.post_process_hook(song=song, temp_target=tmp)
|
used_source.page.post_process_hook(song=song, temp_target=tmp)
|
||||||
|
|
||||||
if not found_on_disk or self.download_options.process_metadata_if_found:
|
if not found_on_disk or self.download_options.process_metadata_if_found:
|
||||||
write_metadata_to_target(
|
write_metadata_to_target(metadata=song.metadata, target=tmp, song=song)
|
||||||
metadata=song.metadata, target=tmp, song=song)
|
|
||||||
|
|
||||||
# copy the tmp target to the final locations
|
# copy the tmp target to the final locations
|
||||||
for target in song.target_collection:
|
for target in song.target_collection:
|
||||||
|
@ -1,16 +1,27 @@
|
|||||||
from typing_extensions import TypeVar
|
from typing_extensions import TypeVar
|
||||||
|
|
||||||
from .artwork import ArtworkCollection
|
|
||||||
from .collection import Collection
|
|
||||||
from .contact import Contact
|
|
||||||
from .country import Country
|
|
||||||
from .formatted_text import FormattedText
|
|
||||||
from .metadata import ID3Timestamp
|
|
||||||
from .metadata import Mapping as ID3Mapping
|
|
||||||
from .metadata import Metadata
|
|
||||||
from .option import Options
|
from .option import Options
|
||||||
from .parents import OuterProxy
|
|
||||||
from .song import Album, Artist, Label, Lyrics, Song, Target
|
from .metadata import Metadata, Mapping as ID3Mapping, ID3Timestamp
|
||||||
|
|
||||||
from .source import Source, SourceType
|
from .source import Source, SourceType
|
||||||
|
|
||||||
|
from .song import (
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
Artist,
|
||||||
|
Target,
|
||||||
|
Lyrics,
|
||||||
|
Label
|
||||||
|
)
|
||||||
|
|
||||||
|
from .formatted_text import FormattedText
|
||||||
|
from .collection import Collection
|
||||||
|
|
||||||
|
from .country import Country
|
||||||
|
from .contact import Contact
|
||||||
|
|
||||||
|
from .parents import OuterProxy
|
||||||
|
|
||||||
|
from .artwork import Artwork
|
||||||
|
|
||||||
DatabaseObject = OuterProxy
|
DatabaseObject = OuterProxy
|
||||||
|
@ -1,243 +1,64 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from copy import copy
|
from typing import List, Optional, Dict, Tuple, Type, Union, TypedDict
|
||||||
from dataclasses import dataclass, field
|
|
||||||
from functools import cached_property
|
|
||||||
from typing import Dict, List, Optional, Set, Tuple, Type, TypedDict, Union
|
|
||||||
|
|
||||||
from ..connection import Connection
|
|
||||||
from ..utils import create_dataclass_instance, custom_hash
|
|
||||||
from ..utils.config import main_settings
|
|
||||||
from ..utils.enums import PictureType
|
|
||||||
from ..utils.string_processing import hash_url, unify
|
|
||||||
from .collection import Collection
|
from .collection import Collection
|
||||||
from .metadata import ID3Timestamp
|
from .metadata import (
|
||||||
from .metadata import Mapping as id3Mapping
|
Mapping as id3Mapping,
|
||||||
from .metadata import Metadata
|
ID3Timestamp,
|
||||||
|
Metadata
|
||||||
|
)
|
||||||
|
from ..utils.string_processing import unify, hash_url
|
||||||
|
|
||||||
from .parents import OuterProxy as Base
|
from .parents import OuterProxy as Base
|
||||||
from .target import Target
|
|
||||||
from PIL import Image
|
|
||||||
|
|
||||||
import imagehash
|
from ..utils.config import main_settings
|
||||||
|
|
||||||
artwork_connection: Connection = Connection(module="artwork")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
class ArtworkVariant(TypedDict):
|
||||||
class ArtworkVariant:
|
|
||||||
url: str
|
url: str
|
||||||
width: Optional[int] = None
|
width: int
|
||||||
heigth: Optional[int] = None
|
height: int
|
||||||
image_format: Optional[str] = None
|
deviation: float
|
||||||
|
|
||||||
def __hash__(self) -> int:
|
|
||||||
return custom_hash(self.url)
|
|
||||||
|
|
||||||
def __eq__(self, other: ArtworkVariant) -> bool:
|
class Artwork:
|
||||||
return hash(self) == hash(other)
|
def __init__(self, *variants: List[ArtworkVariant]) -> None:
|
||||||
|
self._variant_mapping: Dict[str, ArtworkVariant] = {}
|
||||||
|
|
||||||
def __contains__(self, other: str) -> bool:
|
for variant in variants:
|
||||||
return custom_hash(other) == hash(self.url)
|
self.append(**variant)
|
||||||
|
|
||||||
def __merge__(self, other: ArtworkVariant) -> None:
|
@staticmethod
|
||||||
for key, value in other.__dict__.items():
|
def _calculate_deviation(*dimensions: List[int]) -> float:
|
||||||
if value is None:
|
return sum(abs(d - main_settings["preferred_artwork_resolution"]) for d in dimensions) / len(dimensions)
|
||||||
continue
|
|
||||||
|
|
||||||
if getattr(self, key) is None:
|
def append(self, url: str, width: int = main_settings["preferred_artwork_resolution"], height: int = main_settings["preferred_artwork_resolution"], **kwargs) -> None:
|
||||||
setattr(self, key, value)
|
if url is None:
|
||||||
|
|
||||||
@cached_property
|
|
||||||
def target(self) -> Target:
|
|
||||||
return Target.temp()
|
|
||||||
|
|
||||||
def fetch(self) -> None:
|
|
||||||
global artwork_connection
|
|
||||||
|
|
||||||
r = artwork_connection.get(self.url, name=hash_url(self.url))
|
|
||||||
if r is None:
|
|
||||||
return
|
return
|
||||||
|
|
||||||
self.target.raw_content = r.content
|
self._variant_mapping[hash_url(url=url)] = {
|
||||||
|
"url": url,
|
||||||
@dataclass
|
"width": width,
|
||||||
class Artwork:
|
"height": height,
|
||||||
variants: List[ArtworkVariant] = field(default_factory=list)
|
"deviation": self._calculate_deviation(width, height),
|
||||||
|
}
|
||||||
artwork_type: PictureType = PictureType.OTHER
|
|
||||||
|
|
||||||
def search_variant(self, url: str) -> Optional[ArtworkVariant]:
|
|
||||||
if url is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
for variant in self.variants:
|
|
||||||
if url in variant:
|
|
||||||
return variant
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def __contains__(self, other: str) -> bool:
|
|
||||||
return self.search_variant(other) is not None
|
|
||||||
|
|
||||||
def add_data(self, **kwargs) -> None:
|
|
||||||
variant = self.search_variant(kwargs.get("url"))
|
|
||||||
|
|
||||||
if variant is None:
|
|
||||||
variant, kwargs = create_dataclass_instance(ArtworkVariant, kwargs)
|
|
||||||
self.variants.append(variant)
|
|
||||||
|
|
||||||
variant.__dict__.update(kwargs)
|
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def url(self) -> Optional[str]:
|
def best_variant(self) -> ArtworkVariant:
|
||||||
if len(self.variants) <= 0:
|
if len(self._variant_mapping.keys()) <= 0:
|
||||||
return None
|
return None
|
||||||
return self.variants[0].url
|
return min(self._variant_mapping.values(), key=lambda x: x["deviation"])
|
||||||
|
|
||||||
def fetch(self) -> None:
|
|
||||||
for variant in self.variants:
|
|
||||||
variant.fetch()
|
|
||||||
|
|
||||||
|
|
||||||
class ArtworkCollection:
|
|
||||||
"""
|
|
||||||
Stores all the images/artworks for one data object.
|
|
||||||
|
|
||||||
There could be duplicates before calling ArtworkCollection.compile()
|
|
||||||
_this is called before one object is downloaded automatically._
|
|
||||||
"""
|
|
||||||
|
|
||||||
artwork_type: PictureType = PictureType.OTHER
|
|
||||||
|
|
||||||
def __init__(
|
|
||||||
self,
|
|
||||||
*data: List[Artwork],
|
|
||||||
parent_artworks: Set[ArtworkCollection] = None,
|
|
||||||
crop_images: bool = True,
|
|
||||||
) -> None:
|
|
||||||
# this is used for the song artwork, to fall back to the song artwork
|
|
||||||
self.parent_artworks: Set[ArtworkCollection] = parent_artworks or set()
|
|
||||||
self.crop_images: bool = crop_images
|
|
||||||
|
|
||||||
self._data = []
|
|
||||||
self.extend(data)
|
|
||||||
|
|
||||||
def search_artwork(self, url: str) -> Optional[ArtworkVariant]:
|
|
||||||
for artwork in self._data:
|
|
||||||
if url in artwork:
|
|
||||||
return artwork
|
|
||||||
|
|
||||||
return None
|
|
||||||
|
|
||||||
def __contains__(self, other: str) -> bool:
|
|
||||||
return self.search_artwork(other) is not None
|
|
||||||
|
|
||||||
def _create_new_artwork(self, **kwargs) -> Tuple[Artwork, dict]:
|
|
||||||
kwargs["artwork_type"] = kwargs.get("artwork_type", self.artwork_type)
|
|
||||||
|
|
||||||
return create_dataclass_instance(Artwork, dict(**kwargs))
|
|
||||||
|
|
||||||
def add_data(self, url: str, **kwargs) -> Artwork:
|
|
||||||
kwargs["url"] = url
|
|
||||||
|
|
||||||
artwork = self.search_artwork(url)
|
|
||||||
|
|
||||||
if artwork is None:
|
|
||||||
artwork, kwargs = self._create_new_artwork(**kwargs)
|
|
||||||
self._data.append(artwork)
|
|
||||||
|
|
||||||
artwork.add_data(**kwargs)
|
|
||||||
return artwork
|
|
||||||
|
|
||||||
def append(self, value: Union[Artwork, ArtworkVariant, dict], **kwargs):
|
|
||||||
"""
|
|
||||||
You can append the types Artwork, ArtworkVariant or dict
|
|
||||||
the best option would be to use Artwork and avoid the other options.
|
|
||||||
"""
|
|
||||||
if isinstance(value, dict):
|
|
||||||
kwargs.update(value)
|
|
||||||
value, kwargs = create_dataclass_instance(ArtworkVariant, kwargs)
|
|
||||||
|
|
||||||
if isinstance(value, ArtworkVariant):
|
|
||||||
kwargs["variants"] = [value]
|
|
||||||
value, kwargs = create_dataclass_instance(Artwork, kwargs)
|
|
||||||
|
|
||||||
if isinstance(value, Artwork):
|
|
||||||
self._data.append(value)
|
|
||||||
return
|
|
||||||
|
|
||||||
def extend(self, values: List[Union[Artwork, ArtworkVariant, dict]], **kwargs):
|
|
||||||
for value in values:
|
|
||||||
self.append(value, **kwargs)
|
|
||||||
|
|
||||||
def compile(self, **kwargs) -> None:
|
|
||||||
"""
|
|
||||||
This will make the artworks ready for download and delete duplicates.
|
|
||||||
"""
|
|
||||||
artwork_hashes: list = list()
|
|
||||||
artwork_urls: list = list()
|
|
||||||
for artwork in self._data:
|
|
||||||
index = 0
|
|
||||||
for artwork_variant in artwork.variants:
|
|
||||||
r = artwork_connection.get(
|
|
||||||
url=artwork_variant.url,
|
|
||||||
name=artwork_variant.url,
|
|
||||||
)
|
|
||||||
|
|
||||||
if artwork_variant.url in artwork_urls:
|
|
||||||
artwork.variants.pop(index)
|
|
||||||
continue
|
|
||||||
artwork_urls.append(artwork_variant.url)
|
|
||||||
|
|
||||||
target: Target = artwork_variant.target
|
|
||||||
with target.open("wb") as f:
|
|
||||||
f.write(r.content)
|
|
||||||
|
|
||||||
with Image.open(target.file_path) as img:
|
|
||||||
# https://stackoverflow.com/a/59476938/16804841
|
|
||||||
if img.mode != 'RGB':
|
|
||||||
img = img.convert('RGB')
|
|
||||||
|
|
||||||
try:
|
|
||||||
image_hash = imagehash.crop_resistant_hash(img)
|
|
||||||
except Exception as e:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if image_hash in artwork_hashes:
|
|
||||||
artwork.variants.pop(index)
|
|
||||||
target.delete()
|
|
||||||
continue
|
|
||||||
artwork_hashes.append(image_hash)
|
|
||||||
width, height = img.size
|
|
||||||
if width != height:
|
|
||||||
if width > height:
|
|
||||||
img = img.crop((width // 2 - height // 2, 0, width // 2 + height // 2, height))
|
|
||||||
else:
|
|
||||||
img = img.crop((0, height // 2 - width // 2, width, height // 2 + width // 2))
|
|
||||||
|
|
||||||
# resize the image to the preferred resolution
|
|
||||||
img.thumbnail((main_settings["preferred_artwork_resolution"], main_settings["preferred_artwork_resolution"]))
|
|
||||||
index =+ 1
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def __merge__(self, other: ArtworkCollection, **kwargs) -> None:
|
|
||||||
self.parent_artworks.update(other.parent_artworks)
|
|
||||||
for other_artwork in other._data:
|
|
||||||
for other_variant in other_artwork.variants:
|
|
||||||
if self.__contains__(other_variant.url):
|
|
||||||
continue
|
|
||||||
self.append(ArtworkVariant(other_variant.url))
|
|
||||||
|
|
||||||
|
|
||||||
def __hash__(self) -> int:
|
|
||||||
return id(self)
|
|
||||||
|
|
||||||
def __iter__(self) -> Generator[Artwork, None, None]:
|
|
||||||
yield from self._data
|
|
||||||
|
|
||||||
def get_urls(self) -> Generator[str, None, None]:
|
|
||||||
yield from (artwork.url for artwork in self._data if artwork.url is not None)
|
|
||||||
|
|
||||||
|
def get_variant_name(self, variant: ArtworkVariant) -> str:
|
||||||
|
return f"artwork_{variant['width']}x{variant['height']}_{hash_url(variant['url']).replace('/', '_')}"
|
||||||
|
|
||||||
|
def __merge__(self, other: Artwork, **kwargs) -> None:
|
||||||
|
for key, value in other._variant_mapping.items():
|
||||||
|
if key not in self._variant_mapping:
|
||||||
|
self._variant_mapping[key] = value
|
||||||
|
|
||||||
|
def __eq__(self, other: Artwork) -> bool:
|
||||||
|
if not isinstance(other, Artwork):
|
||||||
|
return False
|
||||||
|
return any(a == b for a, b in zip(self._variant_mapping.keys(), other._variant_mapping.keys()))
|
||||||
|
@ -1,43 +1,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import copy
|
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from typing import TypeVar, Generic, Dict, Optional, Iterable, List, Iterator, Tuple, Generator, Union, Any, Set
|
||||||
from typing import (Any, Callable, Dict, Generator, Generic, Iterable,
|
import copy
|
||||||
Iterator, List, Optional, Set, Tuple, TypeVar, Union)
|
|
||||||
|
|
||||||
from ..utils import BColors, object_trace, output
|
from .parents import OuterProxy
|
||||||
from .parents import InnerData, OuterProxy
|
from ..utils import object_trace
|
||||||
|
from ..utils import output, BColors
|
||||||
|
|
||||||
T = TypeVar('T', bound=OuterProxy)
|
T = TypeVar('T', bound=OuterProxy)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
|
||||||
class AppendHookArguments:
|
|
||||||
"""
|
|
||||||
This class is used to store the arguments for the append hook.
|
|
||||||
The best explanation is with an examples:
|
|
||||||
|
|
||||||
```
|
|
||||||
album = Album()
|
|
||||||
song = Song()
|
|
||||||
album.song_collection.append(song)
|
|
||||||
```
|
|
||||||
|
|
||||||
In this case, the append hook is triggered with the following arguments:
|
|
||||||
```
|
|
||||||
AppendHookArguments(
|
|
||||||
collection=album.song_collection,
|
|
||||||
new_object=song,
|
|
||||||
collection_root_objects=[album]
|
|
||||||
)
|
|
||||||
```
|
|
||||||
"""
|
|
||||||
|
|
||||||
collection: Collection
|
|
||||||
new_object: T
|
|
||||||
collection_root_objects: Set[InnerData]
|
|
||||||
|
|
||||||
class Collection(Generic[T]):
|
class Collection(Generic[T]):
|
||||||
__is_collection__ = True
|
__is_collection__ = True
|
||||||
|
|
||||||
@ -54,7 +27,6 @@ class Collection(Generic[T]):
|
|||||||
sync_on_append: Dict[str, Collection] = None,
|
sync_on_append: Dict[str, Collection] = None,
|
||||||
append_object_to_attribute: Dict[str, T] = None,
|
append_object_to_attribute: Dict[str, T] = None,
|
||||||
extend_object_to_attribute: Dict[str, Collection] = None,
|
extend_object_to_attribute: Dict[str, Collection] = None,
|
||||||
append_callbacks: Set[Callable[[AppendHookArguments], None]] = None,
|
|
||||||
) -> None:
|
) -> None:
|
||||||
self._collection_for: dict = dict()
|
self._collection_for: dict = dict()
|
||||||
|
|
||||||
@ -69,7 +41,6 @@ class Collection(Generic[T]):
|
|||||||
self.sync_on_append: Dict[str, Collection] = sync_on_append or {}
|
self.sync_on_append: Dict[str, Collection] = sync_on_append or {}
|
||||||
self.pull_from: List[Collection] = []
|
self.pull_from: List[Collection] = []
|
||||||
self.push_to: List[Collection] = []
|
self.push_to: List[Collection] = []
|
||||||
self.append_callbacks: Set[Callable[[AppendHookArguments], None]] = append_callbacks or set()
|
|
||||||
|
|
||||||
# This is to cleanly unmap previously mapped items by their id
|
# This is to cleanly unmap previously mapped items by their id
|
||||||
self._indexed_from_id: Dict[int, Dict[str, Any]] = defaultdict(dict)
|
self._indexed_from_id: Dict[int, Dict[str, Any]] = defaultdict(dict)
|
||||||
@ -170,14 +141,6 @@ class Collection(Generic[T]):
|
|||||||
for attribute, new_object in self.append_object_to_attribute.items():
|
for attribute, new_object in self.append_object_to_attribute.items():
|
||||||
other.__getattribute__(attribute).append(new_object, **kwargs)
|
other.__getattribute__(attribute).append(new_object, **kwargs)
|
||||||
|
|
||||||
append_hook_args = AppendHookArguments(
|
|
||||||
collection=self,
|
|
||||||
new_object=other,
|
|
||||||
collection_root_objects=self._collection_for.keys(),
|
|
||||||
)
|
|
||||||
for callback in self.append_callbacks:
|
|
||||||
callback(append_hook_args)
|
|
||||||
|
|
||||||
def append(self, other: Optional[T], **kwargs):
|
def append(self, other: Optional[T], **kwargs):
|
||||||
"""
|
"""
|
||||||
If an object, that represents the same entity exists in a relevant collection,
|
If an object, that represents the same entity exists in a relevant collection,
|
||||||
|
@ -1,32 +1,35 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import copy
|
|
||||||
import random
|
import random
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Dict, List, Optional, Tuple, Type, Union
|
from typing import List, Optional, Dict, Tuple, Type, Union
|
||||||
|
import copy
|
||||||
|
|
||||||
import pycountry
|
import pycountry
|
||||||
|
|
||||||
from ..utils.config import main_settings
|
from ..utils.enums.album import AlbumType, AlbumStatus
|
||||||
from ..utils.enums.album import AlbumStatus, AlbumType
|
from .collection import Collection
|
||||||
from ..utils.enums.colors import BColors
|
|
||||||
from ..utils.shared import DEBUG_PRINT_ID
|
|
||||||
from ..utils.string_processing import unify
|
|
||||||
from .artwork import ArtworkCollection
|
|
||||||
from .collection import AppendHookArguments, Collection
|
|
||||||
from .contact import Contact
|
|
||||||
from .country import Country, Language
|
|
||||||
from .formatted_text import FormattedText
|
from .formatted_text import FormattedText
|
||||||
from .lyrics import Lyrics
|
from .lyrics import Lyrics
|
||||||
from .metadata import ID3Timestamp
|
from .contact import Contact
|
||||||
from .metadata import Mapping as id3Mapping
|
from .artwork import Artwork
|
||||||
from .metadata import Metadata
|
from .metadata import (
|
||||||
|
Mapping as id3Mapping,
|
||||||
|
ID3Timestamp,
|
||||||
|
Metadata
|
||||||
|
)
|
||||||
from .option import Options
|
from .option import Options
|
||||||
from .parents import OuterProxy
|
from .parents import OuterProxy, P
|
||||||
from .parents import OuterProxy as Base
|
|
||||||
from .parents import P
|
|
||||||
from .source import Source, SourceCollection
|
from .source import Source, SourceCollection
|
||||||
from .target import Target
|
from .target import Target
|
||||||
|
from .country import Language, Country
|
||||||
|
from ..utils.shared import DEBUG_PRINT_ID
|
||||||
|
from ..utils.string_processing import unify
|
||||||
|
|
||||||
|
from .parents import OuterProxy as Base
|
||||||
|
|
||||||
|
from ..utils.config import main_settings
|
||||||
|
from ..utils.enums.colors import BColors
|
||||||
|
|
||||||
"""
|
"""
|
||||||
All Objects dependent
|
All Objects dependent
|
||||||
@ -86,7 +89,7 @@ class Song(Base):
|
|||||||
genre: str
|
genre: str
|
||||||
note: FormattedText
|
note: FormattedText
|
||||||
tracksort: int
|
tracksort: int
|
||||||
artwork: ArtworkCollection
|
artwork: Artwork
|
||||||
|
|
||||||
source_collection: SourceCollection
|
source_collection: SourceCollection
|
||||||
target_collection: Collection[Target]
|
target_collection: Collection[Target]
|
||||||
@ -102,7 +105,7 @@ class Song(Base):
|
|||||||
"source_collection": SourceCollection,
|
"source_collection": SourceCollection,
|
||||||
"target_collection": Collection,
|
"target_collection": Collection,
|
||||||
"lyrics_collection": Collection,
|
"lyrics_collection": Collection,
|
||||||
"artwork": ArtworkCollection,
|
"artwork": Artwork,
|
||||||
|
|
||||||
"album_collection": Collection,
|
"album_collection": Collection,
|
||||||
"artist_collection": Collection,
|
"artist_collection": Collection,
|
||||||
@ -130,7 +133,7 @@ class Song(Base):
|
|||||||
feature_artist_list: List[Artist] = None,
|
feature_artist_list: List[Artist] = None,
|
||||||
album_list: List[Album] = None,
|
album_list: List[Album] = None,
|
||||||
tracksort: int = 0,
|
tracksort: int = 0,
|
||||||
artwork: Optional[ArtworkCollection] = None,
|
artwork: Optional[Artwork] = None,
|
||||||
**kwargs
|
**kwargs
|
||||||
) -> None:
|
) -> None:
|
||||||
real_kwargs = copy.copy(locals())
|
real_kwargs = copy.copy(locals())
|
||||||
@ -141,14 +144,6 @@ class Song(Base):
|
|||||||
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("artist_collection", "feature_artist_collection", "album_collection")
|
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("artist_collection", "feature_artist_collection", "album_collection")
|
||||||
TITEL = "title"
|
TITEL = "title"
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def register_artwork_parent(append_hook_arguments: AppendHookArguments):
|
|
||||||
album: Album = append_hook_arguments.new_object
|
|
||||||
|
|
||||||
song: Song
|
|
||||||
for song in append_hook_arguments.collection_root_objects:
|
|
||||||
song.artwork.parent_artworks.add(album.artwork)
|
|
||||||
|
|
||||||
def __init_collections__(self) -> None:
|
def __init_collections__(self) -> None:
|
||||||
self.feature_artist_collection.push_to = [self.artist_collection]
|
self.feature_artist_collection.push_to = [self.artist_collection]
|
||||||
self.artist_collection.pull_from = [self.feature_artist_collection]
|
self.artist_collection.pull_from = [self.feature_artist_collection]
|
||||||
@ -166,7 +161,6 @@ class Song(Base):
|
|||||||
self.feature_artist_collection.extend_object_to_attribute = {
|
self.feature_artist_collection.extend_object_to_attribute = {
|
||||||
"album_collection": self.album_collection
|
"album_collection": self.album_collection
|
||||||
}
|
}
|
||||||
self.album_collection.append_callbacks = set((Song.register_artwork_parent, ))
|
|
||||||
|
|
||||||
def _add_other_db_objects(self, object_type: Type[OuterProxy], object_list: List[OuterProxy]):
|
def _add_other_db_objects(self, object_type: Type[OuterProxy], object_list: List[OuterProxy]):
|
||||||
if object_type is Song:
|
if object_type is Song:
|
||||||
@ -184,10 +178,6 @@ class Song(Base):
|
|||||||
self.album_collection.extend(object_list)
|
self.album_collection.extend(object_list)
|
||||||
return
|
return
|
||||||
|
|
||||||
def _compile(self):
|
|
||||||
self.artwork.compile()
|
|
||||||
|
|
||||||
|
|
||||||
INDEX_DEPENDS_ON = ("title", "isrc", "source_collection")
|
INDEX_DEPENDS_ON = ("title", "isrc", "source_collection")
|
||||||
|
|
||||||
@property
|
@property
|
||||||
@ -259,7 +249,6 @@ class Album(Base):
|
|||||||
albumsort: int
|
albumsort: int
|
||||||
notes: FormattedText
|
notes: FormattedText
|
||||||
|
|
||||||
artwork: ArtworkCollection
|
|
||||||
source_collection: SourceCollection
|
source_collection: SourceCollection
|
||||||
|
|
||||||
song_collection: Collection[Song]
|
song_collection: Collection[Song]
|
||||||
@ -279,7 +268,6 @@ class Album(Base):
|
|||||||
"date": ID3Timestamp,
|
"date": ID3Timestamp,
|
||||||
"notes": FormattedText,
|
"notes": FormattedText,
|
||||||
|
|
||||||
"artwork": lambda: ArtworkCollection(crop_images=False),
|
|
||||||
"source_collection": SourceCollection,
|
"source_collection": SourceCollection,
|
||||||
|
|
||||||
"song_collection": Collection,
|
"song_collection": Collection,
|
||||||
@ -302,7 +290,6 @@ class Album(Base):
|
|||||||
barcode: str = None,
|
barcode: str = None,
|
||||||
albumsort: int = None,
|
albumsort: int = None,
|
||||||
notes: FormattedText = None,
|
notes: FormattedText = None,
|
||||||
artwork: ArtworkCollection = None,
|
|
||||||
source_list: List[Source] = None,
|
source_list: List[Source] = None,
|
||||||
artist_list: List[Artist] = None,
|
artist_list: List[Artist] = None,
|
||||||
song_list: List[Song] = None,
|
song_list: List[Song] = None,
|
||||||
@ -317,13 +304,6 @@ class Album(Base):
|
|||||||
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("song_collection",)
|
DOWNWARDS_COLLECTION_STRING_ATTRIBUTES = ("song_collection",)
|
||||||
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection", "artist_collection")
|
UPWARDS_COLLECTION_STRING_ATTRIBUTES = ("label_collection", "artist_collection")
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def register_artwork_parent(append_hook_arguments: AppendHookArguments):
|
|
||||||
song: Song = append_hook_arguments.new_object
|
|
||||||
|
|
||||||
for root_object in append_hook_arguments.collection_root_objects:
|
|
||||||
song.artwork.parent_artworks.add(root_object.artwork)
|
|
||||||
|
|
||||||
def __init_collections__(self):
|
def __init_collections__(self):
|
||||||
self.feature_artist_collection.push_to = [self.artist_collection]
|
self.feature_artist_collection.push_to = [self.artist_collection]
|
||||||
self.artist_collection.pull_from = [self.feature_artist_collection]
|
self.artist_collection.pull_from = [self.feature_artist_collection]
|
||||||
@ -342,8 +322,6 @@ class Album(Base):
|
|||||||
"label_collection": self.label_collection
|
"label_collection": self.label_collection
|
||||||
}
|
}
|
||||||
|
|
||||||
self.song_collection.append_callbacks = set((Album.register_artwork_parent, ))
|
|
||||||
|
|
||||||
def _add_other_db_objects(self, object_type: Type[OuterProxy], object_list: List[OuterProxy]):
|
def _add_other_db_objects(self, object_type: Type[OuterProxy], object_list: List[OuterProxy]):
|
||||||
if object_type is Song:
|
if object_type is Song:
|
||||||
self.song_collection.extend(object_list)
|
self.song_collection.extend(object_list)
|
||||||
@ -499,8 +477,6 @@ class Artist(Base):
|
|||||||
general_genre: str
|
general_genre: str
|
||||||
unformatted_location: str
|
unformatted_location: str
|
||||||
|
|
||||||
artwork: ArtworkCollection
|
|
||||||
|
|
||||||
source_collection: SourceCollection
|
source_collection: SourceCollection
|
||||||
contact_collection: Collection[Contact]
|
contact_collection: Collection[Contact]
|
||||||
|
|
||||||
@ -517,8 +493,6 @@ class Artist(Base):
|
|||||||
"lyrical_themes": list,
|
"lyrical_themes": list,
|
||||||
"general_genre": lambda: "",
|
"general_genre": lambda: "",
|
||||||
|
|
||||||
"artwork": ArtworkCollection,
|
|
||||||
|
|
||||||
"source_collection": SourceCollection,
|
"source_collection": SourceCollection,
|
||||||
"album_collection": Collection,
|
"album_collection": Collection,
|
||||||
"contact_collection": Collection,
|
"contact_collection": Collection,
|
||||||
@ -537,7 +511,6 @@ class Artist(Base):
|
|||||||
notes: FormattedText = None,
|
notes: FormattedText = None,
|
||||||
lyrical_themes: List[str] = None,
|
lyrical_themes: List[str] = None,
|
||||||
general_genre: str = None,
|
general_genre: str = None,
|
||||||
artwork: ArtworkCollection = None,
|
|
||||||
unformatted_location: str = None,
|
unformatted_location: str = None,
|
||||||
source_list: List[Source] = None,
|
source_list: List[Source] = None,
|
||||||
contact_list: List[Contact] = None,
|
contact_list: List[Contact] = None,
|
||||||
|
@ -1,17 +1,17 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import List, Tuple, TextIO, Union, Optional
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
from pathlib import Path
|
|
||||||
from typing import List, Optional, TextIO, Tuple, Union
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
from ..utils.config import logging_settings, main_settings
|
|
||||||
from ..utils.shared import HIGHEST_ID
|
|
||||||
from ..utils.string_processing import fit_to_file_system
|
|
||||||
from .parents import OuterProxy
|
from .parents import OuterProxy
|
||||||
|
from ..utils.shared import HIGHEST_ID
|
||||||
|
from ..utils.config import main_settings, logging_settings
|
||||||
|
from ..utils.string_processing import fit_to_file_system
|
||||||
|
|
||||||
|
|
||||||
LOGGER = logging.getLogger("target")
|
LOGGER = logging.getLogger("target")
|
||||||
|
|
||||||
@ -31,8 +31,7 @@ class Target(OuterProxy):
|
|||||||
}
|
}
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def temp(cls, name: str = None, file_extension: Optional[str] = None) -> P:
|
def temp(cls, name: str = str(random.randint(0, HIGHEST_ID)), file_extension: Optional[str] = None) -> P:
|
||||||
name = name or str(random.randint(0, HIGHEST_ID))
|
|
||||||
if file_extension is not None:
|
if file_extension is not None:
|
||||||
name = f"{name}.{file_extension}"
|
name = f"{name}.{file_extension}"
|
||||||
|
|
||||||
@ -118,11 +117,3 @@ class Target(OuterProxy):
|
|||||||
|
|
||||||
def read_bytes(self) -> bytes:
|
def read_bytes(self) -> bytes:
|
||||||
return self.file_path.read_bytes()
|
return self.file_path.read_bytes()
|
||||||
|
|
||||||
@property
|
|
||||||
def raw_content(self) -> bytes:
|
|
||||||
return self.file_path.read_bytes()
|
|
||||||
|
|
||||||
@raw_content.setter
|
|
||||||
def raw_content(self, content: bytes):
|
|
||||||
self.file_path.write_bytes(content)
|
|
||||||
|
@ -1,5 +1,6 @@
|
|||||||
from .encyclopaedia_metallum import EncyclopaediaMetallum
|
from .encyclopaedia_metallum import EncyclopaediaMetallum
|
||||||
from .musify import Musify
|
from .musify import Musify
|
||||||
|
from .musicbrainz import Musicbrainz
|
||||||
from .youtube import YouTube
|
from .youtube import YouTube
|
||||||
from .youtube_music import YoutubeMusic
|
from .youtube_music import YoutubeMusic
|
||||||
from .bandcamp import Bandcamp
|
from .bandcamp import Bandcamp
|
||||||
|
@ -1,22 +1,33 @@
|
|||||||
import json
|
|
||||||
from enum import Enum
|
|
||||||
from typing import List, Optional, Type
|
from typing import List, Optional, Type
|
||||||
from urllib.parse import urlparse, urlunparse
|
from urllib.parse import urlparse, urlunparse
|
||||||
|
import json
|
||||||
import pycountry
|
from enum import Enum
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
import pycountry
|
||||||
|
|
||||||
from ..connection import Connection
|
from ..objects import Source, DatabaseObject
|
||||||
from ..objects import (Album, Artist, ArtworkCollection, Contact,
|
|
||||||
DatabaseObject, FormattedText, ID3Timestamp, Label,
|
|
||||||
Lyrics, Song, Source, SourceType, Target)
|
|
||||||
from ..utils import dump_to_file
|
|
||||||
from ..utils.config import logging_settings, main_settings
|
|
||||||
from ..utils.enums import ALL_SOURCE_TYPES, SourceType
|
|
||||||
from ..utils.shared import DEBUG
|
|
||||||
from ..utils.string_processing import clean_song_title
|
|
||||||
from ..utils.support_classes.download_result import DownloadResult
|
|
||||||
from .abstract import Page
|
from .abstract import Page
|
||||||
|
from ..objects import (
|
||||||
|
Artist,
|
||||||
|
Source,
|
||||||
|
SourceType,
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
Label,
|
||||||
|
Target,
|
||||||
|
Contact,
|
||||||
|
ID3Timestamp,
|
||||||
|
Lyrics,
|
||||||
|
FormattedText,
|
||||||
|
Artwork,
|
||||||
|
)
|
||||||
|
from ..connection import Connection
|
||||||
|
from ..utils import dump_to_file
|
||||||
|
from ..utils.enums import SourceType, ALL_SOURCE_TYPES
|
||||||
|
from ..utils.support_classes.download_result import DownloadResult
|
||||||
|
from ..utils.string_processing import clean_song_title
|
||||||
|
from ..utils.config import main_settings, logging_settings
|
||||||
|
from ..utils.shared import DEBUG
|
||||||
|
|
||||||
if DEBUG:
|
if DEBUG:
|
||||||
from ..utils import dump_to_file
|
from ..utils import dump_to_file
|
||||||
@ -228,11 +239,6 @@ class Bandcamp(Page):
|
|||||||
for subsoup in html_music_grid.find_all("li"):
|
for subsoup in html_music_grid.find_all("li"):
|
||||||
artist.album_collection.append(self._parse_album(soup=subsoup, initial_source=source))
|
artist.album_collection.append(self._parse_album(soup=subsoup, initial_source=source))
|
||||||
|
|
||||||
# artist artwork
|
|
||||||
artist_artwork: BeautifulSoup = soup.find("img", {"class":"band-photo"})
|
|
||||||
if artist_artwork is not None:
|
|
||||||
artist.artwork.add_data(artist_artwork.get("data-src", artist_artwork.get("src")))
|
|
||||||
|
|
||||||
for i, data_blob_soup in enumerate(soup.find_all("div", {"id": ["pagedata", "collectors-data"]})):
|
for i, data_blob_soup in enumerate(soup.find_all("div", {"id": ["pagedata", "collectors-data"]})):
|
||||||
data_blob = data_blob_soup["data-blob"]
|
data_blob = data_blob_soup["data-blob"]
|
||||||
|
|
||||||
@ -247,7 +253,7 @@ class Bandcamp(Page):
|
|||||||
artist.source_collection.append(source)
|
artist.source_collection.append(source)
|
||||||
return artist
|
return artist
|
||||||
|
|
||||||
def _parse_track_element(self, track: dict, artwork: ArtworkCollection) -> Optional[Song]:
|
def _parse_track_element(self, track: dict, artwork: Artwork) -> Optional[Song]:
|
||||||
lyrics_list: List[Lyrics] = []
|
lyrics_list: List[Lyrics] = []
|
||||||
|
|
||||||
_lyrics: Optional[str] = track.get("item", {}).get("recordingOf", {}).get("lyrics", {}).get("text")
|
_lyrics: Optional[str] = track.get("item", {}).get("recordingOf", {}).get("lyrics", {}).get("text")
|
||||||
@ -281,15 +287,9 @@ class Bandcamp(Page):
|
|||||||
artist_source_list = []
|
artist_source_list = []
|
||||||
if "@id" in artist_data:
|
if "@id" in artist_data:
|
||||||
artist_source_list = [Source(self.SOURCE_TYPE, _parse_artist_url(artist_data["@id"]))]
|
artist_source_list = [Source(self.SOURCE_TYPE, _parse_artist_url(artist_data["@id"]))]
|
||||||
|
|
||||||
|
|
||||||
source_list: List[Source] = [source]
|
|
||||||
if "mainEntityOfPage" in data or "@id" in data:
|
|
||||||
source_list.append(Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"])))
|
|
||||||
|
|
||||||
album = Album(
|
album = Album(
|
||||||
title=data["name"].strip(),
|
title=data["name"].strip(),
|
||||||
source_list=source_list,
|
source_list=[Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]))],
|
||||||
date=ID3Timestamp.strptime(data["datePublished"], "%d %b %Y %H:%M:%S %Z"),
|
date=ID3Timestamp.strptime(data["datePublished"], "%d %b %Y %H:%M:%S %Z"),
|
||||||
artist_list=[Artist(
|
artist_list=[Artist(
|
||||||
name=artist_data["name"].strip(),
|
name=artist_data["name"].strip(),
|
||||||
@ -297,7 +297,7 @@ class Bandcamp(Page):
|
|||||||
)]
|
)]
|
||||||
)
|
)
|
||||||
|
|
||||||
artwork: ArtworkCollection = ArtworkCollection()
|
artwork: Artwork = Artwork()
|
||||||
|
|
||||||
def _get_artwork_url(_data: dict) -> Optional[str]:
|
def _get_artwork_url(_data: dict) -> Optional[str]:
|
||||||
if "image" in _data:
|
if "image" in _data:
|
||||||
@ -308,14 +308,15 @@ class Bandcamp(Page):
|
|||||||
|
|
||||||
_artwork_url = _get_artwork_url(data)
|
_artwork_url = _get_artwork_url(data)
|
||||||
if _artwork_url is not None:
|
if _artwork_url is not None:
|
||||||
artwork.add_data(url=_artwork_url, width=350, height=350)
|
artwork.append(url=_artwork_url, width=350, height=350)
|
||||||
else:
|
else:
|
||||||
for album_release in data.get("albumRelease", []):
|
for album_release in data.get("albumRelease", []):
|
||||||
_artwork_url = _get_artwork_url(album_release)
|
_artwork_url = _get_artwork_url(album_release)
|
||||||
if _artwork_url is not None:
|
if _artwork_url is not None:
|
||||||
artwork.add_data(url=_artwork_url, width=350, height=350)
|
artwork.append(url=_artwork_url, width=350, height=350)
|
||||||
break
|
break
|
||||||
|
|
||||||
|
|
||||||
for i, track_json in enumerate(data.get("track", {}).get("itemListElement", [])):
|
for i, track_json in enumerate(data.get("track", {}).get("itemListElement", [])):
|
||||||
if DEBUG:
|
if DEBUG:
|
||||||
dump_to_file(f"album_track_{i}.json", json.dumps(track_json), is_json=True, exit_after_dump=False)
|
dump_to_file(f"album_track_{i}.json", json.dumps(track_json), is_json=True, exit_after_dump=False)
|
||||||
@ -361,29 +362,17 @@ class Bandcamp(Page):
|
|||||||
for key, value in other_data.get("trackinfo", [{}])[0].get("file", {"": None}).items():
|
for key, value in other_data.get("trackinfo", [{}])[0].get("file", {"": None}).items():
|
||||||
mp3_url = value
|
mp3_url = value
|
||||||
|
|
||||||
source_list: List[Source] = [source]
|
|
||||||
if "mainEntityOfPage" in data or "@id" in data:
|
|
||||||
source_list.append(Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]), audio_url=mp3_url))
|
|
||||||
|
|
||||||
source_list_album: List[Source] = [source]
|
|
||||||
if "@id" in album_data:
|
|
||||||
source_list_album.append(Source(self.SOURCE_TYPE, album_data["@id"]))
|
|
||||||
|
|
||||||
source_list_artist: List[Source] = [source]
|
|
||||||
if "@id" in artist_data:
|
|
||||||
source_list_artist.append(Source(self.SOURCE_TYPE, _parse_artist_url(artist_data["@id"])))
|
|
||||||
|
|
||||||
song = Song(
|
song = Song(
|
||||||
title=clean_song_title(data["name"], artist_name=artist_data["name"]),
|
title=clean_song_title(data["name"], artist_name=artist_data["name"]),
|
||||||
source_list=source_list,
|
source_list=[source, Source(self.SOURCE_TYPE, data.get("mainEntityOfPage", data["@id"]), audio_url=mp3_url)],
|
||||||
album_list=[Album(
|
album_list=[Album(
|
||||||
title=album_data["name"].strip(),
|
title=album_data["name"].strip(),
|
||||||
date=ID3Timestamp.strptime(data["datePublished"], "%d %b %Y %H:%M:%S %Z"),
|
date=ID3Timestamp.strptime(data["datePublished"], "%d %b %Y %H:%M:%S %Z"),
|
||||||
source_list=source_list_album
|
source_list=[Source(self.SOURCE_TYPE, album_data["@id"])]
|
||||||
)],
|
)],
|
||||||
artist_list=[Artist(
|
artist_list=[Artist(
|
||||||
name=artist_data["name"].strip(),
|
name=artist_data["name"].strip(),
|
||||||
source_list=source_list_artist
|
source_list=[Source(self.SOURCE_TYPE, _parse_artist_url(artist_data["@id"]))]
|
||||||
)],
|
)],
|
||||||
lyrics_list=self._fetch_lyrics(soup=soup)
|
lyrics_list=self._fetch_lyrics(soup=soup)
|
||||||
)
|
)
|
||||||
|
@ -1,23 +1,33 @@
|
|||||||
import simplejson as json
|
|
||||||
from json_unescape import escape_json, unescape_json
|
|
||||||
from enum import Enum
|
|
||||||
from typing import List, Optional, Type
|
from typing import List, Optional, Type
|
||||||
from urllib.parse import urlencode, urlparse, urlunparse
|
from urllib.parse import urlparse, urlunparse, urlencode
|
||||||
|
import json
|
||||||
import pycountry
|
from enum import Enum
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
import pycountry
|
||||||
|
|
||||||
from ..connection import Connection
|
from ..objects import Source, DatabaseObject
|
||||||
from ..objects import (Album, Artist, ArtworkCollection, Contact,
|
|
||||||
DatabaseObject, FormattedText, ID3Timestamp, Label,
|
|
||||||
Lyrics, Song, Source, SourceType, Target)
|
|
||||||
from ..utils import dump_to_file, traverse_json_path
|
|
||||||
from ..utils.config import logging_settings, main_settings
|
|
||||||
from ..utils.enums import ALL_SOURCE_TYPES, SourceType
|
|
||||||
from ..utils.shared import DEBUG
|
|
||||||
from ..utils.string_processing import clean_song_title
|
|
||||||
from ..utils.support_classes.download_result import DownloadResult
|
|
||||||
from .abstract import Page
|
from .abstract import Page
|
||||||
|
from ..objects import (
|
||||||
|
Artist,
|
||||||
|
Source,
|
||||||
|
SourceType,
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
Label,
|
||||||
|
Target,
|
||||||
|
Contact,
|
||||||
|
ID3Timestamp,
|
||||||
|
Lyrics,
|
||||||
|
FormattedText,
|
||||||
|
Artwork,
|
||||||
|
)
|
||||||
|
from ..connection import Connection
|
||||||
|
from ..utils import dump_to_file, traverse_json_path
|
||||||
|
from ..utils.enums import SourceType, ALL_SOURCE_TYPES
|
||||||
|
from ..utils.support_classes.download_result import DownloadResult
|
||||||
|
from ..utils.string_processing import clean_song_title
|
||||||
|
from ..utils.config import main_settings, logging_settings
|
||||||
|
from ..utils.shared import DEBUG
|
||||||
|
|
||||||
if DEBUG:
|
if DEBUG:
|
||||||
from ..utils import dump_to_file
|
from ..utils import dump_to_file
|
||||||
@ -46,34 +56,34 @@ class Genius(Page):
|
|||||||
|
|
||||||
return Song
|
return Song
|
||||||
|
|
||||||
def add_to_artwork(self, artwork: ArtworkCollection, url: str):
|
def add_to_artwork(self, artwork: Artwork, url: str):
|
||||||
if url is None:
|
if url is None:
|
||||||
return
|
return
|
||||||
|
|
||||||
url_frags = url.split(".")
|
url_frags = url.split(".")
|
||||||
if len(url_frags) < 2:
|
if len(url_frags) < 2:
|
||||||
artwork.add_data(url=url)
|
artwork.append(url=url)
|
||||||
return
|
return
|
||||||
|
|
||||||
dimensions = url_frags[-2].split("x")
|
dimensions = url_frags[-2].split("x")
|
||||||
if len(dimensions) < 2:
|
if len(dimensions) < 2:
|
||||||
artwork.add_data(url=url)
|
artwork.append(url=url)
|
||||||
return
|
return
|
||||||
|
|
||||||
if len(dimensions) == 3:
|
if len(dimensions) == 3:
|
||||||
dimensions = dimensions[:-1]
|
dimensions = dimensions[:-1]
|
||||||
|
|
||||||
try:
|
try:
|
||||||
artwork.add_data(url=url, width=int(dimensions[0]), height=int(dimensions[1]))
|
artwork.append(url=url, width=int(dimensions[0]), height=int(dimensions[1]))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
artwork.add_data(url=url)
|
artwork.append(url=url)
|
||||||
|
|
||||||
def parse_api_object(self, data: dict) -> Optional[DatabaseObject]:
|
def parse_api_object(self, data: dict) -> Optional[DatabaseObject]:
|
||||||
if data is None:
|
if data is None:
|
||||||
return None
|
return None
|
||||||
object_type = data.get("_type")
|
object_type = data.get("_type")
|
||||||
|
|
||||||
artwork = ArtworkCollection()
|
artwork = Artwork()
|
||||||
self.add_to_artwork(artwork, data.get("header_image_url"))
|
self.add_to_artwork(artwork, data.get("header_image_url"))
|
||||||
self.add_to_artwork(artwork, data.get("image_url"))
|
self.add_to_artwork(artwork, data.get("image_url"))
|
||||||
|
|
||||||
@ -269,8 +279,7 @@ class Genius(Page):
|
|||||||
# get the contents that are between `JSON.parse('` and `');`
|
# get the contents that are between `JSON.parse('` and `');`
|
||||||
content = self.get_json_content_from_response(r, start="window.__PRELOADED_STATE__ = JSON.parse('", end="');\n window.__APP_CONFIG__ = ")
|
content = self.get_json_content_from_response(r, start="window.__PRELOADED_STATE__ = JSON.parse('", end="');\n window.__APP_CONFIG__ = ")
|
||||||
if content is not None:
|
if content is not None:
|
||||||
#IMPLEMENT FIX FROM HAZEL
|
content = content.replace("\\\\", "\\").replace('\\"', '"').replace("\\'", "'")
|
||||||
content = escape_json(content)
|
|
||||||
data = json.loads(content)
|
data = json.loads(content)
|
||||||
|
|
||||||
lyrics_html = traverse_json_path(data, "songPage.lyricsData.body.html", default=None)
|
lyrics_html = traverse_json_path(data, "songPage.lyricsData.body.html", default=None)
|
||||||
|
145
music_kraken/pages/musicbrainz.py
Normal file
145
music_kraken/pages/musicbrainz.py
Normal file
@ -0,0 +1,145 @@
|
|||||||
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import Enum
|
||||||
|
from typing import List, Optional, Type, Union, Generator, Dict, Any
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
import pycountry
|
||||||
|
import musicbrainzngs
|
||||||
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
|
from ..connection import Connection
|
||||||
|
from .abstract import Page
|
||||||
|
from ..utils.enums import SourceType, ALL_SOURCE_TYPES
|
||||||
|
from ..utils.enums.album import AlbumType, AlbumStatus
|
||||||
|
from ..objects import (
|
||||||
|
Artist,
|
||||||
|
Source,
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
ID3Timestamp,
|
||||||
|
FormattedText,
|
||||||
|
Label,
|
||||||
|
Target,
|
||||||
|
DatabaseObject,
|
||||||
|
Lyrics,
|
||||||
|
Artwork
|
||||||
|
)
|
||||||
|
from ..utils.config import logging_settings, main_settings
|
||||||
|
from ..utils import string_processing, shared
|
||||||
|
from ..utils.string_processing import clean_song_title
|
||||||
|
from ..utils.support_classes.query import Query
|
||||||
|
from ..utils.support_classes.download_result import DownloadResult
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class Musicbrainz(Page):
|
||||||
|
SOURCE_TYPE = ALL_SOURCE_TYPES.MUSICBRAINZ
|
||||||
|
|
||||||
|
HOST = "https://musicbrainz.org"
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
musicbrainzngs.set_useragent("mk", "1")
|
||||||
|
|
||||||
|
super().__init__(*args, **kwargs)
|
||||||
|
|
||||||
|
def general_search(self, search_query: str) -> List[DatabaseObject]:
|
||||||
|
search_results = []
|
||||||
|
|
||||||
|
#Artist
|
||||||
|
search_results += self.artist_search(search_query).copy()
|
||||||
|
|
||||||
|
#Album
|
||||||
|
search_results += self.album_search(search_query).copy()
|
||||||
|
|
||||||
|
#Song
|
||||||
|
search_results += self.song_search(search_query).copy()
|
||||||
|
|
||||||
|
return search_results
|
||||||
|
|
||||||
|
def artist_search(self, search_query: str) -> List[Artist]:
|
||||||
|
artist_list = []
|
||||||
|
|
||||||
|
#Artist
|
||||||
|
artist_dict_list: list = musicbrainzngs.search_artists(search_query)['artist-list']
|
||||||
|
artist_source_list: List[Source] = []
|
||||||
|
for artist_dict in artist_dict_list:
|
||||||
|
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/artist/" + artist_dict['id']))
|
||||||
|
artist_list.append(Artist(
|
||||||
|
name=artist_dict['name'],
|
||||||
|
source_list=artist_source_list
|
||||||
|
))
|
||||||
|
|
||||||
|
return artist_list
|
||||||
|
|
||||||
|
def song_search(self, search_query: str) -> List[Song]:
|
||||||
|
song_list = []
|
||||||
|
|
||||||
|
#Song
|
||||||
|
song_dict_list: list = musicbrainzngs.search_recordings(search_query)['recording-list']
|
||||||
|
song_source_list: List[Source] = []
|
||||||
|
for song_dict in song_dict_list:
|
||||||
|
song_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/recording/" + song_dict['id']))
|
||||||
|
song_list.append(Song(
|
||||||
|
title=song_dict['title'],
|
||||||
|
source_list=song_source_list
|
||||||
|
))
|
||||||
|
|
||||||
|
return song_list
|
||||||
|
|
||||||
|
def album_search(self, search_query: str) -> List[Album]:
|
||||||
|
album_list = []
|
||||||
|
|
||||||
|
#Album
|
||||||
|
album_dict_list: list = musicbrainzngs.search_release_groups(search_query)['release-group-list']
|
||||||
|
album_source_list: List[Source] = []
|
||||||
|
for album_dict in album_dict_list:
|
||||||
|
album_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/release-group/" + album_dict['id']))
|
||||||
|
album_list.append(Album(
|
||||||
|
title=album_dict['title'],
|
||||||
|
source_list=album_source_list
|
||||||
|
))
|
||||||
|
|
||||||
|
return album_list
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
|
||||||
|
album_list = []
|
||||||
|
|
||||||
|
#Album
|
||||||
|
album_dict_list: list = musicbrainzngs.search_release_groups(search_query)['release-group-list']
|
||||||
|
album_source_list: List[Source] = []
|
||||||
|
for album_dict in album_dict_list:
|
||||||
|
album_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/release-group/" + album_dict['id']))
|
||||||
|
album_list.append(Album(
|
||||||
|
title=album_dict['title'],
|
||||||
|
source_list=album_source_list
|
||||||
|
))
|
||||||
|
|
||||||
|
def fetch_artist(self, source: Source, stop_at_level: int = 1) -> Artist:
|
||||||
|
artist_list = []
|
||||||
|
|
||||||
|
#Artist
|
||||||
|
artist_dict_list: list = musicbrainzngs.search_artists(search_query)['artist-list']
|
||||||
|
artist_source_list: List[Source] = []
|
||||||
|
for artist_dict in artist_dict_list:
|
||||||
|
artist_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/artist/" + artist_dict['id']))
|
||||||
|
artist_list.append(Artist(
|
||||||
|
name=artist_dict['name'],
|
||||||
|
source_list=artist_source_list,
|
||||||
|
))
|
||||||
|
|
||||||
|
def fetch_song(self, source: Source, stop_at_level: int = 1) -> Song:
|
||||||
|
song_list = []
|
||||||
|
|
||||||
|
#Song
|
||||||
|
song_dict_list: list = musicbrainzngs.search_recordings(search_query)['recording-list']
|
||||||
|
song_source_list: List[Source] = []
|
||||||
|
for song_dict in song_dict_list:
|
||||||
|
song_source_list.append(Source(self.SOURCE_TYPE, self.HOST + "/recording/" + song_dict['id']))
|
||||||
|
song_list.append(Song(
|
||||||
|
title=song_dict['title'],
|
||||||
|
source_list=song_source_list
|
||||||
|
))
|
||||||
|
|
||||||
|
|
@ -1,25 +1,34 @@
|
|||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from enum import Enum
|
||||||
from typing import Any, Dict, Generator, List, Optional, Type, Union
|
from typing import List, Optional, Type, Union, Generator, Dict, Any
|
||||||
from urllib.parse import urlparse
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
import pycountry
|
import pycountry
|
||||||
from bs4 import BeautifulSoup
|
from bs4 import BeautifulSoup
|
||||||
|
|
||||||
from ..connection import Connection
|
from ..connection import Connection
|
||||||
from ..objects import (Album, Artist, DatabaseObject,
|
|
||||||
FormattedText, ID3Timestamp, Label, Lyrics, Song,
|
|
||||||
Source, Target)
|
|
||||||
from ..objects.artwork import (Artwork, ArtworkVariant, ArtworkCollection)
|
|
||||||
from ..utils import shared, string_processing
|
|
||||||
from ..utils.config import logging_settings, main_settings
|
|
||||||
from ..utils.enums import ALL_SOURCE_TYPES, SourceType
|
|
||||||
from ..utils.enums.album import AlbumStatus, AlbumType
|
|
||||||
from ..utils.string_processing import clean_song_title
|
|
||||||
from ..utils.support_classes.download_result import DownloadResult
|
|
||||||
from ..utils.support_classes.query import Query
|
|
||||||
from .abstract import Page
|
from .abstract import Page
|
||||||
|
from ..utils.enums import SourceType, ALL_SOURCE_TYPES
|
||||||
|
from ..utils.enums.album import AlbumType, AlbumStatus
|
||||||
|
from ..objects import (
|
||||||
|
Artist,
|
||||||
|
Source,
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
ID3Timestamp,
|
||||||
|
FormattedText,
|
||||||
|
Label,
|
||||||
|
Target,
|
||||||
|
DatabaseObject,
|
||||||
|
Lyrics,
|
||||||
|
Artwork
|
||||||
|
)
|
||||||
|
from ..utils.config import logging_settings, main_settings
|
||||||
|
from ..utils import string_processing, shared
|
||||||
|
from ..utils.string_processing import clean_song_title
|
||||||
|
from ..utils.support_classes.query import Query
|
||||||
|
from ..utils.support_classes.download_result import DownloadResult
|
||||||
|
|
||||||
"""
|
"""
|
||||||
https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent
|
https://musify.club/artist/ghost-bath-280348?_pjax=#bodyContent
|
||||||
@ -448,17 +457,17 @@ class Musify(Page):
|
|||||||
for album_info in soup.find_all("ul", {"class": "album-info"}):
|
for album_info in soup.find_all("ul", {"class": "album-info"}):
|
||||||
list_element: BeautifulSoup = album_info.find("li")
|
list_element: BeautifulSoup = album_info.find("li")
|
||||||
|
|
||||||
if list_element is not None:
|
if list_element is not None:
|
||||||
artist_soup: BeautifulSoup
|
artist_soup: BeautifulSoup
|
||||||
for artist_soup in list_element.find_all("a"):
|
for artist_soup in list_element.find_all("a"):
|
||||||
artist_source_list = []
|
artist_source_list = []
|
||||||
href = artist_soup["href"]
|
href = artist_soup["href"]
|
||||||
if href is not None:
|
if href is not None:
|
||||||
artist_source_list = [Source(self.SOURCE_TYPE, self.HOST + href)]
|
artist_source_list = [Source(self.SOURCE_TYPE, self.HOST + href)]
|
||||||
artist_list.append(Artist(
|
artist_list.append(Artist(
|
||||||
name=artist_soup.text.strip(),
|
name=artist_soup.text.strip(),
|
||||||
source_list=artist_source_list
|
source_list=artist_source_list
|
||||||
))
|
))
|
||||||
|
|
||||||
# breadcrums
|
# breadcrums
|
||||||
breadcrumb_list_element_list: List[BeautifulSoup] = soup.find_all("ol", {"class": "breadcrumb"})
|
breadcrumb_list_element_list: List[BeautifulSoup] = soup.find_all("ol", {"class": "breadcrumb"})
|
||||||
@ -476,11 +485,11 @@ class Musify(Page):
|
|||||||
|
|
||||||
track_name = list_points[4].text.strip()
|
track_name = list_points[4].text.strip()
|
||||||
|
|
||||||
# album artwork
|
# artwork
|
||||||
artwork: ArtworkCollection = ArtworkCollection()
|
artwork: Artwork = Artwork()
|
||||||
album_image_element_list: List[BeautifulSoup] = soup.find_all("img", {"class": "album-img"})
|
album_image_element_list: List[BeautifulSoup] = soup.find_all("img", {"class": "album-img"})
|
||||||
for album_image_element in album_image_element_list:
|
for album_image_element in album_image_element_list:
|
||||||
artwork.add_data(url=album_image_element.get("data-src", album_image_element.get("src")))
|
artwork.append(url=album_image_element.get("data-src", album_image_element.get("src")))
|
||||||
|
|
||||||
# lyrics
|
# lyrics
|
||||||
lyrics_container: List[BeautifulSoup] = soup.find_all("div", {"id": "tabLyrics"})
|
lyrics_container: List[BeautifulSoup] = soup.find_all("div", {"id": "tabLyrics"})
|
||||||
@ -745,18 +754,11 @@ class Musify(Page):
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
|
self.LOGGER.debug(f"Raw datetime doesn't match time format %Y-%m-%d: {raw_datetime}")
|
||||||
|
|
||||||
# album artwork
|
|
||||||
album_artwork: ArtworkCollection = ArtworkCollection()
|
|
||||||
album_artwork_list: List[BeautifulSoup] = soup.find_all("img", {"class":"artist-img"})
|
|
||||||
for album_artwork in album_artwork_list:
|
|
||||||
album_artwork.add_data(url=album_artwork.get("data-src", album_artwork.get("src")))
|
|
||||||
|
|
||||||
return Album(
|
return Album(
|
||||||
title=name,
|
title=name,
|
||||||
source_list=source_list,
|
source_list=source_list,
|
||||||
artist_list=artist_list,
|
artist_list=artist_list,
|
||||||
date=date,
|
date=date
|
||||||
artwork=album_artwork
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
|
def fetch_album(self, source: Source, stop_at_level: int = 1) -> Album:
|
||||||
@ -793,8 +795,6 @@ class Musify(Page):
|
|||||||
new_song = self._parse_song_card(card_soup)
|
new_song = self._parse_song_card(card_soup)
|
||||||
album.song_collection.append(new_song)
|
album.song_collection.append(new_song)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
album.update_tracksort()
|
album.update_tracksort()
|
||||||
|
|
||||||
return album
|
return album
|
||||||
@ -914,18 +914,11 @@ class Musify(Page):
|
|||||||
if note_soup is not None:
|
if note_soup is not None:
|
||||||
notes.html = note_soup.decode_contents()
|
notes.html = note_soup.decode_contents()
|
||||||
|
|
||||||
# get artist profile artwork
|
|
||||||
main_artist_artwork: ArtworkCollection = ArtworkCollection()
|
|
||||||
artist_image_element_list: List[BeautifulSoup] = soup.find_all("img", {"class":"artist-img"})
|
|
||||||
for artist_image_element in artist_image_element_list:
|
|
||||||
main_artist_artwork.add_data(url=artist_image_element.get("data-src", artist_image_element.get("src")))
|
|
||||||
|
|
||||||
return Artist(
|
return Artist(
|
||||||
name=name,
|
name=name,
|
||||||
country=country,
|
country=country,
|
||||||
source_list=source_list,
|
source_list=source_list,
|
||||||
notes=notes,
|
notes=notes
|
||||||
artwork=main_artist_artwork
|
|
||||||
)
|
)
|
||||||
|
|
||||||
def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None, **kwargs) -> Album:
|
def _parse_album_card(self, album_card: BeautifulSoup, artist_name: str = None, **kwargs) -> Album:
|
||||||
@ -1063,29 +1056,19 @@ class Musify(Page):
|
|||||||
|
|
||||||
artist.album_collection.append(album)
|
artist.album_collection.append(album)
|
||||||
|
|
||||||
def _fetch_artist_artwork(self, source: str, artist: Artist, **kwargs):
|
|
||||||
# artist artwork
|
|
||||||
artwork_gallery = self.get_soup_from_response(self.connection.get(source.strip().strip("/") + "/photos"))
|
|
||||||
if artwork_gallery is not None:
|
|
||||||
gallery_body_content: BeautifulSoup = artwork_gallery.find(id="bodyContent")
|
|
||||||
gallery_image_element_list: List[BeautifulSoup] = gallery_body_content.find_all("img")
|
|
||||||
for gallery_image_element in gallery_image_element_list:
|
|
||||||
artist.artwork.append(ArtworkVariant(url=gallery_image_element.get("data-src", gallery_image_element.get("src")), width=247, heigth=247))
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_artist(self, source: Source, **kwargs) -> Artist:
|
def fetch_artist(self, source: Source, **kwargs) -> Artist:
|
||||||
"""
|
"""
|
||||||
TODO
|
TODO
|
||||||
[x] discography
|
[x] discography
|
||||||
[x] attributes
|
[x] attributes
|
||||||
[x] picture gallery
|
[] picture gallery
|
||||||
"""
|
"""
|
||||||
|
|
||||||
url = parse_url(source.url)
|
url = parse_url(source.url)
|
||||||
|
|
||||||
artist = self._fetch_initial_artist(url, source=source, **kwargs)
|
artist = self._fetch_initial_artist(url, source=source, **kwargs)
|
||||||
self._fetch_artist_discography(artist, url, artist.name, **kwargs)
|
self._fetch_artist_discography(artist, url, artist.name, **kwargs)
|
||||||
self._fetch_artist_artwork(url.url, artist, **kwargs)
|
|
||||||
return artist
|
return artist
|
||||||
|
|
||||||
def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label:
|
def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label:
|
||||||
|
@ -1,33 +1,46 @@
|
|||||||
from __future__ import annotations, unicode_literals
|
from __future__ import unicode_literals, annotations
|
||||||
|
|
||||||
import json
|
from typing import Dict, List, Optional, Set, Type
|
||||||
|
from urllib.parse import urlparse, urlunparse, quote, parse_qs, urlencode
|
||||||
import logging
|
import logging
|
||||||
import random
|
import random
|
||||||
import re
|
import json
|
||||||
from collections import defaultdict
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
import re
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from typing import Dict, List, Optional, Set, Type
|
from collections import defaultdict
|
||||||
from urllib.parse import parse_qs, quote, urlencode, urlparse, urlunparse
|
|
||||||
|
|
||||||
import youtube_dl
|
import youtube_dl
|
||||||
from youtube_dl.extractor.youtube import YoutubeIE
|
from youtube_dl.extractor.youtube import YoutubeIE
|
||||||
from youtube_dl.utils import DownloadError
|
from youtube_dl.utils import DownloadError
|
||||||
|
|
||||||
from ...connection import Connection
|
|
||||||
from ...objects import Album, Artist, ArtworkCollection
|
|
||||||
from ...objects import DatabaseObject as DataObject
|
|
||||||
from ...objects import (FormattedText, ID3Timestamp, Label, Lyrics, Song,
|
|
||||||
Source, Target)
|
|
||||||
from ...utils import dump_to_file, get_current_millis, traverse_json_path
|
|
||||||
from ...utils.config import logging_settings, main_settings, youtube_settings
|
|
||||||
from ...utils.enums import ALL_SOURCE_TYPES, SourceType
|
|
||||||
from ...utils.enums.album import AlbumType
|
|
||||||
from ...utils.exception.config import SettingValueError
|
from ...utils.exception.config import SettingValueError
|
||||||
|
from ...utils.config import main_settings, youtube_settings, logging_settings
|
||||||
from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING
|
from ...utils.shared import DEBUG, DEBUG_YOUTUBE_INITIALIZING
|
||||||
from ...utils.string_processing import clean_song_title
|
from ...utils.string_processing import clean_song_title
|
||||||
from ...utils.support_classes.download_result import DownloadResult
|
from ...utils import get_current_millis, traverse_json_path
|
||||||
|
|
||||||
|
from ...utils import dump_to_file
|
||||||
|
|
||||||
from ..abstract import Page
|
from ..abstract import Page
|
||||||
|
from ...objects import (
|
||||||
|
DatabaseObject as DataObject,
|
||||||
|
Source,
|
||||||
|
FormattedText,
|
||||||
|
ID3Timestamp,
|
||||||
|
Artwork,
|
||||||
|
Artist,
|
||||||
|
Song,
|
||||||
|
Album,
|
||||||
|
Label,
|
||||||
|
Target,
|
||||||
|
Lyrics,
|
||||||
|
)
|
||||||
|
from ...connection import Connection
|
||||||
|
from ...utils.enums import SourceType, ALL_SOURCE_TYPES
|
||||||
|
from ...utils.enums.album import AlbumType
|
||||||
|
from ...utils.support_classes.download_result import DownloadResult
|
||||||
|
|
||||||
from ._list_render import parse_renderer
|
from ._list_render import parse_renderer
|
||||||
from ._music_object_render import parse_run_element
|
from ._music_object_render import parse_run_element
|
||||||
from .super_youtube import SuperYouTube
|
from .super_youtube import SuperYouTube
|
||||||
@ -425,7 +438,6 @@ class YoutubeMusic(SuperYouTube):
|
|||||||
data: dict = r.json()
|
data: dict = r.json()
|
||||||
header = data.get("header", {})
|
header = data.get("header", {})
|
||||||
musicDetailHeaderRenderer = header.get("musicDetailHeaderRenderer", {})
|
musicDetailHeaderRenderer = header.get("musicDetailHeaderRenderer", {})
|
||||||
musicImmersiveHeaderRenderer = header.get("musicImmersiveHeaderRenderer", {})
|
|
||||||
|
|
||||||
title_runs: List[dict] = musicDetailHeaderRenderer.get("title", {}).get("runs", [])
|
title_runs: List[dict] = musicDetailHeaderRenderer.get("title", {}).get("runs", [])
|
||||||
subtitle_runs: List[dict] = musicDetailHeaderRenderer.get("subtitle", {}).get("runs", [])
|
subtitle_runs: List[dict] = musicDetailHeaderRenderer.get("subtitle", {}).get("runs", [])
|
||||||
@ -438,11 +450,6 @@ class YoutubeMusic(SuperYouTube):
|
|||||||
renderer_list = r.json().get("contents", {}).get("singleColumnBrowseResultsRenderer", {}).get("tabs", [{}])[
|
renderer_list = r.json().get("contents", {}).get("singleColumnBrowseResultsRenderer", {}).get("tabs", [{}])[
|
||||||
0].get("tabRenderer", {}).get("content", {}).get("sectionListRenderer", {}).get("contents", [])
|
0].get("tabRenderer", {}).get("content", {}).get("sectionListRenderer", {}).get("contents", [])
|
||||||
|
|
||||||
# fetch artist artwork
|
|
||||||
artist_thumbnails = musicImmersiveHeaderRenderer.get("thumbnail", {}).get("musicThumbnailRenderer", {}).get("thumbnail", {}).get("thumbnails", {})
|
|
||||||
for artist_thumbnail in artist_thumbnails:
|
|
||||||
artist.artwork.append(artist_thumbnail)
|
|
||||||
|
|
||||||
if DEBUG:
|
if DEBUG:
|
||||||
for i, content in enumerate(renderer_list):
|
for i, content in enumerate(renderer_list):
|
||||||
dump_to_file(f"{i}-artists-renderer.json", json.dumps(content), is_json=True, exit_after_dump=False)
|
dump_to_file(f"{i}-artists-renderer.json", json.dumps(content), is_json=True, exit_after_dump=False)
|
||||||
@ -490,11 +497,6 @@ class YoutubeMusic(SuperYouTube):
|
|||||||
header = data.get("header", {})
|
header = data.get("header", {})
|
||||||
musicDetailHeaderRenderer = header.get("musicDetailHeaderRenderer", {})
|
musicDetailHeaderRenderer = header.get("musicDetailHeaderRenderer", {})
|
||||||
|
|
||||||
# album artwork
|
|
||||||
album_thumbnails = musicDetailHeaderRenderer.get("thumbnail", {}).get("croppedSquareThumbnailRenderer", {}).get("thumbnail", {}).get("thumbnails", {})
|
|
||||||
for album_thumbnail in album_thumbnails:
|
|
||||||
album.artwork.append(value=album_thumbnail)
|
|
||||||
|
|
||||||
title_runs: List[dict] = musicDetailHeaderRenderer.get("title", {}).get("runs", [])
|
title_runs: List[dict] = musicDetailHeaderRenderer.get("title", {}).get("runs", [])
|
||||||
subtitle_runs: List[dict] = musicDetailHeaderRenderer.get("subtitle", {}).get("runs", [])
|
subtitle_runs: List[dict] = musicDetailHeaderRenderer.get("subtitle", {}).get("runs", [])
|
||||||
|
|
||||||
@ -644,7 +646,7 @@ class YoutubeMusic(SuperYouTube):
|
|||||||
note=ydl_res.get("descriptions"),
|
note=ydl_res.get("descriptions"),
|
||||||
album_list=album_list,
|
album_list=album_list,
|
||||||
length=int(ydl_res.get("duration", 0)) * 1000,
|
length=int(ydl_res.get("duration", 0)) * 1000,
|
||||||
artwork=ArtworkCollection(*ydl_res.get("thumbnails", [])),
|
artwork=Artwork(*ydl_res.get("thumbnails", [])),
|
||||||
artist_list=artist_list,
|
artist_list=artist_list,
|
||||||
source_list=[Source(
|
source_list=[Source(
|
||||||
self.SOURCE_TYPE,
|
self.SOURCE_TYPE,
|
||||||
@ -683,7 +685,7 @@ class YoutubeMusic(SuperYouTube):
|
|||||||
for album in song.album_list:
|
for album in song.album_list:
|
||||||
album.album_type = AlbumType.LIVE_ALBUM
|
album.album_type = AlbumType.LIVE_ALBUM
|
||||||
for thumbnail in video_details.get("thumbnails", []):
|
for thumbnail in video_details.get("thumbnails", []):
|
||||||
song.artwork.add_data(**thumbnail)
|
song.artwork.append(**thumbnail)
|
||||||
|
|
||||||
song.lyrics_collection.append(self.fetch_lyrics(browse_id, playlist_id=request_data.get("playlistId")))
|
song.lyrics_collection.append(self.fetch_lyrics(browse_id, playlist_id=request_data.get("playlistId")))
|
||||||
|
|
||||||
|
@ -1,18 +1,15 @@
|
|||||||
import inspect
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
from datetime import datetime
|
import inspect
|
||||||
from functools import lru_cache
|
from typing import List, Union
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, List, Union
|
|
||||||
|
|
||||||
|
from .shared import DEBUG, DEBUG_LOGGING, DEBUG_DUMP, DEBUG_TRACE, DEBUG_OBJECT_TRACE, DEBUG_OBJECT_TRACE_CALLSTACK
|
||||||
from .config import config, read_config, write_config
|
from .config import config, read_config, write_config
|
||||||
from .enums.colors import BColors
|
from .enums.colors import BColors
|
||||||
from .hacking import merge_args
|
|
||||||
from .path_manager import LOCATIONS
|
from .path_manager import LOCATIONS
|
||||||
from .shared import (DEBUG, DEBUG_DUMP, DEBUG_LOGGING, DEBUG_OBJECT_TRACE,
|
from .hacking import merge_args
|
||||||
DEBUG_OBJECT_TRACE_CALLSTACK, DEBUG_TRACE, URL_PATTERN)
|
|
||||||
from .string_processing import hash_url, is_url, unify
|
|
||||||
|
|
||||||
"""
|
"""
|
||||||
IO functions
|
IO functions
|
||||||
@ -129,34 +126,3 @@ def get_current_millis() -> int:
|
|||||||
|
|
||||||
def get_unix_time() -> int:
|
def get_unix_time() -> int:
|
||||||
return int(datetime.now().timestamp())
|
return int(datetime.now().timestamp())
|
||||||
|
|
||||||
|
|
||||||
@lru_cache
|
|
||||||
def custom_hash(value: Any) -> int:
|
|
||||||
if is_url(value):
|
|
||||||
value = hash_url(value)
|
|
||||||
elif isinstance(value, str):
|
|
||||||
try:
|
|
||||||
value = int(value)
|
|
||||||
except ValueError:
|
|
||||||
value = unify(value)
|
|
||||||
|
|
||||||
return hash(value)
|
|
||||||
|
|
||||||
|
|
||||||
def create_dataclass_instance(t, data: dict):
|
|
||||||
"""Creates an instance of a dataclass with the given data.
|
|
||||||
It filters out all data key, which has no attribute in the dataclass.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
t (Type): The dataclass type class
|
|
||||||
data (dict): the attribute to pass into the constructor
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
Tuple[Type, dict]: The created instance and a dict, containing the data, which was not used in the creation
|
|
||||||
"""
|
|
||||||
|
|
||||||
needed_data = {k: v for k, v in data.items() if k in t.__dataclass_fields__}
|
|
||||||
removed_data = {k: v for k, v in data.items() if k not in t.__dataclass_fields__}
|
|
||||||
|
|
||||||
return t(**needed_data), removed_data
|
|
||||||
|
@ -1,8 +1,11 @@
|
|||||||
from typing import Tuple
|
from typing import Tuple
|
||||||
|
|
||||||
from .config import Config
|
from .config import Config
|
||||||
from .config_files import main_config, logging_config, youtube_config
|
from .config_files import (
|
||||||
|
main_config,
|
||||||
|
logging_config,
|
||||||
|
youtube_config,
|
||||||
|
)
|
||||||
|
|
||||||
_sections: Tuple[Config, ...] = (
|
_sections: Tuple[Config, ...] = (
|
||||||
main_config.config,
|
main_config.config,
|
||||||
|
@ -59,6 +59,11 @@ Reference for the logging formats: https://docs.python.org/3/library/logging.htm
|
|||||||
description="The logger for the musify scraper.",
|
description="The logger for the musify scraper.",
|
||||||
default_value="musify"
|
default_value="musify"
|
||||||
),
|
),
|
||||||
|
LoggerAttribute(
|
||||||
|
name="musicbrainz_logger",
|
||||||
|
description="The logger for the musicbrainz scraper.",
|
||||||
|
default_value="musicbrainz"
|
||||||
|
),
|
||||||
LoggerAttribute(
|
LoggerAttribute(
|
||||||
name="youtube_logger",
|
name="youtube_logger",
|
||||||
description="The logger for the youtube scraper.",
|
description="The logger for the youtube scraper.",
|
||||||
|
@ -18,7 +18,6 @@ config = Config((
|
|||||||
AudioFormatAttribute(name="audio_format", default_value="mp3", description="""Music Kraken will stream the audio into this format.
|
AudioFormatAttribute(name="audio_format", default_value="mp3", description="""Music Kraken will stream the audio into this format.
|
||||||
You can use Audio formats which support ID3.2 and ID3.1,
|
You can use Audio formats which support ID3.2 and ID3.1,
|
||||||
but you will have cleaner Metadata using ID3.2."""),
|
but you will have cleaner Metadata using ID3.2."""),
|
||||||
Attribute(name="image_format", default_value="jpeg", description="This Changes the format in which images are getting downloaded"),
|
|
||||||
|
|
||||||
Attribute(name="result_history", default_value=True, description="""If enabled, you can go back to the previous results.
|
Attribute(name="result_history", default_value=True, description="""If enabled, you can go back to the previous results.
|
||||||
The consequence is a higher meory consumption, because every result is saved."""),
|
The consequence is a higher meory consumption, because every result is saved."""),
|
||||||
@ -29,7 +28,6 @@ The further you choose to be able to go back, the higher the memory usage.
|
|||||||
EmptyLine(),
|
EmptyLine(),
|
||||||
|
|
||||||
Attribute(name="preferred_artwork_resolution", default_value=1000),
|
Attribute(name="preferred_artwork_resolution", default_value=1000),
|
||||||
Attribute(name="download_artist_artworks", default_value=True, description="Enables the fetching of artist galleries."),
|
|
||||||
|
|
||||||
EmptyLine(),
|
EmptyLine(),
|
||||||
|
|
||||||
@ -46,7 +44,6 @@ This means for example, the Studio Albums and EP's are always in front of Single
|
|||||||
- album_type
|
- album_type
|
||||||
The folder music kraken should put the songs into."""),
|
The folder music kraken should put the songs into."""),
|
||||||
Attribute(name="download_file", default_value="{song}.{audio_format}", description="The filename of the audio file."),
|
Attribute(name="download_file", default_value="{song}.{audio_format}", description="The filename of the audio file."),
|
||||||
Attribute(name="artist_artwork_path", default_value="{genre}/{artist}/{artist}_{image_number}.{image_format}", description="The Path to download artist images to."),
|
|
||||||
SelectAttribute(name="album_type_blacklist", default_value=[
|
SelectAttribute(name="album_type_blacklist", default_value=[
|
||||||
"Compilation Album",
|
"Compilation Album",
|
||||||
"Live Album",
|
"Live Album",
|
||||||
@ -155,13 +152,10 @@ class SettingsStructure(TypedDict):
|
|||||||
|
|
||||||
# artwork
|
# artwork
|
||||||
preferred_artwork_resolution: int
|
preferred_artwork_resolution: int
|
||||||
image_format: str
|
|
||||||
download_artist_artworks: bool
|
|
||||||
|
|
||||||
# paths
|
# paths
|
||||||
music_directory: Path
|
music_directory: Path
|
||||||
temp_directory: Path
|
temp_directory: Path
|
||||||
artist_artwork_path: Path
|
|
||||||
log_file: Path
|
log_file: Path
|
||||||
not_a_genre_regex: List[str]
|
not_a_genre_regex: List[str]
|
||||||
ffmpeg_binary: Path
|
ffmpeg_binary: Path
|
||||||
|
@ -1,11 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from enum import Enum
|
from typing import Optional, TYPE_CHECKING, Type
|
||||||
from typing import TYPE_CHECKING, Optional, Type
|
|
||||||
|
|
||||||
from mutagen.id3 import PictureType
|
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
if TYPE_CHECKING:
|
||||||
from ...pages.abstract import Page
|
from ...pages.abstract import Page
|
||||||
|
|
||||||
@ -56,73 +52,3 @@ class ALL_SOURCE_TYPES:
|
|||||||
MANUAL = SourceType(name="manual")
|
MANUAL = SourceType(name="manual")
|
||||||
|
|
||||||
PRESET = SourceType(name="preset")
|
PRESET = SourceType(name="preset")
|
||||||
|
|
||||||
|
|
||||||
class PictureType(Enum):
|
|
||||||
"""Enumeration of image types defined by the ID3 standard for the APIC
|
|
||||||
frame, but also reused in WMA/FLAC/VorbisComment.
|
|
||||||
|
|
||||||
This is copied from mutagen.id3.PictureType
|
|
||||||
"""
|
|
||||||
|
|
||||||
OTHER = 0
|
|
||||||
|
|
||||||
FILE_ICON = 1
|
|
||||||
"""32x32 pixels 'file icon' (PNG only)"""
|
|
||||||
|
|
||||||
OTHER_FILE_ICON = 2
|
|
||||||
"""Other file icon"""
|
|
||||||
|
|
||||||
COVER_FRONT = 3
|
|
||||||
"""Cover (front)"""
|
|
||||||
|
|
||||||
COVER_BACK = 4
|
|
||||||
"""Cover (back)"""
|
|
||||||
|
|
||||||
LEAFLET_PAGE = 5
|
|
||||||
"""Leaflet page"""
|
|
||||||
|
|
||||||
MEDIA = 6
|
|
||||||
"""Media (e.g. label side of CD)"""
|
|
||||||
|
|
||||||
LEAD_ARTIST = 7
|
|
||||||
"""Lead artist/lead performer/soloist"""
|
|
||||||
|
|
||||||
ARTIST = 8
|
|
||||||
"""Artist/performer"""
|
|
||||||
|
|
||||||
CONDUCTOR = 9
|
|
||||||
"""Conductor"""
|
|
||||||
|
|
||||||
BAND = 10
|
|
||||||
"""Band/Orchestra"""
|
|
||||||
|
|
||||||
COMPOSER = 11
|
|
||||||
"""Composer"""
|
|
||||||
|
|
||||||
LYRICIST = 12
|
|
||||||
"""Lyricist/text writer"""
|
|
||||||
|
|
||||||
RECORDING_LOCATION = 13
|
|
||||||
"""Recording Location"""
|
|
||||||
|
|
||||||
DURING_RECORDING = 14
|
|
||||||
"""During recording"""
|
|
||||||
|
|
||||||
DURING_PERFORMANCE = 15
|
|
||||||
"""During performance"""
|
|
||||||
|
|
||||||
SCREEN_CAPTURE = 16
|
|
||||||
"""Movie/video screen capture"""
|
|
||||||
|
|
||||||
FISH = 17
|
|
||||||
"""A bright colored fish"""
|
|
||||||
|
|
||||||
ILLUSTRATION = 18
|
|
||||||
"""Illustration"""
|
|
||||||
|
|
||||||
BAND_LOGOTYPE = 19
|
|
||||||
"""Band/artist logotype"""
|
|
||||||
|
|
||||||
PUBLISHER_LOGOTYPE = 20
|
|
||||||
"""Publisher/Studio logotype"""
|
|
||||||
|
@ -1,15 +1,13 @@
|
|||||||
import re
|
from typing import Tuple, Union, Optional
|
||||||
|
from pathlib import Path
|
||||||
import string
|
import string
|
||||||
from functools import lru_cache
|
from functools import lru_cache
|
||||||
from pathlib import Path
|
|
||||||
from typing import Any, Optional, Tuple, Union
|
|
||||||
from urllib.parse import ParseResult, parse_qs, urlparse
|
|
||||||
|
|
||||||
from pathvalidate import sanitize_filename
|
|
||||||
from transliterate import translit
|
|
||||||
from transliterate.exceptions import LanguageDetectionError
|
from transliterate.exceptions import LanguageDetectionError
|
||||||
|
from transliterate import translit
|
||||||
|
from pathvalidate import sanitize_filename
|
||||||
|
from urllib.parse import urlparse, ParseResult, parse_qs
|
||||||
|
|
||||||
from .shared import URL_PATTERN
|
|
||||||
|
|
||||||
COMMON_TITLE_APPENDIX_LIST: Tuple[str, ...] = (
|
COMMON_TITLE_APPENDIX_LIST: Tuple[str, ...] = (
|
||||||
"(official video)",
|
"(official video)",
|
||||||
@ -231,13 +229,3 @@ def shorten_display_url(url: str, max_length: int = 150, chars_at_end: int = 4,
|
|||||||
return url
|
return url
|
||||||
|
|
||||||
return url[:max_length] + shorten_string + url[-chars_at_end:]
|
return url[:max_length] + shorten_string + url[-chars_at_end:]
|
||||||
|
|
||||||
def is_url(value: Any) -> bool:
|
|
||||||
if isinstance(value, ParseResult):
|
|
||||||
return True
|
|
||||||
|
|
||||||
if not isinstance(value, str):
|
|
||||||
return True
|
|
||||||
|
|
||||||
# value has to be a string
|
|
||||||
return re.match(URL_PATTERN, value) is not None
|
|
||||||
|
@ -1,13 +1,9 @@
|
|||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, List, Tuple
|
from typing import List, Tuple
|
||||||
|
|
||||||
if TYPE_CHECKING:
|
from ...utils.config import main_settings, logging_settings
|
||||||
from ...objects import Target
|
|
||||||
|
|
||||||
from ...utils.config import logging_settings, main_settings
|
|
||||||
from ...utils.enums.colors import BColors
|
from ...utils.enums.colors import BColors
|
||||||
|
from ...objects import Target
|
||||||
|
|
||||||
UNIT_PREFIXES: List[str] = ["", "k", "m", "g", "t"]
|
UNIT_PREFIXES: List[str] = ["", "k", "m", "g", "t"]
|
||||||
UNIT_DIVISOR = 1024
|
UNIT_DIVISOR = 1024
|
||||||
|
Loading…
Reference in New Issue
Block a user