From ae547c7de269b404be8d096f2f148366f3fed3f2 Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Tue, 20 Jun 2023 19:30:48 +0200 Subject: [PATCH] refactor and fix --- .idea/sqldialects.xml | 6 - src/actual_donwload.py | 4 +- src/music_kraken/__main__.py | 11 +- src/music_kraken/cli/main_downloader.py | 11 +- src/music_kraken/download/page_attributes.py | 4 +- src/music_kraken/objects/metadata.py | 8 +- src/music_kraken/objects/song.py | 6 +- src/music_kraken/pages/abstract.py | 27 ++-- src/music_kraken/static_files/new_db.sql | 81 ---------- .../static_files/temp_database_structure.sql | 144 ------------------ 10 files changed, 50 insertions(+), 252 deletions(-) delete mode 100644 .idea/sqldialects.xml delete mode 100644 src/music_kraken/static_files/new_db.sql delete mode 100644 src/music_kraken/static_files/temp_database_structure.sql diff --git a/.idea/sqldialects.xml b/.idea/sqldialects.xml deleted file mode 100644 index af742c7..0000000 --- a/.idea/sqldialects.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/src/actual_donwload.py b/src/actual_donwload.py index c60650b..591accb 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -19,6 +19,8 @@ if __name__ == "__main__": fetch_youtube_playlist = [ "s: https://yt.artemislena.eu/playlist?list=OLAK5uy_kcUBiDv5ATbl-R20OjNaZ5G28XFanQOmM" ] + + download_youtube_playlist = ["d: https://www.youtube.com/playlist?list=OLAK5uy_lqI_c6aDF9q4DWJ4TBzt1AFQYx_FXfU4E"] youtube_search = [ "s: #a Zombiez", @@ -26,4 +28,4 @@ if __name__ == "__main__": "d: 5" ] - music_kraken.cli.download(genre="test", command_list=fetch_youtube_playlist) + music_kraken.cli.download(genre="test", command_list=download_youtube_playlist, process_metadata_anyway=True) diff --git a/src/music_kraken/__main__.py b/src/music_kraken/__main__.py index 7a249a8..8aab5a9 100644 --- a/src/music_kraken/__main__.py +++ b/src/music_kraken/__main__.py @@ -16,10 +16,16 @@ if __name__ == "__main__": help="Sets the logging level to debug." ) + parser.add_argument( + '-p', '--force-post-process', + action="store_true", + help="If a to downloaded thing is skipped due to being found on disc,\nit will still update the metadata accordingly." + ) + parser.add_argument( '-t', '--test', action="store_true", - help="For the sake of testing. Equals: '-v -g test'" + help="For the sake of testing. Equals: '-vp -g test'" ) # general arguments @@ -114,5 +120,6 @@ if __name__ == "__main__": cli.download( genre=genre, download_all=arguments.all, - direct_download_url=arguments.url + direct_download_url=arguments.url, + process_metadata_anyway=arguments.force_post_process or arguments.test ) \ No newline at end of file diff --git a/src/music_kraken/cli/main_downloader.py b/src/music_kraken/cli/main_downloader.py index c0affea..72b91f9 100644 --- a/src/music_kraken/cli/main_downloader.py +++ b/src/music_kraken/cli/main_downloader.py @@ -143,7 +143,8 @@ class Downloader: exclude_shady: bool = False, max_displayed_options: int = 10, option_digits: int = 3, - genre: str = None + genre: str = None, + process_metadata_anyway: bool = False, ) -> None: self.pages: Pages = Pages(exclude_pages=exclude_pages, exclude_shady=exclude_shady) @@ -156,6 +157,7 @@ class Downloader: self._result_history: List[Results] = [] self.genre = genre or get_genre() + self.process_metadata_anyway = process_metadata_anyway print() print(f"Downloading to: \"{self.genre}\"") @@ -334,7 +336,7 @@ class Downloader: _result_map: Dict[DatabaseObject, DownloadResult] = dict() for database_object in to_download: - r = self.pages.download(music_object=database_object, genre=self.genre, download_all=download_all) + r = self.pages.download(music_object=database_object, genre=self.genre, download_all=download_all, process_metadata_anyway=self.process_metadata_anyway) _result_map[database_object] = r for music_object, result in _result_map.items(): @@ -386,9 +388,10 @@ def download( genre: str = None, download_all: bool = False, direct_download_url: str = None, - command_list: List[str] = None + command_list: List[str] = None, + process_metadata_anyway: bool = False, ): - shell = Downloader(genre=genre) + shell = Downloader(genre=genre, process_metadata_anyway=process_metadata_anyway) if command_list is not None: for command in command_list: diff --git a/src/music_kraken/download/page_attributes.py b/src/music_kraken/download/page_attributes.py index ee7c18d..9b58087 100644 --- a/src/music_kraken/download/page_attributes.py +++ b/src/music_kraken/download/page_attributes.py @@ -74,7 +74,7 @@ class Pages: return music_object - def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: if not isinstance(music_object, INDEPENDENT_DB_OBJECTS): return DownloadResult(error_message=f"{type(music_object).__name__} can't be downloaded.") @@ -82,7 +82,7 @@ class Pages: audio_pages = self._audio_pages_set.intersection(_page_types) for download_page in audio_pages: - return self._page_instances[download_page].download(music_object=music_object, genre=genre, download_all=download_all) + return self._page_instances[download_page].download(music_object=music_object, genre=genre, download_all=download_all, process_metadata_anyway=process_metadata_anyway) return DownloadResult(error_message=f"No audio source has been found for {music_object}.") diff --git a/src/music_kraken/objects/metadata.py b/src/music_kraken/objects/metadata.py index 488af12..69fff8d 100644 --- a/src/music_kraken/objects/metadata.py +++ b/src/music_kraken/objects/metadata.py @@ -16,7 +16,11 @@ class Mapping(Enum): TITLE = "TIT2" ISRC = "TSRC" LENGTH = "TLEN" # in milliseconds - DATE = "TYER" + # The 'Date' frame is a numeric string in the DDMM format containing the date for the recording. This field is always four characters long. + DATE = "TDAT" + # The 'Time' frame is a numeric string in the HHMM format containing the time for the recording. This field is always four characters long. + TIME = "TIME" + YEAR = "TYER" TRACKNUMBER = "TRCK" TOTALTRACKS = "TRCK" # Stored in the same frame with TRACKNUMBER, separated by '/': e.g. '4/9'. TITLESORTORDER = "TSOT" @@ -298,7 +302,7 @@ class Metadata: if id3_dict is not None: self.add_metadata_dict(id3_dict) - def __setitem__(self, frame, value_list: list, override_existing: bool = True): + def __setitem__(self, frame: Mapping, value_list: list, override_existing: bool = True): if type(value_list) != list: raise ValueError(f"can only set attribute to list, not {type(value_list)}") diff --git a/src/music_kraken/objects/song.py b/src/music_kraken/objects/song.py index 6df2001..0845d11 100644 --- a/src/music_kraken/objects/song.py +++ b/src/music_kraken/objects/song.py @@ -295,7 +295,11 @@ class Album(MainObject): id3Mapping.COPYRIGHT: [self.copyright], id3Mapping.LANGUAGE: [self.iso_639_2_lang], id3Mapping.ALBUM_ARTIST: [a.name for a in self.artist_collection], - id3Mapping.DATE: [self.date.timestamp], + id3Mapping.DATE: [self.date.strftime("%d%m")] if self.date.has_year and self.date.has_month else [], + id3Mapping.TIME: [self.date.strftime(("%H%M"))] if self.date.has_hour and self.date.has_minute else [], + id3Mapping.YEAR: [str(self.date.year).zfill(4)] if self.date.has_year else [], + id3Mapping.RELEASE_DATE: [self.date.timestamp], + id3Mapping.ORIGINAL_RELEASE_DATE: [self.date.timestamp], id3Mapping.ALBUMSORTORDER: [str(self.albumsort)] if self.albumsort is not None else [] }) diff --git a/src/music_kraken/pages/abstract.py b/src/music_kraken/pages/abstract.py index 65ad85f..ba982ba 100644 --- a/src/music_kraken/pages/abstract.py +++ b/src/music_kraken/pages/abstract.py @@ -322,7 +322,7 @@ class Page: def fetch_label(self, source: Source, stop_at_level: int = 1) -> Label: return Label() - def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False) -> DownloadResult: + def download(self, music_object: DatabaseObject, genre: str, download_all: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: naming_dict: NamingDict = NamingDict({"genre": genre}) def fill_naming_objects(naming_music_object: DatabaseObject): @@ -340,10 +340,10 @@ class Page: fill_naming_objects(music_object) - return self._download(music_object, naming_dict, download_all) + return self._download(music_object, naming_dict, download_all, process_metadata_anyway=process_metadata_anyway) - def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False, skip_details: bool = False) -> DownloadResult: + def _download(self, music_object: DatabaseObject, naming_dict: NamingDict, download_all: bool = False, skip_details: bool = False, process_metadata_anyway: bool = False) -> DownloadResult: skip_next_details = skip_details # Skips all releases, that are defined in shared.ALBUM_TYPE_BLACKLIST, if download_all is False @@ -360,7 +360,7 @@ class Page: naming_dict.add_object(music_object) if isinstance(music_object, Song): - return self._download_song(music_object, naming_dict) + return self._download_song(music_object, naming_dict, process_metadata_anyway=process_metadata_anyway) download_result: DownloadResult = DownloadResult() @@ -369,11 +369,17 @@ class Page: sub_ordered_music_object: DatabaseObject for sub_ordered_music_object in collection: - download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all, skip_details=skip_next_details)) + download_result.merge(self._download(sub_ordered_music_object, naming_dict.copy(), download_all, skip_details=skip_next_details, process_metadata_anyway=process_metadata_anyway)) return download_result - def _download_song(self, song: Song, naming_dict: NamingDict): + def _download_song(self, song: Song, naming_dict: NamingDict, process_metadata_anyway: bool = False): + if "genre" not in naming_dict and song.genre is not None: + naming_dict["genre"] = song.genre + + if song.genre is None: + song.genre = naming_dict["genre"] + path_parts = Formatter().parse(DOWNLOAD_PATH) file_parts = Formatter().parse(DOWNLOAD_FILE) new_target = Target( @@ -401,18 +407,21 @@ class Page: target: Target for target in song.target_collection: if target.exists: + if process_metadata_anyway: + target.copy_content(temp_target) found_on_disc = True r.found_on_disk += 1 r.add_target(target) - if found_on_disc: + if found_on_disc and not process_metadata_anyway: self.LOGGER.info(f"{song.option_string} already exists, thus not downloading again.") return r source = sources[0] - - r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) + + if not found_on_disc: + r = self.download_song_to_target(source=source, target=temp_target, desc=song.title) if not r.is_fatal_error: diff --git a/src/music_kraken/static_files/new_db.sql b/src/music_kraken/static_files/new_db.sql deleted file mode 100644 index fd9cd0d..0000000 --- a/src/music_kraken/static_files/new_db.sql +++ /dev/null @@ -1,81 +0,0 @@ -CREATE TABLE Song -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - name TEXT, - isrc TEXT, - length INT, -- length is in milliseconds (could be wrong) - tracksort INT, - genre TEXT, - album_id BIGINT, - FOREIGN KEY(album_id) REFERENCES Album(id) -); - - -CREATE TABLE Source -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - type TEXT NOT NULL, - src TEXT NOT NULL, - url TEXT NOT NULL, - certainty INT NOT NULL DEFAULT 0, -- certainty=0 -> it is definitely a valid source - valid BOOLEAN NOT NULL DEFAULT 1, - song_id BIGINT, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - - -CREATE TABLE Album -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - title TEXT, - label TEXT, - album_status TEXT, - language TEXT, - date TEXT, - date_format TEXT, - country TEXT, - barcode TEXT, - albumsort INT, - is_split BOOLEAN NOT NULL DEFAULT 0 -); - -CREATE TABLE Target -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - file TEXT, - path TEXT, - song_id BIGINT UNIQUE, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - -CREATE TABLE Lyrics -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - text TEXT, - language TEXT, - song_id BIGINT, - FOREIGN KEY(song_id) REFERENCES Song(id) -); - -CREATE TABLE Artist -( - id BIGINT AUTO_INCREMENT PRIMARY KEY, - name TEXT -); - -CREATE TABLE SongArtist -( - song_id BIGINT NOT NULL, - artist_id BIGINT NOT NULL, - is_feature BOOLEAN NOT NULL DEFAULT 0, - FOREIGN KEY(song_id) REFERENCES Song(id), - FOREIGN KEY(artist_id) REFERENCES Artist(id) -); - -CREATE TABLE AlbumArtist -( - album_id BIGINT, - artist_id BIGINT, - FOREIGN KEY(album_id) REFERENCES Album(id), - FOREIGN KEY(artist_id) REFERENCES Artist(id) -); diff --git a/src/music_kraken/static_files/temp_database_structure.sql b/src/music_kraken/static_files/temp_database_structure.sql deleted file mode 100644 index b75653a..0000000 --- a/src/music_kraken/static_files/temp_database_structure.sql +++ /dev/null @@ -1,144 +0,0 @@ -DROP TABLE IF EXISTS artist; -CREATE TABLE artist ( - id TEXT PRIMARY KEY NOT NULL, - mb_id TEXT, - name TEXT -); - -DROP TABLE IF EXISTS artist_release_group; -CREATE TABLE artist_release_group ( - artist_id TEXT NOT NULL, - release_group_id TEXT NOT NULL -); - -DROP TABLE IF EXISTS artist_track; -CREATE TABLE artist_track ( - artist_id TEXT NOT NULL, - track_id TEXT NOT NULL -); - -DROP TABLE IF EXISTS release_group; -CREATE TABLE release_group ( - id TEXT PRIMARY KEY NOT NULL, - albumartist TEXT, - albumsort INT, - musicbrainz_albumtype TEXT, - compilation TEXT, - album_artist_id TEXT -); - -DROP TABLE IF EXISTS release_; -CREATE TABLE release_ ( - id TEXT PRIMARY KEY NOT NULL, - release_group_id TEXT NOT NULL, - title TEXT, - copyright TEXT, - album_status TEXT, - language TEXT, - year TEXT, - date TEXT, - country TEXT, - barcode TEXT -); - -DROP TABLE IF EXISTS track; -CREATE TABLE track ( - id TEXT PRIMARY KEY NOT NULL, - downloaded BOOLEAN NOT NULL DEFAULT 0, - release_id TEXT NOT NULL, - mb_id TEXT, - track TEXT, - length INT, - tracknumber TEXT, - isrc TEXT, - genre TEXT, - lyrics TEXT, - path TEXT, - file TEXT, - url TEXT, - src TEXT -); - -DROP TABLE IF EXISTS lyrics; -CREATE TABLE lyrics ( - track_id TEXT NOT NULL, - text TEXT, - language TEXT -); - -DROP TABLE IF EXISTS target; -CREATE TABLE target ( - track_id TEXT NOT NULL, - file TEXT, - path TEXT -); - -DROP TABLE IF EXISTS source; -CREATE TABLE source ( - track_id TEXT NOT NULL, - src TEXT NOT NULL, - url TEXT NOT NULL, - certainty INT NOT NULL DEFAULT 0, -- certainty=0 -> it is definitly a valid source - valid BOOLEAN NOT NULL DEFAULT 1 -); - -DROP TABLE IF EXISTS easy_id3; -CREATE TABLE easy_id3 ( - track_id TEXT NOT NULL, - album TEXT, - bpm TEXT, - compilation TEXT, - composer TEXT, - copyright TEXT, - encodedby TEXT, - lyricist TEXT, - length TEXT, - media TEXT, - mood TEXT, - grouping TEXT, - title TEXT, - version TEXT, - artist TEXT, - albumartist TEXT, - conductor TEXT, - arranger TEXT, - discnumber TEXT, - organization TEXT, - tracknumber TEXT, - author TEXT, - albumartistsort TEXT, - albumsort TEXT, - composersort TEXT, - artistsort TEXT, - titlesort TEXT, - isrc TEXT, - discsubtitle TEXT, - language TEXT, - genre TEXT, - date TEXT, - originaldate TEXT, - performer TEXT, - musicbrainz_trackid TEXT, - website TEXT, - replaygain_gain TEXT, - replaygain_peak TEXT, - musicbrainz_artistid TEXT, - musicbrainz_albumid TEXT, - musicbrainz_albumartistid TEXT, - musicbrainz_trmid TEXT, - musicip_puid TEXT, - musicip_fingerprint TEXT, - musicbrainz_albumstatus TEXT, - musicbrainz_albumtype TEXT, - releasecountry TEXT, - musicbrainz_discid TEXT, - asin TEXT, - performer TEXT, - barcode TEXT, - catalognumber TEXT, - musicbrainz_releasetrackid TEXT, - musicbrainz_releasegroupid TEXT, - musicbrainz_workid TEXT, - acoustid_fingerprint TEXT, - acoustid_id TEXT -);