music-kraken/music-kraken-core
3
2
Fork 0
Code Issues 10 Pull Requests 1 Packages Projects 2 Releases Wiki Activity

refactor

Browse Source
This commit is contained in:
lars 2022-11-06 18:10:00 +01:00
parent a59244e82d
commit c37fa68937
4 changed files with 446 additions and 480 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/main.py
View File

@ -17,7 +17,8 @@ NOT_A_GENRE = ".", "..", "misc_scripts", "Music", "script", ".git", ".idea"
MUSIC_DIR = os.path.expanduser('~/Music')
TOR = False
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.level = logging.DEBUG
def get_existing_genre():
@ -102,4 +103,4 @@ def cli(start_at: int = 0):
if __name__ == "__main__":
cli(start_at=2)
cli(start_at=0)

166
src/metadata/database.py
View File

@ -4,63 +4,53 @@ import logging
import json
def get_temp_dir():
import tempfile
class Database:
def __init__(self, path_to_db: str, db_structure: str, logger: logging.Logger, reset_anyways: bool = False):
self.logger = logger
self.path_to_db = path_to_db
temp_folder = "music-downloader"
temp_dir = os.path.join(tempfile.gettempdir(), temp_folder)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
return temp_dir
self.connection = sqlite3.connect(self.path_to_db)
self.cursor = self.connection.cursor()
# init database
self.init_db(database_structure=db_structure, reset_anyways=reset_anyways)
# DATABASE_STRUCTURE_FILE = "database_structure.sql"
DATABASE_STRUCTURE_FILE = "src/metadata/database_structure.sql"
TEMP_DIR = get_temp_dir()
DATABASE_FILE = "metadata.db"
db_path = os.path.join(TEMP_DIR, DATABASE_FILE)
connection = sqlite3.connect(db_path)
# connection.row_factory = sqlite3.Row
cursor = connection.cursor()
def init_db(cursor, connection, reset_anyways: bool = False):
def init_db(self, database_structure: str, reset_anyways: bool = False):
# check if db exists
exists = True
try:
query = 'SELECT * FROM track;'
cursor.execute(query)
_ = cursor.fetchall()
self.cursor.execute(query)
_ = self.cursor.fetchall()
except sqlite3.OperationalError:
exists = False
if not exists:
logging.info("Database does not exist yet.")
self.logger.info("Database does not exist yet.")
if reset_anyways or not exists:
# reset the database if reset_anyways is true or if an error has been thrown previously.
logging.info("Creating/Reseting Database.")
self.logger.info("Creating/Reseting Database.")
# read the file
with open(DATABASE_STRUCTURE_FILE, "r") as database_structure_file:
with open(database_structure, "r") as database_structure_file:
query = database_structure_file.read()
cursor.executescript(query)
connection.commit()
self.cursor.executescript(query)
self.connection.commit()
def add_artist(
def add_artist(
self,
musicbrainz_artistid: str,
artist: str = None
):
):
query = "INSERT OR REPLACE INTO artist (id, name) VALUES (?, ?);"
values = musicbrainz_artistid, artist
cursor.execute(query, values)
connection.commit()
self.cursor.execute(query, values)
self.connection.commit()
def add_release_group(
def add_release_group(
self,
musicbrainz_releasegroupid: str,
artist_ids: list,
albumartist: str = None,
@ -68,24 +58,24 @@ def add_release_group(
musicbrainz_albumtype: str = None,
compilation: str = None,
album_artist_id: str = None
):
):
# add adjacency
adjacency_list = []
for artist_id in artist_ids:
adjacency_list.append((artist_id, musicbrainz_releasegroupid))
adjacency_values = tuple(adjacency_list)
adjacency_query = "INSERT OR REPLACE INTO artist_release_group (artist_id, release_group_id) VALUES (?, ?);"
cursor.executemany(adjacency_query, adjacency_values)
connection.commit()
self.cursor.executemany(adjacency_query, adjacency_values)
self.connection.commit()
# add release group
query = "INSERT OR REPLACE INTO release_group (id, albumartist, albumsort, musicbrainz_albumtype, compilation, album_artist_id) VALUES (?, ?, ?, ?, ?, ?);"
values = musicbrainz_releasegroupid, albumartist, albumsort, musicbrainz_albumtype, compilation, album_artist_id
cursor.execute(query, values)
connection.commit()
self.cursor.execute(query, values)
self.connection.commit()
def add_release(
def add_release(
self,
musicbrainz_albumid: str,
release_group_id: str,
title: str = None,
@ -96,38 +86,38 @@ def add_release(
date: str = None,
country: str = None,
barcode: str = None
):
):
query = "INSERT OR REPLACE INTO release_ (id, release_group_id, title, copyright, album_status, language, year, date, country, barcode) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
values = musicbrainz_albumid, release_group_id, title, copyright_, album_status, language, year, date, country, barcode
cursor.execute(query, values)
connection.commit()
self.cursor.execute(query, values)
self.connection.commit()
def add_track(
def add_track(
self,
musicbrainz_releasetrackid: str,
musicbrainz_albumid: str,
feature_aritsts: list,
track: str = None,
isrc: str = None
):
):
# add adjacency
adjacency_list = []
for artist_id in feature_aritsts:
adjacency_list.append((artist_id, musicbrainz_releasetrackid))
adjacency_values = tuple(adjacency_list)
adjacency_query = "INSERT OR REPLACE INTO artist_track (artist_id, track_id) VALUES (?, ?);"
cursor.executemany(adjacency_query, adjacency_values)
connection.commit()
self.cursor.executemany(adjacency_query, adjacency_values)
self.connection.commit()
# add track
query = "INSERT OR REPLACE INTO track (id, release_id, track, isrc) VALUES (?, ?, ?, ?);"
values = musicbrainz_releasetrackid, musicbrainz_albumid, track, isrc
cursor.execute(query, values)
connection.commit()
self.cursor.execute(query, values)
self.connection.commit()
def get_custom_track_querry(custom_where: list) -> str:
@staticmethod
def get_custom_track_query(custom_where: list) -> str:
where_args = [
"track.release_id == release_.id",
"release_group.id == release_.release_group_id",
@ -179,53 +169,44 @@ GROUP BY track.id;
"""
return query
def get_custom_track(self, custom_where: list):
query = Database.get_custom_track_query(custom_where=custom_where)
return [json.loads(i[0]) for i in self.cursor.execute(query)]
def get_custom_track(custom_where: list):
query = get_custom_track_querry(custom_where=custom_where)
return [json.loads(i[0]) for i in cursor.execute(query)]
def get_track_metadata(musicbrainz_releasetrackid: str):
def get_track_metadata(self, musicbrainz_releasetrackid: str):
# this would be vulnerable if musicbrainz_releasetrackid would be user input
resulting_tracks = get_custom_track([f'track.id == "{musicbrainz_releasetrackid}"'])
resulting_tracks = self.get_custom_track([f'track.id == "{musicbrainz_releasetrackid}"'])
if len(resulting_tracks) != 1:
return -1
return resulting_tracks[0]
def get_tracks_to_download(self):
return self.get_custom_track(['track.downloaded == 0'])
def get_tracks_to_download():
return get_custom_track(['track.downloaded == 0'])
def get_tracks_without_src(self):
return self.get_custom_track(["(track.url IS NULL OR track.src IS NULL)"])
def get_tracks_without_isrc(self):
return self.get_custom_track(["track.isrc IS NULL"])
def get_tracks_without_src():
return get_custom_track(["(track.url IS NULL OR track.src IS NULL)"])
def get_tracks_without_filepath(self):
return self.get_custom_track(["(track.file IS NULL OR track.path IS NULL OR track.genre IS NULL)"])
def get_tracks_without_isrc():
return get_custom_track(["track.isrc IS NULL"])
def get_tracks_without_filepath():
return get_custom_track(["(track.file IS NULL OR track.path IS NULL OR track.genre IS NULL)"])
def update_download_status(track_id: str):
def update_download_status(self, track_id: str):
pass
def set_download_data(track_id: str, url: str, src: str):
def set_download_data(self, track_id: str, url: str, src: str):
query = f"""
UPDATE track
SET url = ?,
src = ?
WHERE '{track_id}' == id;
"""
cursor.execute(query, (url, src))
connection.commit()
self.cursor.execute(query, (url, src))
self.connection.commit()
def set_filepath(track_id: str, file: str, path: str, genre: str):
def set_filepath(self, track_id: str, file: str, path: str, genre: str):
query = f"""
UPDATE track
SET file = ?,
@ -233,14 +214,27 @@ SET file = ?,
genre = ?
WHERE '{track_id}' == id;
"""
cursor.execute(query, (file, path, genre))
connection.commit()
self.cursor.execute(query, (file, path, genre))
self.connection.commit()
init_db(cursor=cursor, connection=connection, reset_anyways=False)
if __name__ == "__main__":
logging.basicConfig(level=logging.DEBUG)
import tempfile
for track in get_tracks_without_isrc():
print(track['track'], [artist['name'] for artist in track['artists']])
temp_folder = "music-downloader"
temp_dir = os.path.join(tempfile.gettempdir(), temp_folder)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
temp_dir = get_temp_dir()
DATABASE_FILE = "metadata.db"
DATABASE_STRUCTURE_FILE = "database_structure.sql"
db_path = os.path.join(TEMP_DIR, DATABASE_FILE)
logging.basicConfig()
logger = logging.getLogger("database")
logger.setLevel(logging.DEBUG)
database = Database(os.path.join(temp_dir, "metadata.db"), os.path.join(temp_dir, "database_structure.sql"), logger,
reset_anyways=True)

55
src/metadata/database_structure.sql
View File

@ -1,55 +0,0 @@
DROP TABLE IF EXISTS artist;
CREATE TABLE artist (
id TEXT PRIMARY KEY NOT NULL,
name TEXT
);
DROP TABLE IF EXISTS artist_release_group;
CREATE TABLE artist_release_group (
artist_id TEXT NOT NULL,
release_group_id TEXT NOT NULL
);
DROP TABLE IF EXISTS artist_track;
CREATE TABLE artist_track (
artist_id TEXT NOT NULL,
track_id TEXT NOT NULL
);
DROP TABLE IF EXISTS release_group;
CREATE TABLE release_group (
id TEXT PRIMARY KEY NOT NULL,
albumartist TEXT,
albumsort INT,
musicbrainz_albumtype TEXT,
compilation TEXT,
album_artist_id TEXT
);
DROP TABLE IF EXISTS release_;
CREATE TABLE release_ (
id TEXT PRIMARY KEY NOT NULL,
release_group_id TEXT NOT NULL,
title TEXT,
copyright TEXT,
album_status TEXT,
language TEXT,
year TEXT,
date TEXT,
country TEXT,
barcode TEXT
);
DROP TABLE IF EXISTS track;
CREATE TABLE track (
id TEXT PRIMARY KEY NOT NULL,
downloaded BOOLEAN NOT NULL DEFAULT 0,
release_id TEXT NOT NULL,
track TEXT,
isrc TEXT,
genre TEXT,
path TEXT,
file TEXT,
url TEXT,
src TEXT
);

126
src/metadata/download.py
View File

@ -1,10 +1,8 @@
from typing import List
import musicbrainzngs
import pandas as pd
import logging
from metadata.object_handeling import get_elem_from_obj, parse_music_brainz_date
from metadata import database
from object_handeling import get_elem_from_obj, parse_music_brainz_date
# I don't know if it would be feesable to set up my own mb instance
# https://github.com/metabrainz/musicbrainz-docker
@ -17,13 +15,23 @@ musicbrainzngs.set_useragent("metadata receiver", "0.1", "https://github.com/HeI
# IMPORTANT DOCUMENTATION WHICH CONTAINS FOR EXAMPLE THE INCLUDES
# https://python-musicbrainzngs.readthedocs.io/en/v0.7.1/api/#getting-data
class Artist:
class MetadataDownloader:
def __init__(self, database, logger: logging.Logger):
self.database = database
self.logger = logger
class Artist:
def __init__(
self,
database,
logger,
musicbrainz_artistid: str,
release_groups: List = [],
new_release_groups: bool = True
):
self.database = database
self.logger = logger
"""
release_groups: list
"""
@ -48,7 +56,9 @@ class Artist:
release_groups.sort(key=lambda x: x['first-release-date'])
for i, release_group in enumerate(release_groups):
self.release_groups.append(ReleaseGroup(
self.release_groups.append(MetadataDownloader.ReleaseGroup(
self.database,
self.logger,
musicbrainz_releasegroupid=release_group['id'],
artists=[self],
albumsort=i + 1
@ -59,21 +69,24 @@ class Artist:
return f"id: {self.musicbrainz_artistid}\nname: {self.artist}\n{newline.join([str(release_group) for release_group in self.release_groups])}"
def save(self):
logging.info(f"artist: {self}")
database.add_artist(
self.logger.info(f"artist: {self}")
self.database.add_artist(
musicbrainz_artistid=self.musicbrainz_artistid,
artist=self.artist
)
class ReleaseGroup:
class ReleaseGroup:
def __init__(
self,
database,
logger,
musicbrainz_releasegroupid: str,
artists: List[Artist] = [],
artists = [],
albumsort: int = None,
only_download_distinct_releases: bool = True
):
self.database = database
self.logger = logger
"""
split_artists: list -> if len > 1: album_artist=VariousArtists
releases: list
@ -95,7 +108,8 @@ class ReleaseGroup:
continue
self.append_artist(artist_id)
self.albumartist = "Various Artists" if len(self.artists) > 1 else self.artists[0].artist
self.album_artist_id = None if self.albumartist == "Various Artists" else self.artists[0].musicbrainz_artistid
self.album_artist_id = None if self.albumartist == "Various Artists" else self.artists[
0].musicbrainz_artistid
self.albumsort = albumsort
self.musicbrainz_albumtype = get_elem_from_obj(release_group_data, ['primary-type'])
@ -113,8 +127,8 @@ class ReleaseGroup:
return f"{newline.join([str(release_group) for release_group in self.releases])}"
def save(self):
logging.info(f"caching release_group {self}")
database.add_release_group(
self.logger.info(f"caching release_group {self}")
self.database.add_release_group(
musicbrainz_releasegroupid=self.musicbrainz_releasegroupid,
artist_ids=[artist.musicbrainz_artistid for artist in self.artists],
albumartist=self.albumartist,
@ -124,7 +138,7 @@ class ReleaseGroup:
album_artist_id=self.album_artist_id
)
def append_artist(self, artist_id: str) -> Artist:
def append_artist(self, artist_id: str):
for existing_artist in self.artists:
if artist_id == existing_artist.musicbrainz_artistid:
return existing_artist
@ -136,7 +150,7 @@ class ReleaseGroup:
musicbrainz_albumid = get_elem_from_obj(release_data, ['id'])
if musicbrainz_albumid is None:
return
self.releases.append(Release(musicbrainz_albumid, release_group=self))
self.releases.append(MetadataDownloader.Release(self.database, self.logger, musicbrainz_albumid, release_group=self))
def append_distinct_releases(self, release_datas: List[dict]):
titles = {}
@ -154,13 +168,16 @@ class ReleaseGroup:
for release_data in release_datas:
self.append_release(release_data)
class Release:
class Release:
def __init__(
self,
database,
logger,
musicbrainz_albumid: str,
release_group: ReleaseGroup = None
release_group = None
):
self.database = database
self.logger = logger
"""
release_group: ReleaseGroup
tracks: list
@ -191,8 +208,8 @@ class Release:
return f"{self.title} ©{self.copyright} {self.album_status}"
def save(self):
logging.info(f"caching release {self}")
database.add_release(
self.logger.info(f"caching release {self}")
self.database.add_release(
musicbrainz_albumid=self.musicbrainz_albumid,
release_group_id=self.release_group.musicbrainz_releasegroupid,
title=self.title,
@ -211,15 +228,18 @@ class Release:
if musicbrainz_releasetrackid is None:
continue
self.tracklist.append(Track(musicbrainz_releasetrackid, self))
self.tracklist.append(MetadataDownloader.Track(self.database, self.logger, musicbrainz_releasetrackid, self))
class Track:
class Track:
def __init__(
self,
database,
logger,
musicbrainz_releasetrackid: str,
release: Release = None
release = None
):
self.database = database
self.logger = logger
"""
release: Release
feature_artists: list
@ -229,7 +249,9 @@ class Track:
self.release = release
self.artists = []
result = musicbrainzngs.get_recording_by_id(self.musicbrainz_releasetrackid, includes=["artists", "releases", "recording-rels", "isrcs", "work-level-rels"])
result = musicbrainzngs.get_recording_by_id(self.musicbrainz_releasetrackid,
includes=["artists", "releases", "recording-rels", "isrcs",
"work-level-rels"])
recording_data = result['recording']
for artist_data in get_elem_from_obj(recording_data, ['artist-credit'], return_if_none=[]):
self.append_artist(get_elem_from_obj(artist_data, ['artist', 'id']))
@ -243,9 +265,9 @@ class Track:
return f"{self.title}: {self.isrc}"
def save(self):
logging.info(f"caching track {self}")
self.logger.info(f"caching track {self}")
database.add_track(
self.database.add_track(
musicbrainz_releasetrackid=self.musicbrainz_releasetrackid,
musicbrainz_albumid=self.release.musicbrainz_albumid,
feature_aritsts=[artist.musicbrainz_artistid for artist in self.artists],
@ -253,51 +275,55 @@ class Track:
isrc=self.isrc
)
def append_artist(self, artist_id: str) -> Artist:
def append_artist(self, artist_id: str):
if artist_id is None:
return
for existing_artist in self.artists:
if artist_id == existing_artist.musicbrainz_artistid:
return existing_artist
new_artist = Artist(artist_id, new_release_groups=False)
new_artist = MetadataDownloader.Artist(self.database, self.logger, artist_id, new_release_groups=False)
self.artists.append(new_artist)
return new_artist
def download(option: dict):
def download(self, option: dict):
type_ = option['type']
mb_id = option['id']
metadata_list = []
if type_ == "artist":
artist = Artist(mb_id)
print(artist)
self.Artist(self.database, self.logger, mb_id)
elif type_ == "release_group":
self.ReleaseGroup(self.database, self.logger, mb_id)
elif type_ == "release":
metadata_list = download_release(mb_id)
self.Release(self.database, self.logger, mb_id)
elif type_ == "track":
metadata_list = download_track(mb_id)
print(metadata_list)
metadata_df = pd.DataFrame(metadata_list)
# metadata_df.to_csv(os.path.join(self.temp, file))
return metadata_df
self.Track(self.database, self.logger, mb_id)
if __name__ == "__main__":
"""
import tempfile
import os
TEMP_FOLDER = "music-downloader"
TEMP_DIR = os.path.join(tempfile.gettempdir(), TEMP_FOLDER)
if not os.path.exists(TEMP_DIR):
os.mkdir(TEMP_DIR)
"""
temp_folder = "music-downloader"
temp_dir = os.path.join(tempfile.gettempdir(), temp_folder)
if not os.path.exists(temp_dir):
os.mkdir(temp_dir)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logging.basicConfig(level=logging.DEBUG)
db_logger = logging.getLogger("database")
db_logger.setLevel(logging.DEBUG)
download({'id': '5cfecbe4-f600-45e5-9038-ce820eedf3d1', 'type': 'artist'})
import database
database_ = database.Database(os.path.join(temp_dir, "metadata.db"),
os.path.join(temp_dir, "database_structure.sql"), db_logger,
reset_anyways=True)
download_logger = logging.getLogger("metadata downloader")
download_logger.setLevel(logging.INFO)
downloader = MetadataDownloader(database_, download_logger)
downloader.download({'id': '5cfecbe4-f600-45e5-9038-ce820eedf3d1', 'type': 'artist'})
# download({'id': '4b9af532-ef7e-42ab-8b26-c466327cb5e0', 'type': 'release'})
# download({'id': 'c24ed9e7-6df9-44de-8570-975f1a5a75d1', 'type': 'track'})