much progress on new db integration

This commit is contained in:
Hellow 2022-12-06 23:44:42 +01:00
parent cbb56499bf
commit 4b60ed7555
10 changed files with 284 additions and 85 deletions

View File

@ -8,5 +8,17 @@
<jdbc-url>jdbc:sqlite:/tmp/music-downloader/metadata.db</jdbc-url> <jdbc-url>jdbc:sqlite:/tmp/music-downloader/metadata.db</jdbc-url>
<working-dir>$ProjectFileDir$</working-dir> <working-dir>$ProjectFileDir$</working-dir>
</data-source> </data-source>
<data-source source="LOCAL" name="test" uuid="eb0321e0-ad51-46e5-8c59-97956edf1699">
<driver-ref>sqlite.xerial</driver-ref>
<synchronize>true</synchronize>
<jdbc-driver>org.sqlite.JDBC</jdbc-driver>
<jdbc-url>jdbc:sqlite:$PROJECT_DIR$/src/test.db</jdbc-url>
<working-dir>$ProjectFileDir$</working-dir>
<libraries>
<library>
<url>file://$APPLICATION_CONFIG_DIR$/jdbc-drivers/Xerial SQLiteJDBC/3.39.2/sqlite-jdbc-3.39.2.jar</url>
</library>
</libraries>
</data-source>
</component> </component>
</project> </project>

6
.idea/sqldialects.xml Normal file
View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="SqlDialectMappings">
<file url="file://$PROJECT_DIR$/src/music_kraken/static_files/new_db.sql" dialect="SQLite" />
</component>
</project>

View File

@ -43,8 +43,8 @@ with open('version', 'r') as version_file:
setup( setup(
name='music-kraken', name='music-kraken',
version=version, version=version,
description='An extensive music downloader crawling the internet. It gets its metadata from a couple metadata ' description='An extensive music downloader crawling the internet. It gets its metadata from a couple of metadata '
'provider, and it scrapes the audiofiles.', 'providers, and it scrapes the audiofiles.',
long_description=long_description, long_description=long_description,
long_description_content_type='text/markdown', long_description_content_type='text/markdown',
author='Hellow2', author='Hellow2',

Binary file not shown.

View File

@ -8,51 +8,33 @@ from music_kraken import (
import music_kraken.database.new_database as db import music_kraken.database.new_database as db
cache = music_kraken.database.new_database.Database("test.db") cache = music_kraken.database.new_database.Database("test.db")
cache.reset() cache.reset()
song = Song( song = Song(
title="Vein Deep in the Solution", title="Vein Deep in the Solution",
release_name="One Final Action", release_name="One Final Action",
length=666,
target=Target(file="~/Music/genre/artist/album/song.mp3", path="~/Music/genre/artist/album"), target=Target(file="~/Music/genre/artist/album/song.mp3", path="~/Music/genre/artist/album"),
metadata={ metadata={
"album": "One Final Action" "album": "One Final Action"
}, },
lyrics=[ lyrics=[
Lyrics(text="these are some depressive lyrics", language="en") Lyrics(text="these are some depressive lyrics", language="en"),
Lyrics(text="test", language="en")
], ],
sources=[ sources=[
Source(src="youtube", url="https://youtu.be/dfnsdajlhkjhsd") Source(src="youtube", url="https://youtu.be/dfnsdajlhkjhsd"),
Source(src="musify", url="https://ln.topdf.de/Music-Kraken/")
] ]
) )
cache.push([song]) song_ref = song.reference
print(song_ref)
""" lyrics = Lyrics(text="these are some Lyrics that don't belong to any Song", language="en")
music_kraken.clear_cache()
artist = music_kraken.Artist( cache.push([song, lyrics])
name="I'm in a Coffin"
)
song = Song( cache.pull_single_song(song_ref=song_ref)
title="Vein Deep in the Solution",
release_name="One Final Action",
target=Target(file="~/Music/genre/artist/album/song.mp3", path="~/Music/genre/artist/album"),
metadata={
"album": "One Final Action"
},
lyrics=[
Lyrics(text="these are some depressive lyrics", language="en")
],
sources=[
Source(src="youtube", url="https://youtu.be/dfnsdajlhkjhsd")
]
)
print(song)
print(song.id)
# music_kraken.fetch_sources([song])
"""

View File

@ -1,9 +1,10 @@
import sqlite3 import sqlite3
import os import os
import logging import logging
from typing import List from typing import List, Tuple
from pkg_resources import resource_string from pkg_resources import resource_string
from .objects.database_object import Reference
from .objects import ( from .objects import (
Song, Song,
Lyrics, Lyrics,
@ -15,12 +16,34 @@ from .objects import (
logger = logging.getLogger("database") logger = logging.getLogger("database")
# Due to this not being deployed on a Server **HOPEFULLY**
# I don't need to parameterize stuff like the where and
# use complicated query builder
SONG_QUERY = """
SELECT
Song.id AS song_id, Song.name AS title, Song.isrc AS isrc, Song.length AS length,
Target.id AS target_id, Target.file AS file, Target.path AS path
FROM Song
LEFT JOIN Target ON Song.id=Target.song_id
WHERE Song.id="{song_id}";
"""
SOURCE_QUERY = """
SELECT id, src, url, song_id
FROM Source
WHERE {where};
"""
LYRICS_QUERY = """
SELECT id, text, language, song_id
FROM Lyrics
WHERE {where};
"""
class Database: class Database:
def __init__(self, database_file: str): def __init__(self, database_file: str):
self.database_file: str = database_file self.database_file: str = database_file
self.connection, self.cursor = self.reset_cursor()
self.connection = sqlite3.connect(self.database_file)
self.cursor = self.connection.cursor() self.cursor = self.connection.cursor()
def reset(self): def reset(self):
@ -36,14 +59,21 @@ class Database:
os.remove(self.database_file) os.remove(self.database_file)
# newly creating the database # newly creating the database
self.connection = sqlite3.connect(self.database_file) self.reset_cursor()
self.cursor = self.connection.cursor()
query = resource_string("music_kraken", "static_files/new_db.sql").decode('utf-8') query = resource_string("music_kraken", "static_files/new_db.sql").decode('utf-8')
# fill the database with the schematic # fill the database with the schematic
self.cursor.executescript(query) self.cursor.executescript(query)
self.connection.commit() self.connection.commit()
def reset_cursor(self) -> Tuple[sqlite3.Connection, sqlite3.Cursor]:
self.connection = sqlite3.connect(self.database_file)
# This is necessary that fetching rows returns dicts instead of tuple
self.connection.row_factory = sqlite3.Row
self.cursor = self.connection.cursor()
return self.connection, self.cursor
def push_one(self, db_object: Song | Lyrics | Target | Artist | Source): def push_one(self, db_object: Song | Lyrics | Target | Artist | Source):
if type(db_object) == Song: if type(db_object) == Song:
return self.push_song(song=db_object) return self.push_song(song=db_object)
@ -82,27 +112,143 @@ class Database:
name - title name - title
""" """
table = "Song" table = "Song"
query = f"INSERT OR REPLACE INTO {table} (id, name) VALUES (?, ?);" query = f"INSERT OR REPLACE INTO {table} (id, name, isrc, length) VALUES (?, ?, ?, ?);"
values = ( values = (
song.id, song.id,
song.title song.title,
song.isrc,
song.length
) )
self.cursor.execute(query, values) self.cursor.execute(query, values)
self.connection.commit() self.connection.commit()
def push_lyrics(self, lyrics: Lyrics): # add sources
pass for source in song.sources:
self.push_source(source=source)
# add lyrics
for single_lyrics in song.lyrics:
self.push_lyrics(lyrics=single_lyrics)
# add target
self.push_target(target=song.target)
def push_lyrics(self, lyrics: Lyrics, ):
if lyrics.song_ref_id is None:
logger.warning("the Lyrics don't refer to a song")
table = "Lyrics"
query = f"INSERT OR REPLACE INTO {table} (id, song_id, text, language) VALUES (?, ?, ?, ?);"
values = (
lyrics.id,
lyrics.song_ref_id,
lyrics.text,
lyrics.language
)
self.cursor.execute(query, values)
self.connection.commit()
def push_source(self, source: Source):
if source.song_ref_id is None:
logger.warning("the Source don't refer to a song")
table = "Source"
query = f"INSERT OR REPLACE INTO {table} (id, song_id, src, url) VALUES (?, ?, ?, ?);"
values = (
source.id,
source.song_ref_id,
source.src,
source.url
)
self.cursor.execute(query, values)
self.connection.commit()
def push_target(self, target: Target): def push_target(self, target: Target):
pass if target.song_ref_id is None:
logger.warning("the Target doesn't refer to a song")
table = "Target"
query = f"INSERT OR REPLACE INTO {table} (id, song_id, file, path) VALUES (?, ?, ?, ?);"
values = (
target.id,
target.song_ref_id,
target.file,
target.path
)
self.cursor.execute(query, values)
self.connection.commit()
def push_artist(self, artist: Artist): def push_artist(self, artist: Artist):
pass pass
def push_source(self, source: Source): def pull_lyrics(self, song_ref: Reference = None, lyrics_ref: Reference = None) -> List[Lyrics]:
pass pass
def pull_sources(self, song_ref: Reference = None, source_ref: Reference = None) -> List[Source]:
"""
Gets a list of sources. if source_ref is passed in the List will most likely only
contain one Element if everything goes accordingly.
**If neither song_ref nor source_ref are passed in it will return ALL sources**
:param song_ref:
:param source_ref:
:return:
"""
where = "1=1"
if song_ref is not None:
where = f"song_id=\"{song_ref.id}\""
elif source_ref is not None:
where = f"id=\"{source_ref.id}\""
query = SOURCE_QUERY.format(where=where)
self.cursor.execute(query)
source_rows = self.cursor.fetchall()
return [Source(
id_=source_row['id'],
src=source_row['src'],
url=source_row['url']
) for source_row in source_rows]
def pull_single_song(self, song_ref: Reference = None) -> Song:
"""
This function is used to get one song (including its children like Sources etc)
from one song id (a reference object)
:param song_ref:
:return requested_song:
"""
if song_ref.id is None:
raise ValueError("The Song ref doesn't point anywhere. Remember to use the debugger.")
query = SONG_QUERY.format(song_id=song_ref.id)
self.cursor.execute(query)
song_rows = self.cursor.fetchall()
if len(song_rows) == 0:
logger.warning(f"No song found for the id {song_ref.id}")
return Song()
if len(song_rows) > 1:
logger.warning(f"Multiple Songs found for the id {song_ref.id}. Defaulting to the first one.")
song_result = song_rows[0]
song = Song(
id_=song_result['song_id'],
title=song_result['title'],
isrc=song_result['isrc'],
length=song_result['length'],
target=Target(
id_=song_result['target_id'],
file=song_result['file'],
path=song_result['path']
),
sources=self.pull_sources(song_ref=song_ref)
)
return song
if __name__ == "__main__": if __name__ == "__main__":
cache = Database("") cache = Database("")

View File

@ -4,10 +4,14 @@ from ...utils.shared import (
SONG_LOGGER as logger SONG_LOGGER as logger
) )
class Reference: class Reference:
def __init__(self, id_: str) -> None: def __init__(self, id_: str) -> None:
self.id = id_ self.id = id_
def __str__(self):
return f"references to an object with the id: {self.id}"
class DatabaseObject: class DatabaseObject:
def __init__(self, id_: str = None) -> None: def __init__(self, id_: str = None) -> None:

View File

@ -12,13 +12,33 @@ from .database_object import (
) )
class SongAttribute:
def __init__(self, song_ref: Reference = None):
# the reference to the song the lyrics belong to
self.song_ref = song_ref
def add_song(self, song_ref: Reference):
self.song_ref = song_ref
def get_ref_song_id(self):
if self.song_ref is None:
return None
return self.song_ref.id
def set_ref_song_id(self, song_id):
self.song_ref = Reference(song_id)
song_ref_id = property(fget=get_ref_song_id, fset=set_ref_song_id)
class Metadata: class Metadata:
""" """
Shall only be read or edited via the Song object. Shall only be read or edited via the Song object.
For this reason there is no reference to the song needed. For this reason there is no reference to the song needed.
""" """
def __init__(self, data: dict = {}) -> None: def __init__(self, data: dict = {}) -> None:
self.data = {} self.data = data
def get_all_metadata(self): def get_all_metadata(self):
return list(self.data.items()) return list(self.data.items())
@ -33,7 +53,7 @@ class Metadata:
return self.data[item] return self.data[item]
class Source(DatabaseObject): class Source(DatabaseObject, SongAttribute):
""" """
create somehow like that create somehow like that
```python ```python
@ -41,23 +61,30 @@ class Source(DatabaseObject):
Source(src="youtube", url="https://youtu.be/dfnsdajlhkjhsd") Source(src="youtube", url="https://youtu.be/dfnsdajlhkjhsd")
``` ```
""" """
def __init__(self, id_: str = None, src: str = None, url: str = None) -> None: def __init__(self, id_: str = None, src: str = None, url: str = None) -> None:
super().__init__(id_=id_) DatabaseObject.__init__(self, id_=id_)
SongAttribute.__init__(self)
self.src = src self.src = src
self.url = url self.url = url
def __str__(self):
return f"{self.src}: {self.url}"
class Target(DatabaseObject):
class Target(DatabaseObject, SongAttribute):
""" """
create somehow like that create somehow like that
```python ```python
# I know path is pointles, and I will change that (don't worry about backwards compatibility there) # I know path is pointless, and I will change that (don't worry about backwards compatibility there)
Target(file="~/Music/genre/artist/album/song.mp3", path="~/Music/genre/artist/album") Target(file="~/Music/genre/artist/album/song.mp3", path="~/Music/genre/artist/album")
``` ```
""" """
def __init__(self, id_:str = None, file: str = None, path: str = None) -> None:
super().__init__(id_=id_) def __init__(self, id_: str = None, file: str = None, path: str = None) -> None:
DatabaseObject.__init__(self, id_=id_)
SongAttribute.__init__(self)
self._file = file self._file = file
self._path = path self._path = path
@ -96,9 +123,10 @@ class Target(DatabaseObject):
exists_on_disc = property(fget=get_exists_on_disc) exists_on_disc = property(fget=get_exists_on_disc)
class Lyrics(DatabaseObject): class Lyrics(DatabaseObject, SongAttribute):
def __init__(self, text: str, language: str, id_: str = None) -> None: def __init__(self, text: str, language: str, id_: str = None) -> None:
super().__init__(id_=id_) DatabaseObject.__init__(self, id_=id_)
SongAttribute.__init__(self)
self.text = text self.text = text
self.language = language self.language = language
@ -133,29 +161,33 @@ class Song(DatabaseObject):
self.title: str | None = title self.title: str | None = title
self.release_name: str | None = release_name self.release_name: str | None = release_name
self.isrc: str | None = isrc self.isrc: str | None = isrc
self.length: int | None = length self.length_: int | None = length
self.artist_names = artist_names self.artist_names = artist_names
self.metadata = Metadata(data=metadata) self.metadata = Metadata(data=metadata)
if sources is None: if sources is None:
sources = [] sources = []
self.sources: List[Source] = sources self.sources: List[Source] = sources
for source in self.sources:
source.add_song(self.reference)
if target is None: if target is None:
target = Target() target = Target()
self.target: Target = target self.target: Target = target
self.target.add_song(self.reference)
if lyrics is None: if lyrics is None:
lyrics = [] lyrics = []
self.lyrics: List[Lyrics] = lyrics self.lyrics: List[Lyrics] = lyrics
for lyrics_ in self.lyrics:
lyrics_.add_song(self.reference)
self.release_ref = release_ref self.release_ref = release_ref
self.artist_refs = artist_refs self.artist_refs = artist_refs
def __str__(self) -> str: def __str__(self) -> str:
return f"\"{self.title}\" by {', '.join([str(a) for a in self.artists])}" return f"\"{self.title}\" by {', '.join(self.artist_names)}"
def __repr__(self) -> str: def __repr__(self) -> str:
return self.__str__() return self.__str__()
@ -167,7 +199,20 @@ class Song(DatabaseObject):
return self.isrc is not None return self.isrc is not None
def get_artist_names(self) -> List[str]: def get_artist_names(self) -> List[str]:
return [a.name for a in self.artists] return self.artist_names
def get_length(self):
if self.length_ is None:
return None
return int(self.length_)
def set_length(self, length: int):
if type(length) != int:
raise TypeError(f"length of a song must be of the type int not {type(length)}")
self.length_ = length
length = property(fget=get_length, fset=set_length)
if __name__ == "__main__": if __name__ == "__main__":
""" """

View File

@ -1,13 +1,19 @@
CREATE TABLE Song CREATE TABLE Song
( (
id BIGINT AUTO_INCREMENT PRIMARY KEY, id BIGINT AUTO_INCREMENT PRIMARY KEY,
name TEXT name TEXT,
isrc TEXT,
length INT -- length is in milliseconds (could be wrong)
); );
CREATE TABLE Source CREATE TABLE Source
( (
id BIGINT AUTO_INCREMENT PRIMARY KEY, id BIGINT AUTO_INCREMENT PRIMARY KEY,
src TEXT NOT NULL,
url TEXT NOT NULL,
certainty INT NOT NULL DEFAULT 0, -- certainty=0 -> it is definitely a valid source
valid BOOLEAN NOT NULL DEFAULT 1,
song_id BIGINT, song_id BIGINT,
FOREIGN KEY(song_id) REFERENCES Song(id) FOREIGN KEY(song_id) REFERENCES Song(id)
); );
@ -29,13 +35,17 @@ CREATE TABLE Album
CREATE TABLE Target CREATE TABLE Target
( (
id BIGINT AUTO_INCREMENT PRIMARY KEY, id BIGINT AUTO_INCREMENT PRIMARY KEY,
song_id BIGINT, file TEXT NOT NULL,
path TEXT,
song_id BIGINT UNIQUE,
FOREIGN KEY(song_id) REFERENCES Song(id) FOREIGN KEY(song_id) REFERENCES Song(id)
); );
CREATE TABLE Lyrics CREATE TABLE Lyrics
( (
id BIGINT AUTO_INCREMENT PRIMARY KEY, id BIGINT AUTO_INCREMENT PRIMARY KEY,
text TEXT,
language TEXT,
song_id BIGINT, song_id BIGINT,
FOREIGN KEY(song_id) REFERENCES Song(id) FOREIGN KEY(song_id) REFERENCES Song(id)
); );
@ -55,9 +65,3 @@ CREATE TABLE AlbumArtist
FOREIGN KEY(album_id) REFERENCES Album(id), FOREIGN KEY(album_id) REFERENCES Album(id),
FOREIGN KEY(artist_id) REFERENCES Artist(id) FOREIGN KEY(artist_id) REFERENCES Artist(id)
); );
SELECT
Song.id,
Song.name
FROM Song

BIN
src/test.db Normal file

Binary file not shown.