feat: added development scripts
This commit is contained in:
parent
81e57b645c
commit
88d618a3f8
@ -1,197 +0,0 @@
|
||||
from typing import List, Union, Type, Optional
|
||||
from peewee import (
|
||||
SqliteDatabase,
|
||||
PostgresqlDatabase,
|
||||
MySQLDatabase,
|
||||
Model,
|
||||
CharField,
|
||||
IntegerField,
|
||||
BooleanField,
|
||||
ForeignKeyField,
|
||||
TextField
|
||||
)
|
||||
|
||||
"""
|
||||
**IMPORTANT**:
|
||||
|
||||
never delete, modify the datatype or add constrains to ANY existing collumns,
|
||||
between the versions, that gets pushed out to the users.
|
||||
Else my function can't update legacy databases, to new databases,
|
||||
while keeping the data of the old ones.
|
||||
|
||||
EVEN if that means to for example keep decimal values stored in strings.
|
||||
(not in my codebase though.)
|
||||
"""
|
||||
|
||||
|
||||
class BaseModel(Model):
|
||||
notes: str = CharField(null=True)
|
||||
|
||||
class Meta:
|
||||
database = None
|
||||
|
||||
@classmethod
|
||||
def Use(cls, database: Union[SqliteDatabase, PostgresqlDatabase, MySQLDatabase]) -> Model:
|
||||
cls._meta.database = database
|
||||
return cls
|
||||
|
||||
def use(self, database: Union[SqliteDatabase, PostgresqlDatabase, MySQLDatabase]) -> Model:
|
||||
self._meta.database = database
|
||||
return self
|
||||
|
||||
class ObjectModel(BaseModel):
|
||||
id: str = CharField(primary_key=True)
|
||||
|
||||
class MainModel(BaseModel):
|
||||
additional_arguments: str = CharField(null=True)
|
||||
notes: str = CharField(null=True)
|
||||
|
||||
|
||||
class Song(MainModel):
|
||||
"""A class representing a song in the music database."""
|
||||
|
||||
title: str = CharField(null=True)
|
||||
isrc: str = CharField(null=True)
|
||||
length: int = IntegerField(null=True)
|
||||
tracksort: int = IntegerField(null=True)
|
||||
genre: str = CharField(null=True)
|
||||
|
||||
|
||||
class Album(MainModel):
|
||||
"""A class representing an album in the music database."""
|
||||
|
||||
title: str = CharField(null=True)
|
||||
album_status: str = CharField(null=True)
|
||||
album_type: str = CharField(null=True)
|
||||
language: str = CharField(null=True)
|
||||
date_string: str = CharField(null=True)
|
||||
date_format: str = CharField(null=True)
|
||||
barcode: str = CharField(null=True)
|
||||
albumsort: int = IntegerField(null=True)
|
||||
|
||||
|
||||
class Artist(MainModel):
|
||||
"""A class representing an artist in the music database."""
|
||||
|
||||
name: str = CharField(null=True)
|
||||
country: str = CharField(null=True)
|
||||
formed_in_date: str = CharField(null=True)
|
||||
formed_in_format: str = CharField(null=True)
|
||||
general_genre: str = CharField(null=True)
|
||||
|
||||
|
||||
class Label(MainModel):
|
||||
name: str = CharField(null=True)
|
||||
|
||||
|
||||
class Target(ObjectModel):
|
||||
"""A class representing a target of a song in the music database."""
|
||||
|
||||
file: str = CharField()
|
||||
path: str = CharField()
|
||||
song = ForeignKeyField(Song, backref='targets')
|
||||
|
||||
|
||||
class Lyrics(ObjectModel):
|
||||
"""A class representing lyrics of a song in the music database."""
|
||||
|
||||
text: str = TextField()
|
||||
language: str = CharField()
|
||||
song = ForeignKeyField(Song, backref='lyrics')
|
||||
|
||||
|
||||
class Source(BaseModel):
|
||||
"""A class representing a source of a song in the music database."""
|
||||
ContentTypes = Union[Song, Album, Artist, Lyrics]
|
||||
|
||||
page: str = CharField()
|
||||
url: str = CharField()
|
||||
|
||||
content_type: str = CharField()
|
||||
content_id: int = CharField()
|
||||
# content: ForeignKeyField = ForeignKeyField('self', backref='content_items', null=True)
|
||||
|
||||
@property
|
||||
def content_object(self) -> Union[Song, Album, Artist]:
|
||||
"""Get the content associated with the source as an object."""
|
||||
if self.content_type == 'Song':
|
||||
return Song.get(Song.id == self.content_id)
|
||||
if self.content_type == 'Album':
|
||||
return Album.get(Album.id == self.content_id)
|
||||
if self.content_type == 'Artist':
|
||||
return Artist.get(Artist.id == self.content_id)
|
||||
if self.content_type == 'Label':
|
||||
return Label.get(Label.id == self.content_id)
|
||||
if self.content_type == 'Lyrics':
|
||||
return Lyrics.get(Lyrics.id == self.content_id)
|
||||
|
||||
|
||||
@content_object.setter
|
||||
def content_object(self, value: Union[Song, Album, Artist]) -> None:
|
||||
"""Set the content associated with the source as an object."""
|
||||
self.content_type = value.__class__.__name__
|
||||
self.content_id = value.id
|
||||
|
||||
|
||||
class SongArtist(BaseModel):
|
||||
"""A class representing the relationship between a song and an artist."""
|
||||
|
||||
song: ForeignKeyField = ForeignKeyField(Song, backref='song_artists')
|
||||
artist: ForeignKeyField = ForeignKeyField(Artist, backref='song_artists')
|
||||
is_feature: bool = BooleanField(default=False)
|
||||
|
||||
|
||||
class ArtistAlbum(BaseModel):
|
||||
"""A class representing the relationship between an album and an artist."""
|
||||
|
||||
album: ForeignKeyField = ForeignKeyField(Album, backref='album_artists')
|
||||
artist: ForeignKeyField = ForeignKeyField(Artist, backref='album_artists')
|
||||
|
||||
|
||||
class AlbumSong(BaseModel):
|
||||
"""A class representing the relationship between an album and an song."""
|
||||
album: ForeignKeyField = ForeignKeyField(Album, backref='album_artists')
|
||||
song: ForeignKeyField = ForeignKeyField(Song, backref='album_artists')
|
||||
|
||||
|
||||
class LabelAlbum(BaseModel):
|
||||
label: ForeignKeyField = ForeignKeyField(Label, backref='label_album')
|
||||
album: ForeignKeyField = ForeignKeyField(Album, backref='label_album')
|
||||
|
||||
|
||||
class LabelArtist(BaseModel):
|
||||
label: ForeignKeyField = ForeignKeyField(Label, backref='label_artist')
|
||||
artist: ForeignKeyField = ForeignKeyField(Artist, backref='label_artists')
|
||||
|
||||
|
||||
ALL_MODELS = [
|
||||
Song,
|
||||
Album,
|
||||
Artist,
|
||||
Source,
|
||||
Lyrics,
|
||||
ArtistAlbum,
|
||||
Target,
|
||||
SongArtist
|
||||
]
|
||||
|
||||
if __name__ == "__main__":
|
||||
database_1 = SqliteDatabase(":memory:")
|
||||
database_1.create_tables([Song.Use(database_1)])
|
||||
database_2 = SqliteDatabase(":memory:")
|
||||
database_2.create_tables([Song.Use(database_2)])
|
||||
|
||||
# creating songs, adding it to db_2 if i is even, else to db_1
|
||||
for i in range(100):
|
||||
song = Song(name=str(i) + "hs")
|
||||
|
||||
db_to_use = database_2 if i % 2 == 0 else database_1
|
||||
song.use(db_to_use).save()
|
||||
|
||||
print("database 1")
|
||||
for song in Song.Use(database_1).select():
|
||||
print(song.name)
|
||||
|
||||
print("database 2")
|
||||
for song in Song.Use(database_1).select():
|
||||
print(song.name)
|
||||
@ -1,188 +0,0 @@
|
||||
# Standard library
|
||||
from typing import Optional, Union, List
|
||||
from enum import Enum
|
||||
from playhouse.migrate import *
|
||||
|
||||
# third party modules
|
||||
from peewee import (
|
||||
SqliteDatabase,
|
||||
MySQLDatabase,
|
||||
PostgresqlDatabase,
|
||||
)
|
||||
|
||||
# own modules
|
||||
from . import (
|
||||
data_models,
|
||||
write
|
||||
)
|
||||
from .. import objects
|
||||
|
||||
|
||||
class DatabaseType(Enum):
|
||||
SQLITE = "sqlite"
|
||||
POSTGRESQL = "postgresql"
|
||||
MYSQL = "mysql"
|
||||
|
||||
class Database:
|
||||
database: Union[SqliteDatabase, PostgresqlDatabase, MySQLDatabase]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db_type: DatabaseType,
|
||||
db_name: str,
|
||||
db_user: Optional[str] = None,
|
||||
db_password: Optional[str] = None,
|
||||
db_host: Optional[str] = None,
|
||||
db_port: Optional[int] = None
|
||||
):
|
||||
self.db_type = db_type
|
||||
self.db_name = db_name
|
||||
self.db_user = db_user
|
||||
self.db_password = db_password
|
||||
self.db_host = db_host
|
||||
self.db_port = db_port
|
||||
|
||||
self.initialize_database()
|
||||
|
||||
def create_database(self) -> Union[SqliteDatabase, PostgresqlDatabase, MySQLDatabase]:
|
||||
"""Create a database instance based on the configured database type and parameters.
|
||||
|
||||
Returns:
|
||||
The created database instance, or None if an invalid database type was specified.
|
||||
"""
|
||||
|
||||
# SQLITE
|
||||
if self.db_type == DatabaseType.SQLITE:
|
||||
return SqliteDatabase(self.db_name)
|
||||
|
||||
# POSTGRES
|
||||
if self.db_type == DatabaseType.POSTGRESQL:
|
||||
return PostgresqlDatabase(
|
||||
self.db_name,
|
||||
user=self.db_user,
|
||||
password=self.db_password,
|
||||
host=self.db_host,
|
||||
port=self.db_port,
|
||||
)
|
||||
|
||||
# MYSQL
|
||||
if self.db_type == DatabaseType.MYSQL:
|
||||
return MySQLDatabase(
|
||||
self.db_name,
|
||||
user=self.db_user,
|
||||
password=self.db_password,
|
||||
host=self.db_host,
|
||||
port=self.db_port,
|
||||
)
|
||||
|
||||
raise ValueError("Invalid database type specified.")
|
||||
|
||||
|
||||
@property
|
||||
def migrator(self) -> SchemaMigrator:
|
||||
if self.db_type == DatabaseType.SQLITE:
|
||||
return SqliteMigrator(self.database)
|
||||
|
||||
if self.db_type == DatabaseType.MYSQL:
|
||||
return MySQLMigrator(self.database)
|
||||
|
||||
if self.db_type == DatabaseType.POSTGRESQL:
|
||||
return PostgresqlMigrator(self.database)
|
||||
|
||||
def initialize_database(self):
|
||||
"""
|
||||
Connect to the database
|
||||
initialize the previously defined databases
|
||||
create tables if they don't exist.
|
||||
"""
|
||||
self.database = self.create_database()
|
||||
self.database.connect()
|
||||
|
||||
migrator = self.migrator
|
||||
|
||||
for model in data_models.ALL_MODELS:
|
||||
model = model.Use(self.database)
|
||||
|
||||
if self.database.table_exists(model):
|
||||
migration_operations = [
|
||||
migrator.add_column(
|
||||
"some field", field[0], field[1]
|
||||
)
|
||||
for field in model._meta.fields.items()
|
||||
]
|
||||
|
||||
migrate(*migration_operations)
|
||||
else:
|
||||
self.database.create_tables([model], safe=True)
|
||||
|
||||
#self.database.create_tables([model.Use(self.database) for model in data_models.ALL_MODELS], safe=True)
|
||||
|
||||
"""
|
||||
upgrade old databases.
|
||||
If a collumn has been added in a new version this adds it to old Tables,
|
||||
without deleting the data in legacy databases
|
||||
"""
|
||||
|
||||
for model in data_models.ALL_MODELS:
|
||||
model = model.Use(self.database)
|
||||
|
||||
|
||||
|
||||
print(model._meta.fields)
|
||||
|
||||
def push(self, database_object: objects.DatabaseObject):
|
||||
"""
|
||||
Adds a new music object to the database using the corresponding method from the `write` session.
|
||||
When possible, rather use the `push_many` function.
|
||||
This gets even more important, when using a remote database server.
|
||||
|
||||
Args:
|
||||
database_object (objects.MusicObject): The music object to add to the database.
|
||||
|
||||
Returns:
|
||||
The newly added music object.
|
||||
"""
|
||||
|
||||
with write.WritingSession(self.database) as writing_session:
|
||||
if isinstance(database_object, objects.Song):
|
||||
return writing_session.add_song(database_object)
|
||||
|
||||
if isinstance(database_object, objects.Album):
|
||||
return writing_session.add_album(database_object)
|
||||
|
||||
if isinstance(database_object, objects.Artist):
|
||||
return writing_session.add_artist(database_object)
|
||||
|
||||
def push_many(self, database_objects: List[objects.DatabaseObject]) -> None:
|
||||
"""
|
||||
Adds a list of MusicObject instances to the database.
|
||||
This function sends only needs one querry for each type of table added.
|
||||
Beware that if you have for example an object like this:
|
||||
- Album
|
||||
- Song
|
||||
- Song
|
||||
you already have 3 different Tables.
|
||||
|
||||
Unlike the function `push`, this function doesn't return the added database objects.
|
||||
|
||||
Args:
|
||||
database_objects: List of MusicObject instances to be added to the database.
|
||||
"""
|
||||
|
||||
with write.WritingSession(self.database) as writing_session:
|
||||
for obj in database_objects:
|
||||
if isinstance(obj, objects.Song):
|
||||
writing_session.add_song(obj)
|
||||
continue
|
||||
|
||||
if isinstance(obj, objects.Album):
|
||||
writing_session.add_album(obj)
|
||||
continue
|
||||
|
||||
if isinstance(obj, objects.Artist):
|
||||
writing_session.add_artist(obj)
|
||||
continue
|
||||
|
||||
|
||||
def __del__(self):
|
||||
self.database.close()
|
||||
@ -1,16 +0,0 @@
|
||||
import music_kraken
|
||||
|
||||
import logging
|
||||
print("Setting logging-level to DEBUG")
|
||||
logging.getLogger().setLevel(logging.DEBUG)
|
||||
|
||||
if __name__ == "__main__":
|
||||
commands = [
|
||||
"s: #a PTK",
|
||||
"26",
|
||||
"d: 1",
|
||||
]
|
||||
|
||||
|
||||
music_kraken.cli.download(genre="test", command_list=commands, process_metadata_anyway=True)
|
||||
_ = "debug"
|
||||
@ -1,181 +0,0 @@
|
||||
from music_kraken.objects import (
|
||||
Song,
|
||||
Album,
|
||||
Artist,
|
||||
Label,
|
||||
Source,
|
||||
DatabaseObject
|
||||
)
|
||||
from music_kraken.objects.collection import Collection
|
||||
from music_kraken.utils.enums import SourcePages
|
||||
|
||||
from music_kraken.objects.lint_default_factories import lint
|
||||
|
||||
lint()
|
||||
|
||||
"""
|
||||
song = Song(title="Sad Story", isrc="testTest")
|
||||
other_song = Song(title="hihi", genre="dsbm")
|
||||
|
||||
song.merge(other_song)
|
||||
|
||||
print(song.__dict__)
|
||||
print(other_song.__dict__)
|
||||
|
||||
other_song.title = ":3"
|
||||
|
||||
print(song.__dict__)
|
||||
print(other_song.__dict__)
|
||||
|
||||
|
||||
print(song)
|
||||
print(type(song).__dict__["__annotations__"])
|
||||
"""
|
||||
"""
|
||||
only_smile = Artist(
|
||||
name="Only Smile",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/")],
|
||||
main_album_list=[
|
||||
Album(
|
||||
title="Few words...",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/few-words")],
|
||||
song_list=[
|
||||
Song(title="Everything will be fine"),
|
||||
Song(title="Only Smile"),
|
||||
Song(title="Dear Diary"),
|
||||
Song(title="Sad Story")
|
||||
],
|
||||
artist_list=[
|
||||
Artist(
|
||||
name="Only Smile",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/")],
|
||||
main_album_list=[
|
||||
Album(
|
||||
title="Few words...",
|
||||
source_list=[
|
||||
Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/few-words")],
|
||||
song_list=[
|
||||
Song(title="Everything will be fine"),
|
||||
Song(title="Only Smile"),
|
||||
Song(title="Dear Diary"),
|
||||
Song(title="Sad Story")
|
||||
],
|
||||
artist_list=[
|
||||
Artist(
|
||||
name="Only Smile",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/")]
|
||||
)
|
||||
]
|
||||
),
|
||||
Album(
|
||||
title="Your best friend",
|
||||
source_list=[
|
||||
Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/your-best-friend")]
|
||||
)
|
||||
]
|
||||
),
|
||||
Artist(
|
||||
name="Only Smile",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/")],
|
||||
main_album_list=[
|
||||
Album(
|
||||
title="Few words...",
|
||||
source_list=[
|
||||
Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/few-words")],
|
||||
song_list=[
|
||||
Song(title="Everything will be fine"),
|
||||
Song(title="Only Smile"),
|
||||
Song(title="Dear Diary"),
|
||||
Song(title="Sad Story")
|
||||
],
|
||||
artist_list=[
|
||||
Artist(
|
||||
name="Only Smile",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/")]
|
||||
)
|
||||
]
|
||||
),
|
||||
Album(
|
||||
title="Your best friend",
|
||||
source_list=[
|
||||
Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/your-best-friend")]
|
||||
)
|
||||
]
|
||||
)
|
||||
]
|
||||
),
|
||||
Album(
|
||||
title="Your best friend",
|
||||
source_list=[Source(SourcePages.BANDCAMP, "https://onlysmile.bandcamp.com/album/your-best-friend")]
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
objects_by_id = {}
|
||||
|
||||
|
||||
def add_to_objects_dump(db_obj: DatabaseObject):
|
||||
objects_by_id[db_obj.id] = db_obj
|
||||
|
||||
for collection in db_obj.all_collections:
|
||||
for new_db_obj in collection:
|
||||
if new_db_obj.id not in objects_by_id:
|
||||
add_to_objects_dump(new_db_obj)
|
||||
|
||||
|
||||
add_to_objects_dump(only_smile)
|
||||
|
||||
for _id, _object in objects_by_id.items():
|
||||
print(_id, _object, sep=": ")
|
||||
|
||||
print(only_smile)
|
||||
|
||||
c = Collection([Song(title="hi"), Song(title="hi2"), Song(title="hi3")])
|
||||
c1 = Collection([Song(title="he"), Song(title="hi5")])
|
||||
c11 = Collection([Song(title="wow how ultra subby", isrc="hiii")])
|
||||
c2 = Collection([Song(title="heeee")])
|
||||
|
||||
b = Collection([Song(title="some b"), Song(title="other b")])
|
||||
b1 = Collection([Song(title="sub b")])
|
||||
b11 = Collection([Song(title="This shouldn't work")])
|
||||
|
||||
b1.contain_collection_inside(b11)
|
||||
|
||||
b.contain_collection_inside(b1)
|
||||
b.contain_collection_inside(c1)
|
||||
|
||||
c.contain_collection_inside(c1)
|
||||
c.contain_collection_inside(c2)
|
||||
|
||||
c1.contain_collection_inside(c11)
|
||||
c1.contain_collection_inside(c11)
|
||||
|
||||
print(c.data)
|
||||
print(c1.data)
|
||||
|
||||
c.append(Song(title="after creation"))
|
||||
|
||||
other_song = Song(title="has same isrc", isrc="hiii", genre="hssss")
|
||||
print(c.contains(other_song))
|
||||
c11.append(other_song)
|
||||
print(other_song)
|
||||
|
||||
|
||||
print()
|
||||
print(c.data, len(c))
|
||||
print(c1.data)
|
||||
print([(obj.genre or "various") + ":" + obj.title for obj in c.data])
|
||||
|
||||
print()
|
||||
print("c: ", c)
|
||||
print("b: ", b)
|
||||
|
||||
c.sync_with_other_collection(b)
|
||||
print("synced: ")
|
||||
|
||||
print("c: ", c)
|
||||
print("b: ", b)
|
||||
|
||||
print(c.data)
|
||||
print(c._data)
|
||||
"""
|
||||
@ -1,42 +0,0 @@
|
||||
from music_kraken import objects
|
||||
from music_kraken.pages import EncyclopaediaMetallum
|
||||
|
||||
|
||||
def search():
|
||||
results = EncyclopaediaMetallum._raw_search("#a Ghost Bath")
|
||||
print(results)
|
||||
print(results[0].source_collection)
|
||||
|
||||
|
||||
def fetch_artist():
|
||||
artist = objects.Artist(
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.MUSIFY, "https://musify.club/artist/psychonaut-4-83193"),
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/bands/Ghost_Bath/3540372489")
|
||||
]
|
||||
)
|
||||
|
||||
artist: objects.Artist = EncyclopaediaMetallum.fetch_details(artist)
|
||||
print(artist.options)
|
||||
|
||||
|
||||
def fetch_album():
|
||||
album = objects.Album(
|
||||
source_list=[objects.Source(objects.SourcePages.MUSIFY,
|
||||
"https://musify.club/release/linkin-park-hybrid-theory-2000-188")]
|
||||
)
|
||||
|
||||
album: objects.Album = EncyclopaediaMetallum.fetch_details(album)
|
||||
print(album.options)
|
||||
|
||||
song: objects.Song
|
||||
for artist in album.artist_collection:
|
||||
print(artist.id, artist.name)
|
||||
for song in album.song_collection:
|
||||
for artist in song.main_artist_collection:
|
||||
print(artist.id, artist.name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
fetch_artist()
|
||||
@ -1,704 +0,0 @@
|
||||
Metadata-Version: 2.1
|
||||
Name: music-kraken
|
||||
Version: 1.2.2
|
||||
Summary: An extensive music downloader crawling the internet. It gets its metadata from a couple of metadata providers, and it scrapes the audiofiles.
|
||||
Home-page: https://github.com/HeIIow2/music-downloader
|
||||
Author: Hellow2
|
||||
Author-email: Hellow2@outlook.de
|
||||
License: UNKNOWN
|
||||
Platform: UNKNOWN
|
||||
Description-Content-Type: text/markdown
|
||||
License-File: LICENSE
|
||||
|
||||
# Music Kraken
|
||||
|
||||
<img align="right" src="assets/logo.svg" width=300>
|
||||
|
||||
1. [Installlation](#installation)
|
||||
2. [Command Line Usage](#quick-guide)
|
||||
3. [Contribute](#contribute)
|
||||
4. [Matrix Space](#matrix-space), if you don't wanna read: **[Invite](https://matrix.to/#/#music-kraken:matrix.org)**
|
||||
|
||||
5. [Library Usage / Python Interface](#programming-interface--use-as-library)
|
||||
6. [About Metadata](#metadata)
|
||||
7. [About the Audio](#download)
|
||||
8. [About the Lyrics](#lyrics)
|
||||
|
||||
---
|
||||
|
||||
## Installation
|
||||
|
||||
You can find and get this project from either [PyPI](https://pypi.org/project/music-kraken/) as a Python-Package,
|
||||
or simply the source code from [GitHub](https://github.com/HeIIow2/music-downloader). Note that even though
|
||||
everything **SHOULD** work cross-platform, I have only tested it on Ubuntu.
|
||||
If you enjoy this project, feel free to give it a star on GitHub.
|
||||
|
||||
```sh
|
||||
# Install it with
|
||||
pip install music-kraken
|
||||
|
||||
# and simply run it like this:
|
||||
music-kraken
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
- ffmpeg
|
||||
- pandoc
|
||||
|
||||
### Notes for Python 3.9
|
||||
|
||||
Unfortunately I use features that newly git introduced in [Python 3.10](https://docs.python.org/3/library/types.html#types.UnionType).
|
||||
So unfortunately you **CAN'T** run this programm with python 3.9. [#10][i10]
|
||||
|
||||
### Notes for WSL
|
||||
|
||||
If you choose to run it in WSL, make sure ` ~/.local/bin` is added to your `$PATH` [#2][i2]
|
||||
|
||||
## Quick-Guide
|
||||
|
||||
**Genre:** First, the cli asks you to input a genre you want to download to. The options it gives you (if it gives you any) are all the folders you have in the music directory. You can also just input a new one.
|
||||
|
||||
**What to download:** After that it prompts you for a search. Here are a couple examples how you can search:
|
||||
|
||||
```
|
||||
> #a <any artist>
|
||||
searches for the artist <any artist>
|
||||
|
||||
> #a <any artist> #r <any releas>
|
||||
searches for the release (album) <any release> by the artist <any artist>
|
||||
|
||||
> #r <any release> Me #t <any track>
|
||||
searches for the track <any track> from the release <any relaese>
|
||||
```
|
||||
|
||||
After searching with this syntax, it prompts you with multiple results. You can either choose one of those by inputing its id `int`, or you can search for a new query.
|
||||
|
||||
After you chose either an artist, a release group, a release, or a track by its id, download it by inputting the string `ok`. My downloader will download it automatically for you.
|
||||
|
||||
---
|
||||
|
||||
## CONTRIBUTE
|
||||
|
||||
I am happy about every pull request. To contribute look [here](contribute.md).
|
||||
|
||||
## Matrix Space
|
||||
|
||||
<img align="right" src="assets/element_logo.png" width=100>
|
||||
|
||||
I decided against creating a discord server, due to piracy communities get often banned from discord. A good and free Alternative are Matrix Spaces. I reccomend the use of the Client [Element](https://element.io/download). It is completely open source.
|
||||
|
||||
**Click [this link](https://matrix.to/#/#music-kraken:matrix.org) _([https://matrix.to/#/#music-kraken:matrix.org](https://matrix.to/#/#music-kraken:matrix.org))_ to join.**
|
||||
|
||||
---
|
||||
|
||||
# Programming Interface / Use as Library
|
||||
|
||||
This application is $100\%$ centered around Data. Thus the most important thing for working with musik kraken is, to understand how I structured the data.
|
||||
|
||||
## quick Overview
|
||||
|
||||
- explanation of the [Data Model](#data-model)
|
||||
- how to use the [Data Objects](#data-objects)
|
||||
|
||||
```mermaid
|
||||
---
|
||||
title: Quick Overview
|
||||
---
|
||||
sequenceDiagram
|
||||
|
||||
participant pg as Page (eg. YouTube, MB, Musify, ...)
|
||||
participant obj as DataObjects (eg. Song, Artist, ...)
|
||||
participant db as DataBase
|
||||
|
||||
obj ->> db: write
|
||||
db ->> obj: read
|
||||
|
||||
pg -> obj: find a source for any page, for object.
|
||||
obj -> pg: add more detailed data from according page.
|
||||
obj -> pg: if available download audio to target.
|
||||
```
|
||||
|
||||
## Data Model
|
||||
|
||||
The Data Structure, that the whole programm is built on looks as follows:
|
||||
|
||||
```mermaid
|
||||
---
|
||||
title: Music Data
|
||||
---
|
||||
erDiagram
|
||||
|
||||
|
||||
|
||||
Target {
|
||||
|
||||
}
|
||||
|
||||
Lyrics {
|
||||
|
||||
}
|
||||
|
||||
Song {
|
||||
|
||||
}
|
||||
|
||||
Album {
|
||||
|
||||
}
|
||||
|
||||
Artist {
|
||||
|
||||
}
|
||||
|
||||
Label {
|
||||
|
||||
}
|
||||
|
||||
Source {
|
||||
|
||||
}
|
||||
|
||||
Source }o--|| Song : from
|
||||
Source }o--|| Lyrics : from
|
||||
Source }o--|| Album : from
|
||||
Source }o--|| Artist : from
|
||||
Source }o--|| Label : from
|
||||
|
||||
Song }o--o{ Album : AlbumSong
|
||||
Album }o--o{ Artist : ArtistAlbum
|
||||
Song }o--o{ Artist : features
|
||||
|
||||
Label }o--o{ Album : LabelAlbum
|
||||
Label }o--o{ Artist : LabelSong
|
||||
|
||||
Song ||--o{ Lyrics : contains
|
||||
Song ||--o{ Target : points
|
||||
```
|
||||
|
||||
Ok now this **WILL** look intimidating, thus I break it down quickly.
|
||||
*That is also the reason I didn't add all Attributes here.*
|
||||
|
||||
The most important Entities are:
|
||||
|
||||
- Song
|
||||
- Album
|
||||
- Artist
|
||||
- Label
|
||||
|
||||
All of them *(and Lyrics)* can have multiple Sources, and every Source can only Point to one of those Element.
|
||||
|
||||
The `Target` Entity represents the location on the hard drive a Song has. One Song can have multiple download Locations.
|
||||
|
||||
The `Lyrics` Entity simply represents the Lyrics of each Song. One Song can have multiple Lyrics, e.g. Translations.
|
||||
|
||||
Here is the simplified Diagramm without only the main Entities.
|
||||
|
||||
|
||||
```mermaid
|
||||
---
|
||||
title: simplified Music Data
|
||||
---
|
||||
erDiagram
|
||||
|
||||
Song {
|
||||
|
||||
}
|
||||
|
||||
Album {
|
||||
|
||||
}
|
||||
|
||||
Artist {
|
||||
|
||||
}
|
||||
|
||||
Label {
|
||||
|
||||
}
|
||||
|
||||
Song }o--o{ Album : AlbumSong
|
||||
Album }o--o{ Artist : ArtistAlbum
|
||||
Song }o--o{ Artist : features
|
||||
|
||||
Label }o--o{ Album : LabelAlbum
|
||||
Label }o--o{ Artist : LabelSong
|
||||
|
||||
```
|
||||
|
||||
Looks way more manageable, doesn't it?
|
||||
|
||||
The reason every relation here is a `n:m` *(many to many)* relation is not, that it makes sense in the aspekt of modeling reality, but to be able to put data from many Sources in the same Data Model.
|
||||
Every Service models Data a bit different, and projecting a one-to-many relationship to a many to many relationship without data loss is easy. The other way around it is basically impossible
|
||||
|
||||
## Data Objects
|
||||
|
||||
> Not 100% accurate yet and *might* change slightly
|
||||
|
||||
### Creation
|
||||
|
||||
```python
|
||||
# importing the libraries I build on
|
||||
from music_kraken import objects
|
||||
|
||||
import pycountry
|
||||
|
||||
|
||||
song = objects.Song(
|
||||
genre="HS Core",
|
||||
title="Vein Deep in the Solution",
|
||||
length=666,
|
||||
isrc="US-S1Z-99-00001",
|
||||
tracksort=2,
|
||||
target=[
|
||||
objects.Target(file="song.mp3", path="example")
|
||||
],
|
||||
lyrics_list=[
|
||||
objects.Lyrics(text="these are some depressive lyrics", language="en"),
|
||||
objects.Lyrics(text="Dies sind depressive Lyrics", language="de")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.YOUTUBE, "https://youtu.be/dfnsdajlhkjhsd"),
|
||||
objects.Source(objects.SourcePages.MUSIFY, "https://ln.topdf.de/Music-Kraken/")
|
||||
],
|
||||
album_list=[
|
||||
objects.Album(
|
||||
title="One Final Action",
|
||||
date=objects.ID3Timestamp(year=1986, month=3, day=1),
|
||||
language=pycountry.languages.get(alpha_2="en"),
|
||||
label_list=[
|
||||
objects.Label(name="an album label")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM, "https://www.metal-archives.com/albums/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
]
|
||||
),
|
||||
],
|
||||
main_artist_list=[
|
||||
objects.Artist(
|
||||
name="I'm in a coffin",
|
||||
source_list=[
|
||||
objects.Source(
|
||||
objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/bands/I%27m_in_a_Coffin/127727"
|
||||
)
|
||||
]
|
||||
),
|
||||
objects.Artist(name="some_split_artist")
|
||||
],
|
||||
feature_artist_list=[
|
||||
objects.Artist(
|
||||
name="Ruffiction",
|
||||
label_list=[
|
||||
objects.Label(name="Ruffiction Productions")
|
||||
]
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
print(song.option_string)
|
||||
for album in song.album_collection:
|
||||
print(album.option_string)
|
||||
for artist in song.main_artist_collection:
|
||||
print(artist.option_string)
|
||||
```
|
||||
|
||||
|
||||
|
||||
If you just want to start implementing, then just use the code example, I don't care.
|
||||
For those who don't want any bugs and use it as intended *(which is recommended, cuz I am only one person so there are defs bugs)* continue reading.
|
||||
|
||||
## Appending and Merging data
|
||||
|
||||
If you want to append for example a Song to an Album, you obviously need to check beforehand if the Song already exists in the Album, and if so, you need to merge their data in one Song object, to not loose any Information.
|
||||
|
||||
Fortunately I implemented all of this functionality in [objects.Collection](#collection).append(music_object).
|
||||
I made a flow chart showing how it works:
|
||||
|
||||
```mermaid
|
||||
---
|
||||
title: "Collection.append(music_object: MusicObject)"
|
||||
---
|
||||
flowchart TD
|
||||
exist("""
|
||||
<b>Check if music_object already exists.</b>
|
||||
<hr>
|
||||
Gets all indexing values with <code>music_object.indexing_values</code>.
|
||||
If any returned value exists in <code>Collection._attribute_to_object_map</code>,
|
||||
the music_object exists
|
||||
""")
|
||||
|
||||
subgraph merge["Merging"]
|
||||
|
||||
_merge("""merges the passed in object in the already
|
||||
existing whith <code>existing.merge(new)</code>""")
|
||||
|
||||
_map("""In case a new source or something simmilar
|
||||
has been addet, it maps the existing object again.
|
||||
""")
|
||||
|
||||
_merge --> _map
|
||||
|
||||
end
|
||||
|
||||
subgraph add["Adding"]
|
||||
|
||||
__map("""map the values from <code>music_object.indexing_values</code>
|
||||
to <code>Collection._attribute_to_object_map</code> by writing
|
||||
those values in the map as keys, and the class I wanna add as values.
|
||||
""")
|
||||
|
||||
_add("""add the new music object to <code>_data</code>""")
|
||||
|
||||
__map --> _add
|
||||
|
||||
end
|
||||
|
||||
exist-->|"if it doesn't exist"|add --> return
|
||||
exist-->|"if already exists"|merge --> return
|
||||
```
|
||||
|
||||
This is Implemented in [music_kraken.objects.Collection.append()](src/music_kraken/objects/collection.py).
|
||||
|
||||
The <u>indexing values</u> are defined in the superclass [DatabaseObject](src/music_kraken/objects/parents.py) and get implemented for each Object seperately. I will just give as example its implementation for the `Song` class:
|
||||
|
||||
```python
|
||||
@property
|
||||
def indexing_values(self) -> List[Tuple[str, object]]:
|
||||
return [
|
||||
('id', self.id),
|
||||
('title', self.unified_title),
|
||||
('barcode', self.barcode),
|
||||
*[('url', source.url) for source in self.source_collection]
|
||||
]
|
||||
```
|
||||
|
||||
## Classes and Objects
|
||||
|
||||
### music_kraken.objects
|
||||
|
||||
#### Collection
|
||||
|
||||
#### Song
|
||||
|
||||
So as you can see, the probably most important Class is the `music_kraken.Song` class. It is used to save the song in *(duh)*.
|
||||
|
||||
It has handful attributes, where half of em are self-explanatory, like `title` or `genre`. The ones like `isrc` are only relevant to you, if you know what it is, so I won't elaborate on it.
|
||||
|
||||
Interesting is the `date`. It uses a custom class. More on that [here](#music_krakenid3timestamp).
|
||||
|
||||
#### ID3Timestamp
|
||||
|
||||
For multiple Reasons I don't use the default `datetime.datetime` class.
|
||||
|
||||
The most important reason is, that you need to pass in at least year, month and day. For every other values there are default values, that are indistinguishable from values that are directly passed in. But I need optional values. The ID3 standart allows default values. Additionally `datetime.datetime` is immutable, thus I can't inherint all the methods. Sorry.
|
||||
|
||||
Anyway you can create those custom objects easily.
|
||||
|
||||
```python
|
||||
from music_kraken import ID3Timestamp
|
||||
|
||||
# returns an instance of ID3Timestamp with the current time
|
||||
ID3Timestamp.now()
|
||||
|
||||
# yea
|
||||
ID3Timestamp(year=1986, month=3, day=1)
|
||||
```
|
||||
|
||||
you can pass in the Arguments:
|
||||
- year
|
||||
- month
|
||||
- day
|
||||
- hour
|
||||
- minute
|
||||
- second
|
||||
|
||||
:)
|
||||
|
||||
# Old implementation
|
||||
|
||||
> IF U USE THIS NOW YOU ARE DUMB *no offense thoug*. IT ISN'T FINISHED AND THE STUFF YOU CODE NOW WILL BE BROKEN TOMORROW
|
||||
> SOON YOU CAN THOUGH
|
||||
|
||||
If you want to use this project, or parts from it in your own projects from it,
|
||||
make sure to be familiar with [Python Modules](https://docs.python.org/3/tutorial/modules.html).
|
||||
Further and better documentation including code examples are yet to come, so here is the rough
|
||||
module structure for now. (Should be up-to-date, but no guarantees)
|
||||
|
||||
If you simply want to run the builtin minimal cli just do this:
|
||||
```python
|
||||
from music_kraken import cli
|
||||
|
||||
cli()
|
||||
```
|
||||
|
||||
### Search for Metadata
|
||||
|
||||
The whole program takes the data it processes further from the cache, a sqlite database.
|
||||
So before you can do anything, you will need to fill it with the songs you want to download (*or create song objects manually, but more on that later*).
|
||||
|
||||
For now the base of everything is [musicbrainz][mb], so you need to get the musicbrainz `id` and `type`. The `id` corresponds to either
|
||||
- an artist
|
||||
- a release group
|
||||
- a release
|
||||
- a recording/track).
|
||||
|
||||
To get this info, you first have to initialize a search object (`music_kraken.MetadataSearch`).
|
||||
|
||||
```python
|
||||
search_object = music_kraken.MetadataSearch()
|
||||
```
|
||||
|
||||
Then you need an initial "text search" to get some options you can choose from. For
|
||||
this you can either specify artists releases and whatever directly with one of the following functions:
|
||||
|
||||
```python
|
||||
# you can directly specify artist, release group, release or recording/track
|
||||
multiple_options = search_object.search_from_text(artist=input("input the name of the artist: "))
|
||||
# you can specify a query see the simple integrated cli on how to use the query
|
||||
multiple_options = search_object.search_from_query(query=input("input the query: "))
|
||||
```
|
||||
|
||||
Both methods return an instance of `MultipleOptions`, which can be directly converted to a string.
|
||||
|
||||
```python
|
||||
print(multiple_options)
|
||||
```
|
||||
|
||||
After the first "*text search*" you can either again search the same way as before,
|
||||
or you can further explore one of the options from the previous search.
|
||||
To explore and select one options from `MultipleOptions`, simply call `MetadataSearch.choose(self, index: int)`.
|
||||
The index represents the number in the previously returned instance of MultipleOptions.
|
||||
The selected Option will be selected and can be downloaded in the next step.
|
||||
|
||||
*Thus, this has to be done **after either search_from_text or search_from_query***
|
||||
|
||||
```python
|
||||
# choosing the best matching band
|
||||
multiple_options = search_object.choose(0)
|
||||
# choosing the first ever release group of this band
|
||||
multiple_options = search_object.choose(1)
|
||||
# printing out the current options
|
||||
print(multiple_options)
|
||||
```
|
||||
|
||||
This process can be repeated indefinitely (until you run out of memory).
|
||||
A search history is kept in the Search instance. You could go back to
|
||||
the previous search (without any loading time) like this:
|
||||
|
||||
```python
|
||||
multiple_options = search_object.get_previous_options()
|
||||
```
|
||||
|
||||
### Downloading Metadata / Filling up the Cache
|
||||
|
||||
You can download following metadata:
|
||||
- an artist (the whole discography)
|
||||
- a release group
|
||||
- a release
|
||||
- a track/recording
|
||||
|
||||
If you got an instance of `MetadataSearch`, like I elaborated [previously](#search-for-metadata), downloading every piece of metadata from the currently selected Option is really quite easy.
|
||||
|
||||
```python
|
||||
from music_kraken import fetch_metadata_from_search
|
||||
|
||||
# this is it :)
|
||||
music_kraken.fetch_metadata_from_search(search_object)
|
||||
```
|
||||
|
||||
If you already know what you want to download you can skip the search instance and simply do the following.
|
||||
|
||||
```python
|
||||
from music_kraken import fetch_metadata
|
||||
|
||||
# might change and break after I add multiple metadata sources which I will
|
||||
|
||||
fetch_metadata(id_=musicbrainz_id, type=metadata_type)
|
||||
```
|
||||
The option type is a string (*I'm sorry for not making it an enum I know its a bad pratice*), which can
|
||||
have following values:
|
||||
- 'artist'
|
||||
- 'release_group'
|
||||
- 'release'
|
||||
- 'recording'
|
||||
|
||||
**PAY ATTENTION TO TYPOS, IT'S CASE SENSITIVE**
|
||||
|
||||
The musicbrainz id is just the id of the object from musicbrainz.
|
||||
|
||||
After following those steps, it might take a couple seconds/minutes to execute, but then the Cache will be filled.
|
||||
|
||||
|
||||
### Cache / Temporary Database
|
||||
|
||||
All the data, the functions that download stuff use, can be gotten from the temporary database / cache.
|
||||
The cache can be simply used like this:
|
||||
|
||||
```python
|
||||
music_kraken.test_db
|
||||
```
|
||||
|
||||
When fetching any song data from the cache, you will get it as Song
|
||||
object (music_kraken.Song). There are multiple methods
|
||||
to get different sets of Songs. The names explain the methods pretty
|
||||
well:
|
||||
|
||||
```python
|
||||
from music_kraken import cache
|
||||
|
||||
# gets a single track specified by the id
|
||||
cache.get_track_metadata(id: str)
|
||||
|
||||
# gets a list of tracks.
|
||||
cache.get_tracks_to_download()
|
||||
cache.get_tracks_without_src()
|
||||
cache.get_tracks_without_isrc()
|
||||
cache.get_tracks_without_filepath()
|
||||
```
|
||||
|
||||
The id always is a musicbrainz id and distinct for every track.
|
||||
|
||||
### Setting the Target
|
||||
|
||||
By default the music downloader doesn't know where to save the music file, if downloaded. To set those variables (the directory to save the file in and the filepath), it is enough to run one single command:
|
||||
|
||||
```python
|
||||
from music_kraken import set_target
|
||||
|
||||
# adds file path, file directory and the genre to the database
|
||||
set_target(genre="some test genre")
|
||||
```
|
||||
|
||||
The concept of genres is too loose, to definitely say, this band exclusively plays this genre, or this song is this genre. This doesn't work manually, this will never work automatically. Thus, I've decided to just use the genre as category, to sort the artists and songs by. Most Music players support that.
|
||||
|
||||
As a result of this decision you will have to pass the genre in this function.
|
||||
|
||||
### Get Audio
|
||||
|
||||
This is most likely the most useful and unique feature of this Project. If the cache is filled, you can get audio sources for the songs you only have the metadata, and download them. This works for most songs. I'd guess for about 97% (?)
|
||||
|
||||
First of you will need a List of song objects `music_kraken.Song`. As [mentioned above](#cache--temporary-database), you could get a list like that from the cache.
|
||||
|
||||
```python
|
||||
# Here is an Example
|
||||
from music_kraken import (
|
||||
cache,
|
||||
fetch_sources,
|
||||
fetch_audios
|
||||
)
|
||||
|
||||
# scanning pages, searching for a download and storing results
|
||||
fetch_sources(cache.get_tracks_without_src())
|
||||
|
||||
# downloading all previously fetched sources to previously defined targets
|
||||
fetch_audios(cache.get_tracks_to_download())
|
||||
|
||||
```
|
||||
|
||||
*Note:*
|
||||
To download audio two cases have to be met:
|
||||
1. [The target](#setting-the-target) has to be set beforehand
|
||||
2. The sources have to be fetched beforehand
|
||||
|
||||
---
|
||||
|
||||
## Metadata
|
||||
|
||||
First the metadata has to be downloaded. The best api to do so is undeniably [Musicbrainz][mb]. This is a result of them being a website with a large Database spanning over all Genres.
|
||||
|
||||
### Musicbrainz
|
||||
|
||||
|
||||
|
||||
To fetch from [Musicbrainz][mb] we first have to know what to fetch. A good start is to get an input query, which can be just put into the MB-Api. It then returns a list of possible artists, releases and recordings.
|
||||
|
||||
If the following chosen element is an artist, its discography + a couple tracks are printed, if a release is chosen, the artists + tracklist + release is outputted, If a track is chosen its artists and releases are shown.
|
||||
|
||||
For now, it doesn't if the discography or tracklist is chosen.
|
||||
|
||||
### Metadata to fetch
|
||||
|
||||
I orient on which metadata to download on the keys in `mutagen.EasyID3`. The following I fetch and tag the MP3 with:
|
||||
- title
|
||||
- artist
|
||||
- albumartist
|
||||
- tracknumber
|
||||
- albumsort can sort albums cronological
|
||||
- titlesort is just set to the tracknumber to sort by track order to sort correctly
|
||||
- isrc
|
||||
- musicbrainz_artistid
|
||||
- musicbrainz_albumid
|
||||
- musicbrainz_albumartistid
|
||||
- musicbrainz_albumstatus
|
||||
- language
|
||||
- musicbrainz_albumtype
|
||||
- releasecountry
|
||||
- barcode
|
||||
|
||||
#### albumsort/titlesort
|
||||
|
||||
Those Tags are for the musicplayer to not sort for Example the albums of a band alphabetically, but in another way. I set it just to chronological order
|
||||
|
||||
#### isrc
|
||||
|
||||
This is the **international standart release code**. With this a track can be identified 99% of the time, if it is known and the website has a search api for that. Obviously this will get important later.
|
||||
|
||||
## Download
|
||||
|
||||
Now that the metadata is downloaded and cached, download sources need to be sound, because one can't listen to metadata. Granted it would be amazing if that would be possible.
|
||||
|
||||
### Musify
|
||||
|
||||
The quickest source to get download links from is to my knowledge [musify](https://musify.club/). It's a Russian music downloading page, where many many songs are available to stream and to download. Due to me not wanting to stress the server to much, I abuse a handy feature nearly every page where you can search suff has. The autocomplete api for the search input. Those always are quite limited in the number of results it returns, but it is optimized to be quick. Thus with the http header `Connection` set to `keep-alive` the bottleneck definitely is not at the speed of those requests.
|
||||
|
||||
For musify the endpoint is following: [https://musify.club/search/suggestions?term={title}](https://musify.club/search/suggestions?term=LornaShore) If the http headers are set correctly, then searching for example for "Lorna Shore" yields following result:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"id":"Lorna Shore",
|
||||
"label":"Lorna Shore",
|
||||
"value":"Lorna Shore",
|
||||
"category":"Исполнители",
|
||||
"image":"https://39s.musify.club/img/68/9561484/25159224.jpg",
|
||||
"url":"/artist/lorna-shore-59611"
|
||||
},
|
||||
{"id":"Immortal","label":"Lorna Shore - Immortal (2020)","value":"Immortal","category":"Релизы","image":"https://39s-a.musify.club/img/70/20335517/52174338.jpg","url":"/release/lorna-shore-immortal-2020-1241300"},
|
||||
{"id":"Immortal","label":"Lorna Shore - Immortal","value":"Immortal","category":"Треки","image":"","url":"/track/lorna-shore-immortal-12475071"}
|
||||
]
|
||||
```
|
||||
|
||||
This is a shortened example for the response the api gives. The results are very Limited, but it is also very efficient to parse. The steps I take are:
|
||||
|
||||
- call the api with the query being the track name
|
||||
- parse the json response to an object
|
||||
- look at how different the title and artist are on every element from the category `Треки`, translated roughly to track or release.
|
||||
- If they match get the download links and cache them.
|
||||
|
||||
### Youtube
|
||||
|
||||
Herte the **isrc** plays a huge role. You probably know it, when you search on youtube for a song, and the music videos has a long intro or the first result is a live version. I don't want those in my music collection, only if the tracks are like this in the official release. Well how can you get around that?
|
||||
|
||||
Turns out if you search for the **isrc** on youtube the results contain the music, like it is on the official release and some japanese meme videos. The tracks I wan't just have the title of the released track, so one can just compare those two.
|
||||
|
||||
For searching, as well as for downloading I use the programm `youtube-dl`, which also has a programming interface for python.
|
||||
|
||||
There are two bottlenecks with this approach though:
|
||||
1. `youtube-dl` is just slow. Actually it has to be, to not get blocked by youtube.
|
||||
2. Ofthen musicbrainz just doesn't give the isrc for some songs.
|
||||
|
||||
|
||||
## Lyrics
|
||||
|
||||
To get the Lyrics, I scrape them, and put those in the USLT ID3 Tags of for example mp3 files. Unfortunately some players, like the one I use, Rhythmbox don't support USLT Lyrics. So I created an Plugin for Rhythmbox. You can find it here: [https://github.com/HeIIow2/rythmbox-id3-lyrics-support](https://github.com/HeIIow2/rythmbox-id3-lyrics-support).
|
||||
|
||||
### Genius
|
||||
|
||||
For the lyrics source the page [https://genius.com/](https://genius.com/) is easily sufficient. It has most songs. Some songs are not present though, but that is fine, because the lyrics are optional anyways.
|
||||
|
||||
|
||||
[i10]: https://github.com/HeIIow2/music-downloader/issues/10
|
||||
[i2]: https://github.com/HeIIow2/music-downloader/issues/2
|
||||
[mb]: https://musicbrainz.org/
|
||||
|
||||
|
||||
@ -1,46 +0,0 @@
|
||||
LICENSE
|
||||
README.md
|
||||
pyproject.toml
|
||||
requirements.txt
|
||||
setup.py
|
||||
version
|
||||
assets/database_structure.sql
|
||||
assets/temp_database_structure.sql
|
||||
src/music_kraken/__init__.py
|
||||
src/music_kraken/__main__.py
|
||||
src/music_kraken.egg-info/PKG-INFO
|
||||
src/music_kraken.egg-info/SOURCES.txt
|
||||
src/music_kraken.egg-info/dependency_links.txt
|
||||
src/music_kraken.egg-info/entry_points.txt
|
||||
src/music_kraken.egg-info/requires.txt
|
||||
src/music_kraken.egg-info/top_level.txt
|
||||
src/music_kraken/database/__init__.py
|
||||
src/music_kraken/database/data_models.py
|
||||
src/music_kraken/database/database.py
|
||||
src/music_kraken/database/object_cache.py
|
||||
src/music_kraken/database/old_database.py
|
||||
src/music_kraken/database/read.py
|
||||
src/music_kraken/database/temp_database.py
|
||||
src/music_kraken/database/write.py
|
||||
src/music_kraken/lyrics/__init__.py
|
||||
src/music_kraken/metadata/__init__.py
|
||||
src/music_kraken/not_used_anymore/__init__.py
|
||||
src/music_kraken/not_used_anymore/fetch_audio.py
|
||||
src/music_kraken/not_used_anymore/fetch_source.py
|
||||
src/music_kraken/not_used_anymore/sources/__init__.py
|
||||
src/music_kraken/not_used_anymore/sources/genius.py
|
||||
src/music_kraken/not_used_anymore/sources/local_files.py
|
||||
src/music_kraken/not_used_anymore/sources/musify.py
|
||||
src/music_kraken/not_used_anymore/sources/source.py
|
||||
src/music_kraken/not_used_anymore/sources/youtube.py
|
||||
src/music_kraken/static_files/temp_database_structure.sql
|
||||
src/music_kraken/tagging/__init__.py
|
||||
src/music_kraken/tagging/id3.py
|
||||
src/music_kraken/target/__init__.py
|
||||
src/music_kraken/target/set_target.py
|
||||
src/music_kraken/utils/__init__.py
|
||||
src/music_kraken/utils/functions.py
|
||||
src/music_kraken/utils/object_handeling.py
|
||||
src/music_kraken/utils/phonetic_compares.py
|
||||
src/music_kraken/utils/shared.py
|
||||
src/music_kraken/utils/string_processing.py
|
||||
@ -1 +0,0 @@
|
||||
|
||||
@ -1,3 +0,0 @@
|
||||
[console_scripts]
|
||||
music-kraken = music_kraken:cli
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
requests~=2.28.1
|
||||
mutagen~=1.46.0
|
||||
musicbrainzngs~=0.7.1
|
||||
jellyfish~=0.9.0
|
||||
pydub~=0.25.1
|
||||
youtube_dl
|
||||
beautifulsoup4~=4.11.1
|
||||
pycountry~=22.3.5
|
||||
python-dateutil~=2.8.2
|
||||
pandoc~=2.3
|
||||
SQLAlchemy
|
||||
setuptools~=60.2.0
|
||||
@ -1 +0,0 @@
|
||||
music_kraken
|
||||
@ -1,5 +0,0 @@
|
||||
import music_kraken
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
music_kraken.cli()
|
||||
@ -1,4 +0,0 @@
|
||||
from music_kraken import gtk_gui
|
||||
|
||||
if __name__ == "__main__":
|
||||
gtk_gui()
|
||||
@ -1,51 +0,0 @@
|
||||
from music_kraken import objects
|
||||
from music_kraken.pages import Musify
|
||||
|
||||
|
||||
def search():
|
||||
results = Musify._raw_search("#a Ghost Bath")
|
||||
print(results)
|
||||
|
||||
|
||||
def fetch_artist():
|
||||
artist = objects.Artist(
|
||||
source_list=[objects.Source(objects.SourcePages.MUSIFY, "https://musify.club/artist/psychonaut-4-83193")]
|
||||
)
|
||||
|
||||
artist = objects.Artist(
|
||||
source_list=[objects.Source(objects.SourcePages.MUSIFY, "https://musify.club/artist/ghost-bath-280348/")]
|
||||
)
|
||||
|
||||
artist = objects.Artist(
|
||||
source_list=[objects.Source(objects.SourcePages.MUSIFY, "https://musify.club/artist/lana-del-rey-124788/releases")]
|
||||
)
|
||||
|
||||
artist: objects.Artist = Musify.fetch_details(artist)
|
||||
print(artist.options)
|
||||
print(artist.main_album_collection[0].source_collection)
|
||||
|
||||
|
||||
def fetch_album():
|
||||
album = objects.Album(
|
||||
source_list=[objects.Source(objects.SourcePages.MUSIFY,
|
||||
"https://musify.club/release/linkin-park-hybrid-theory-2000-188")]
|
||||
)
|
||||
|
||||
album = objects.Album(
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.MUSIFY, "https://musify.club/release/ghost-bath-self-loather-2021-1554266")
|
||||
]
|
||||
)
|
||||
|
||||
album: objects.Album = Musify.fetch_details(album)
|
||||
print(album.options)
|
||||
|
||||
song: objects.Song
|
||||
for artist in album.artist_collection:
|
||||
print(artist.id, artist.name)
|
||||
for song in album.song_collection:
|
||||
for artist in song.main_artist_collection:
|
||||
print(artist.id, artist.name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
fetch_artist()
|
||||
@ -1,14 +0,0 @@
|
||||
|
||||
|
||||
class Foo:
|
||||
class_attr: str
|
||||
class_attr_two: str
|
||||
|
||||
def __init__(self, foo: str, bar) -> None: ...
|
||||
|
||||
|
||||
|
||||
|
||||
f = Foo("fdfasdf", ["fsd", "fsedf"])
|
||||
|
||||
print(f)
|
||||
@ -1,5 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add the parent directory of the current file (i.e., the "tests" directory) to sys.path
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||
@ -1,94 +0,0 @@
|
||||
from mutagen import id3
|
||||
import pycountry
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Add the parent directory of the src package to the Python module search path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from music_kraken import objects
|
||||
|
||||
|
||||
class TestSong(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.album_list = [
|
||||
objects.Album(
|
||||
title="One Final Action",
|
||||
date=objects.ID3Timestamp(year=1986, month=3, day=1),
|
||||
language=pycountry.languages.get(alpha_2="en"),
|
||||
label_list=[
|
||||
objects.Label(name="an album label")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/albums/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
]
|
||||
),
|
||||
]
|
||||
|
||||
self.artist_list = []
|
||||
|
||||
self.main_artist_list=[
|
||||
objects.Artist(
|
||||
name="I'm in a coffin",
|
||||
source_list=[
|
||||
objects.Source(
|
||||
objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/bands/I%27m_in_a_Coffin/127727"
|
||||
)
|
||||
]
|
||||
),
|
||||
objects.Artist(name="some_split_artist")
|
||||
]
|
||||
feature_artist_list=[
|
||||
objects.Artist(
|
||||
name="Ruffiction",
|
||||
label_list=[
|
||||
objects.Label(name="Ruffiction Productions")
|
||||
]
|
||||
)
|
||||
]
|
||||
|
||||
self.artist_list.extend(self.main_artist_list)
|
||||
self.artist_list.extend(feature_artist_list)
|
||||
|
||||
self.song = objects.Song(
|
||||
genre="HS Core",
|
||||
title="Vein Deep in the Solution",
|
||||
length=666,
|
||||
isrc="US-S1Z-99-00001",
|
||||
tracksort=2,
|
||||
target_list=[
|
||||
objects.Target(file="song.mp3", path="example")
|
||||
],
|
||||
lyrics_list=[
|
||||
objects.Lyrics(
|
||||
text="these are some depressive lyrics", language="en"),
|
||||
objects.Lyrics(
|
||||
text="Dies sind depressive Lyrics", language="de")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.YOUTUBE,
|
||||
"https://youtu.be/dfnsdajlhkjhsd"),
|
||||
objects.Source(objects.SourcePages.MUSIFY,
|
||||
"https://ln.topdf.de/Music-Kraken/")
|
||||
],
|
||||
album_list=self.album_list,
|
||||
main_artist_list=self.main_artist_list,
|
||||
feature_artist_list=feature_artist_list,
|
||||
)
|
||||
|
||||
self.song.compile()
|
||||
|
||||
def test_album(self):
|
||||
for artist in self.song.main_artist_collection:
|
||||
for artist_album in artist.main_album_collection:
|
||||
self.assertIn(artist_album, self.song.album_collection)
|
||||
|
||||
def test_artist(self):
|
||||
for album in self.song.album_collection:
|
||||
for album_artist in album.artist_collection:
|
||||
self.assertIn(album_artist, self.song.main_artist_collection)
|
||||
|
||||
@ -1,43 +0,0 @@
|
||||
import sys
|
||||
import os
|
||||
import unittest
|
||||
|
||||
# Add the parent directory of the src package to the Python module search path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from music_kraken import pages
|
||||
from music_kraken.pages import download_center
|
||||
from music_kraken.pages.download_center import page_attributes
|
||||
|
||||
class TestPageSelection(unittest.TestCase):
|
||||
def test_no_shady_pages(self):
|
||||
search = download_center.Download(
|
||||
exclude_shady=True
|
||||
)
|
||||
|
||||
for page in search.pages:
|
||||
self.assertNotIn(page, page_attributes.SHADY_PAGES)
|
||||
|
||||
def test_excluding(self):
|
||||
search = download_center.Download(
|
||||
exclude_pages={pages.EncyclopaediaMetallum}
|
||||
)
|
||||
|
||||
for page in search.pages:
|
||||
self.assertNotEqual(page, pages.EncyclopaediaMetallum)
|
||||
|
||||
|
||||
def test_audio_one(self):
|
||||
search = download_center.Download(
|
||||
exclude_shady=True
|
||||
)
|
||||
|
||||
for audio_page in search.audio_pages:
|
||||
self.assertIn(audio_page, page_attributes.AUDIO_PAGES)
|
||||
|
||||
def test_audio_two(self):
|
||||
search = download_center.Download()
|
||||
|
||||
for audio_page in search.audio_pages:
|
||||
self.assertIn(audio_page, page_attributes.AUDIO_PAGES)
|
||||
|
||||
|
||||
@ -1,239 +0,0 @@
|
||||
from mutagen import id3
|
||||
import pycountry
|
||||
import unittest
|
||||
import sys
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
# Add the parent directory of the src package to the Python module search path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
from music_kraken import objects
|
||||
|
||||
"""
|
||||
Testing the Formatted text is barely possible cuz one false character and it fails.
|
||||
Not worth the trouble
|
||||
"""
|
||||
|
||||
|
||||
class TestSong(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.song = objects.Song(
|
||||
genre="HS Core",
|
||||
title="Vein Deep in the Solution",
|
||||
length=666,
|
||||
isrc="US-S1Z-99-00001",
|
||||
tracksort=2,
|
||||
target_list=[
|
||||
objects.Target(file="song.mp3", path="example")
|
||||
],
|
||||
lyrics_list=[
|
||||
objects.Lyrics(
|
||||
text="these are some depressive lyrics", language="en"),
|
||||
objects.Lyrics(
|
||||
text="Dies sind depressive Lyrics", language="de")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.YOUTUBE,
|
||||
"https://youtu.be/dfnsdajlhkjhsd"),
|
||||
objects.Source(objects.SourcePages.MUSIFY,
|
||||
"https://ln.topdf.de/Music-Kraken/")
|
||||
],
|
||||
album_list=[
|
||||
objects.Album(
|
||||
title="One Final Action",
|
||||
date=objects.ID3Timestamp(year=1986, month=3, day=1),
|
||||
language=pycountry.languages.get(alpha_2="en"),
|
||||
label_list=[
|
||||
objects.Label(name="an album label")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/albums/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
]
|
||||
),
|
||||
],
|
||||
main_artist_list=[
|
||||
objects.Artist(
|
||||
name="I'm in a coffin",
|
||||
source_list=[
|
||||
objects.Source(
|
||||
objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/bands/I%27m_in_a_Coffin/127727"
|
||||
)
|
||||
]
|
||||
),
|
||||
objects.Artist(name="some_split_artist")
|
||||
],
|
||||
feature_artist_list=[
|
||||
objects.Artist(
|
||||
name="Ruffiction",
|
||||
label_list=[
|
||||
objects.Label(name="Ruffiction Productions")
|
||||
]
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
def test_song_genre(self):
|
||||
self.assertEqual(self.song.genre, "HS Core")
|
||||
|
||||
def test_song_title(self):
|
||||
self.assertEqual(self.song.title, "Vein Deep in the Solution")
|
||||
|
||||
def test_song_length(self):
|
||||
self.assertEqual(self.song.length, 666)
|
||||
|
||||
def test_song_isrc(self):
|
||||
self.assertEqual(self.song.isrc, "US-S1Z-99-00001")
|
||||
|
||||
def test_song_tracksort(self):
|
||||
self.assertEqual(self.song.tracksort, 2)
|
||||
|
||||
def test_song_target(self):
|
||||
self.assertEqual(self.song.target_collection[0].file_path, Path("example", "song.mp3"))
|
||||
|
||||
def test_song_lyrics(self):
|
||||
self.assertEqual(len(self.song.lyrics_collection), 2)
|
||||
# the other stuff will be tested in the Lyrics test
|
||||
|
||||
def test_song_source(self):
|
||||
self.assertEqual(len(self.song.source_collection), 2)
|
||||
# again the other stuff will be tested in dedicaded stuff
|
||||
|
||||
|
||||
class TestAlbum(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.album = objects.Album(
|
||||
title="One Final Action",
|
||||
date=objects.ID3Timestamp(year=1986, month=3, day=1),
|
||||
language=pycountry.languages.get(alpha_2="en"),
|
||||
label_list=[
|
||||
objects.Label(name="an album label")
|
||||
],
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/albums/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
]
|
||||
)
|
||||
|
||||
def test_album_title(self):
|
||||
self.assertEqual(self.album.title, "One Final Action")
|
||||
|
||||
def test_album_date(self):
|
||||
self.assertEqual(self.album.date.year, 1986)
|
||||
self.assertEqual(self.album.date.month, 3)
|
||||
self.assertEqual(self.album.date.day, 1)
|
||||
|
||||
def test_album_language(self):
|
||||
self.assertEqual(self.album.language.alpha_2, "en")
|
||||
|
||||
def test_album_label(self):
|
||||
self.assertEqual(self.album.label_collection[0].name, "an album label")
|
||||
|
||||
def test_album_source(self):
|
||||
sp = self.album.source_collection.get_sources_from_page(objects.SourcePages.ENCYCLOPAEDIA_METALLUM)[0]
|
||||
|
||||
self.assertEqual(
|
||||
sp.page_enum, objects.SourcePages.ENCYCLOPAEDIA_METALLUM)
|
||||
self.assertEqual(
|
||||
sp.url, "https://www.metal-archives.com/albums/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
|
||||
|
||||
class TestCollection(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.song_list: objects.song = [
|
||||
objects.Song(title="hasskrank"),
|
||||
objects.Song(title="HaSSkrank"),
|
||||
objects.Song(title="Suicideseason", isrc="uniqueID"),
|
||||
objects.Song(title="same isrc different title", isrc="uniqueID")
|
||||
]
|
||||
self.unified_titels = set(song.unified_title for song in self.song_list)
|
||||
|
||||
self.collection = objects.Collection(
|
||||
element_type=objects.Song,
|
||||
data=self.song_list
|
||||
)
|
||||
|
||||
def test_length(self):
|
||||
# hasskrank gets merged into HaSSkrank
|
||||
self.assertEqual(len(self.collection), 2)
|
||||
|
||||
def test_data(self):
|
||||
"""
|
||||
tests if the every unified name existed
|
||||
"""
|
||||
song: objects.Song
|
||||
for song in self.collection:
|
||||
self.assertIn(song.unified_title, self.unified_titels)
|
||||
|
||||
def test_appending(self):
|
||||
collection = objects.Collection(
|
||||
element_type=objects.Song
|
||||
)
|
||||
|
||||
res = collection.append(self.song_list[0])
|
||||
self.assertEqual(res.was_in_collection, False)
|
||||
self.assertEqual(res.current_element, self.song_list[0])
|
||||
|
||||
res = collection.append(self.song_list[1])
|
||||
self.assertEqual(res.was_in_collection, True)
|
||||
self.assertEqual(res.current_element, self.song_list[0])
|
||||
|
||||
res = collection.append(self.song_list[2])
|
||||
self.assertEqual(res.was_in_collection, False)
|
||||
self.assertEqual(res.current_element, self.song_list[2])
|
||||
|
||||
res = collection.append(self.song_list[3], merge_into_existing=False)
|
||||
self.assertEqual(res.was_in_collection, True)
|
||||
self.assertEqual(res.current_element, self.song_list[3])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class TestLyrics(unittest.TestCase):
|
||||
"""
|
||||
TODO
|
||||
I NEED TO REWRITE LYRICS TAKING FORMATTED TEXT INSTEAD OF JUST STRINGS
|
||||
"""
|
||||
|
||||
def setUp(self):
|
||||
self.lyrics = objects.Lyrics(
|
||||
text="these are some depressive lyrics",
|
||||
language=pycountry.languages.get(alpha_2="en"),
|
||||
source_list=[
|
||||
objects.Source(objects.SourcePages.ENCYCLOPAEDIA_METALLUM,
|
||||
"https://www.metal-archives.com/lyrics/I%27m_in_a_Coffin/One_Final_Action/207614"),
|
||||
objects.Source(objects.SourcePages.MUSIFY,
|
||||
"https://www.musify.com/lyrics/I%27m_in_a_Coffin/One_Final_Action/207614")
|
||||
]
|
||||
)
|
||||
|
||||
def test_lyrics_text(self):
|
||||
self.assertEqual(self.lyrics.text, "these are some depressive lyrics")
|
||||
|
||||
def test_lyrics_language(self):
|
||||
self.assertEqual(self.lyrics.language.alpha_2, "en")
|
||||
|
||||
def test_lyrics_source(self):
|
||||
self.assertEqual(len(self.lyrics.source_collection), 2)
|
||||
|
||||
|
||||
class TestMetadata(unittest.TestCase):
|
||||
|
||||
|
||||
def setUp(self):
|
||||
self.title = "some title"
|
||||
|
||||
self.song = objects.Song(
|
||||
title=self.title
|
||||
)
|
||||
|
||||
def test_song_metadata(self):
|
||||
self.assertEqual(self.song.metadata[objects.ID3Mapping.TITLE], id3.Frames[objects.ID3Mapping.TITLE.value](encoding=3, text=self.title))
|
||||
Loading…
Reference in New Issue
Block a user