From c6d0acff9bb3219f96fa368fef9e3c6b71188e56 Mon Sep 17 00:00:00 2001 From: lars Date: Tue, 15 Nov 2022 00:27:03 +0100 Subject: [PATCH] restructured slightly and addet skript to upload to PyPI --- .gitignore | 4 +- README.md | 2 +- assets/database_structure.sql | 1 + src/build.sh | 17 ++ src/{ => music_kraken}/__init__.py | 0 src/{ => music_kraken}/__main__.py | 0 src/{ => music_kraken}/audio/__init__.py | 0 src/{ => music_kraken}/audio/song.py | 0 src/{ => music_kraken}/download.py | 0 src/{ => music_kraken}/download_links.py | 0 src/{ => music_kraken}/lyrics/__init__.py | 0 src/{ => music_kraken}/lyrics/genius.py | 0 src/{ => music_kraken}/lyrics/lyrics.py | 0 src/{ => music_kraken}/metadata/__init__.py | 0 src/{ => music_kraken}/metadata/database.py | 0 src/{ => music_kraken}/metadata/download.py | 0 .../metadata/object_handeling.py | 0 src/{ => music_kraken}/metadata/search.py | 0 .../music_kraken.egg-info/PKG-INFO | 181 ++++++++++++++++++ .../music_kraken.egg-info/SOURCES.txt | 32 ++++ .../dependency_links.txt | 1 + .../music_kraken.egg-info/requires.txt | 8 + .../music_kraken.egg-info/top_level.txt | 10 + src/{ => music_kraken}/scraping/__init__.py | 0 .../scraping/file_system.py | 0 src/{ => music_kraken}/scraping/musify.py | 0 .../scraping/phonetic_compares.py | 0 .../scraping/youtube_music.py | 0 src/{ => music_kraken}/url_to_path.py | 0 src/{ => music_kraken}/utils/__init__.py | 0 .../utils/object_handeling.py | 0 .../utils/phonetic_compares.py | 0 src/{ => music_kraken}/utils/shared.py | 0 src/pyproject.toml | 20 ++ src/setup.py | 13 ++ src/try-programming-interface.py | 4 + 36 files changed, 291 insertions(+), 2 deletions(-) create mode 100644 src/build.sh rename src/{ => music_kraken}/__init__.py (100%) rename src/{ => music_kraken}/__main__.py (100%) rename src/{ => music_kraken}/audio/__init__.py (100%) rename src/{ => music_kraken}/audio/song.py (100%) rename src/{ => music_kraken}/download.py (100%) rename src/{ => music_kraken}/download_links.py (100%) rename src/{ => music_kraken}/lyrics/__init__.py (100%) rename src/{ => music_kraken}/lyrics/genius.py (100%) rename src/{ => music_kraken}/lyrics/lyrics.py (100%) rename src/{ => music_kraken}/metadata/__init__.py (100%) rename src/{ => music_kraken}/metadata/database.py (100%) rename src/{ => music_kraken}/metadata/download.py (100%) rename src/{ => music_kraken}/metadata/object_handeling.py (100%) rename src/{ => music_kraken}/metadata/search.py (100%) create mode 100644 src/music_kraken/music_kraken.egg-info/PKG-INFO create mode 100644 src/music_kraken/music_kraken.egg-info/SOURCES.txt create mode 100644 src/music_kraken/music_kraken.egg-info/dependency_links.txt create mode 100644 src/music_kraken/music_kraken.egg-info/requires.txt create mode 100644 src/music_kraken/music_kraken.egg-info/top_level.txt rename src/{ => music_kraken}/scraping/__init__.py (100%) rename src/{ => music_kraken}/scraping/file_system.py (100%) rename src/{ => music_kraken}/scraping/musify.py (100%) rename src/{ => music_kraken}/scraping/phonetic_compares.py (100%) rename src/{ => music_kraken}/scraping/youtube_music.py (100%) rename src/{ => music_kraken}/url_to_path.py (100%) rename src/{ => music_kraken}/utils/__init__.py (100%) rename src/{ => music_kraken}/utils/object_handeling.py (100%) rename src/{ => music_kraken}/utils/phonetic_compares.py (100%) rename src/{ => music_kraken}/utils/shared.py (100%) create mode 100644 src/pyproject.toml create mode 100644 src/setup.py create mode 100644 src/try-programming-interface.py diff --git a/.gitignore b/.gitignore index c0df484..1cc6884 100644 --- a/.gitignore +++ b/.gitignore @@ -4,4 +4,6 @@ __pycache__/ *$py.class *.mp3 -*.cache* \ No newline at end of file +*.cache* + +/build/* \ No newline at end of file diff --git a/README.md b/README.md index 7fe9e9a..5633fed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# A Music-Downloader with CLI +# Music Kraken RUN WITH: `python3 -m src` from the project Directory diff --git a/assets/database_structure.sql b/assets/database_structure.sql index b162c88..792e4ee 100644 --- a/assets/database_structure.sql +++ b/assets/database_structure.sql @@ -46,6 +46,7 @@ CREATE TABLE track ( downloaded BOOLEAN NOT NULL DEFAULT 0, release_id TEXT NOT NULL, track TEXT, + length INT, tracknumber TEXT, isrc TEXT, genre TEXT, diff --git a/src/build.sh b/src/build.sh new file mode 100644 index 0000000..13564ae --- /dev/null +++ b/src/build.sh @@ -0,0 +1,17 @@ +# https://packaging.python.org/en/latest/tutorials/packaging-projects/ +cp ../README.md README.md + +echo "building......" +python3 -m pip install --upgrade build +python3 -m build + +echo "cleaning up......." +rm README.md +# shellcheck disable=SC2164 +cd ../build +rm -rf dist +mv ../src/dist . + +echo "uploading......." +# python3 -m pip install --upgrade twine +twine upload dist/* diff --git a/src/__init__.py b/src/music_kraken/__init__.py similarity index 100% rename from src/__init__.py rename to src/music_kraken/__init__.py diff --git a/src/__main__.py b/src/music_kraken/__main__.py similarity index 100% rename from src/__main__.py rename to src/music_kraken/__main__.py diff --git a/src/audio/__init__.py b/src/music_kraken/audio/__init__.py similarity index 100% rename from src/audio/__init__.py rename to src/music_kraken/audio/__init__.py diff --git a/src/audio/song.py b/src/music_kraken/audio/song.py similarity index 100% rename from src/audio/song.py rename to src/music_kraken/audio/song.py diff --git a/src/download.py b/src/music_kraken/download.py similarity index 100% rename from src/download.py rename to src/music_kraken/download.py diff --git a/src/download_links.py b/src/music_kraken/download_links.py similarity index 100% rename from src/download_links.py rename to src/music_kraken/download_links.py diff --git a/src/lyrics/__init__.py b/src/music_kraken/lyrics/__init__.py similarity index 100% rename from src/lyrics/__init__.py rename to src/music_kraken/lyrics/__init__.py diff --git a/src/lyrics/genius.py b/src/music_kraken/lyrics/genius.py similarity index 100% rename from src/lyrics/genius.py rename to src/music_kraken/lyrics/genius.py diff --git a/src/lyrics/lyrics.py b/src/music_kraken/lyrics/lyrics.py similarity index 100% rename from src/lyrics/lyrics.py rename to src/music_kraken/lyrics/lyrics.py diff --git a/src/metadata/__init__.py b/src/music_kraken/metadata/__init__.py similarity index 100% rename from src/metadata/__init__.py rename to src/music_kraken/metadata/__init__.py diff --git a/src/metadata/database.py b/src/music_kraken/metadata/database.py similarity index 100% rename from src/metadata/database.py rename to src/music_kraken/metadata/database.py diff --git a/src/metadata/download.py b/src/music_kraken/metadata/download.py similarity index 100% rename from src/metadata/download.py rename to src/music_kraken/metadata/download.py diff --git a/src/metadata/object_handeling.py b/src/music_kraken/metadata/object_handeling.py similarity index 100% rename from src/metadata/object_handeling.py rename to src/music_kraken/metadata/object_handeling.py diff --git a/src/metadata/search.py b/src/music_kraken/metadata/search.py similarity index 100% rename from src/metadata/search.py rename to src/music_kraken/metadata/search.py diff --git a/src/music_kraken/music_kraken.egg-info/PKG-INFO b/src/music_kraken/music_kraken.egg-info/PKG-INFO new file mode 100644 index 0000000..954c604 --- /dev/null +++ b/src/music_kraken/music_kraken.egg-info/PKG-INFO @@ -0,0 +1,181 @@ +Metadata-Version: 2.1 +Name: music-kraken +Version: 0.0.1 +Summary: An extensive music downloader crawling the internet. It gets its metadata from a couple metadata provider, and it scrapes the audiofiles. +Home-page: https://github.com/HeIIow2/music-downloader +Author: Hellow2 +Author-email: Hellow2 +Project-URL: Homepage, https://github.com/HeIIow2/music-downloader +Classifier: Programming Language :: Python :: 3 +Classifier: Operating System :: OS Independent +Requires-Python: >=3.7 +Description-Content-Type: text/markdown + +# Music Kraken + +RUN WITH: `python3 -m src` from the project Directory + +This programm will first get the metadata of various songs from metadata provider like musicbrainz, and then search for download links on pages like bandcamp. Then it will download the song and edit the metadata according. + +## Metadata + +First the metadata has to be downloaded. The best api to do so is undeniably [Musicbrainz](musicbrainz.org/). This is a result of them being a website with a large Database spanning over all Genres. + +### Musicbrainz + +![Musicbrainz Data Scheme](https://wiki.musicbrainz.org/-/images/9/9e/pymb3-model-core.png) + +To fetch from [Musicbrainz](musicbrainz.org/) we first have to know what to fetch. A good start is to get an input querry, which can be just put into the MB-Api. It then returns a list of possible artists, releases and recordings. + +Then we can output them in the Terminal and ask for further input. Following can be inputed afterwards: + +- `q` to quit +- `ok` to download +- `..` for previous options +- `.` for current options +- `an integer` for this element + +If the following chosen element is an artist, its discography + a couple tracks are outputed, if a release is chosen, the artists + tracklist + release is outputted, If a track is chosen its artists and releases are shown. + +**TO DO** + +- Schow always the whole tracklist of an release if it is chosen +- Show always the whole discography of an artist if it is chosen + +Up to now it doesn't if the discography or tracklist is chosen. + +### Metadata to fetch + +I orient on which metadata to download on the keys in `mutagen.EasyID3` . Following I fatch and thus tag the MP3 with: +- title +- artist +- albumartist +- tracknumber +- albumsort can sort albums cronological +- titlesort is just set to the tracknumber to sort by track order to sort correctly +- isrc +- musicbrainz_artistid +- musicbrainz_albumid +- musicbrainz_albumartistid +- musicbrainz_albumstatus +- language +- musicbrainz_albumtype +- releasecountry +- barcode + +#### albumsort/titlesort + +Those Tags are for the musicplayer to not sort for Example the albums of a band alphabetically, but in another way. I set it just to chronological order + +#### isrc + +This is the **international standart release code**. With this a track can be identified 100% percicely all of the time, if it is known and the website has a search api for that. Obviously this will get important later. + +--- + +## Download + +Now that the metadata is downloaded and cached, download sources need to be sound, because one can't listen to metadata. Granted it would be amazing if that would be possible. + +### Musify + +The quickest source to get download links from is to my knowledge [musify](https://musify.club/). Its a russian music downloading page, where many many songs are available to stream and to download. Due to me not wanting to stress the server to much, I abuse a handy feature nearly every page where you can search suff has. The autocomplete api for the search input. Those always are quite limited in the number of results it returns, but it is optimized to be quick. Thus with the http header `Connection` set to `keep-alive` the bottelneck defently is not at the speed of those requests. + +For musify the endpoint is following: [https://musify.club/search/suggestions?term={title}](https://musify.club/search/suggestions?term=LornaShore) If the http headers are set correctly, then searching for example for "Lorna Shore" yields following result: + +```json +[ + { + "id":"Lorna Shore", + "label":"Lorna Shore", + "value":"Lorna Shore", + "category":"Исполнители", + "image":"https://39s.musify.club/img/68/9561484/25159224.jpg", + "url":"/artist/lorna-shore-59611" + }, + {"id":"Immortal","label":"Lorna Shore - Immortal (2020)","value":"Immortal","category":"Релизы","image":"https://39s-a.musify.club/img/70/20335517/52174338.jpg","url":"/release/lorna-shore-immortal-2020-1241300"}, + {"id":"Immortal","label":"Lorna Shore - Immortal","value":"Immortal","category":"Треки","image":"","url":"/track/lorna-shore-immortal-12475071"} +] +``` + +This is a shortened example for the response the api gives. The results are very Limited, but it is also very efficient to parse. The steps I take are: + +- call the api with the querry being the track name +- parse the json response to an object +- look at how different the title and artist are on every element from the category `Треки`, translated roughly to track or release. +- If they match get the download links and cache them. + +### Youtube + +Herte the **isrc** plays a huge role. You probaply know it, when you search on youtube for a song, and the music videos has a long intro or the first result is a live version. I don't want those in my music collection, only if the tracks are like this in the official release. Well how can you get around that? + +Turns out if you search for the **isrc** on youtube the results contain the music, like it is on the official release and some japanese meme videos. The tracks I wan't just have the title of the released track, so one can just compare those two. + +For searching, as well as for downloading I use the programm `youtube-dl`, which also has a programming interface for python. + +There are two bottlenecks with this approach though: +1. `youtube-dl` is just slow. Actually it has to be, to not get blocked by youtube. +2. Ofthen musicbrainz just doesn't give the isrc for some songs. + +**TODO** +- look at how the isrc id derived an try to generate it for the tracks without directly getting it from mb. + + +**Progress** +- There is a great site whith a huge isrc database [https://isrc.soundexchange.com/](https://isrc.soundexchange.com/). + + +## Lyrics + +To get the Lyrics, I scrape them, and put those in the USLT ID3 Tags of for example mp3 files. Unfortunately some players, like the one I use, Rhythmbox don't support USLT Lyrics. So I created an Plugin for Rhythmbox. You can find it here: [https://github.com/HeIIow2/rythmbox-id3-lyrics-support](https://github.com/HeIIow2/rythmbox-id3-lyrics-support). + +### Genius + +For the lyrics source the page [https://genius.com/](https://genius.com/) is easily sufficient. It has most songs. Some songs are not present though, but that is fine, because the lyrics are optional anyways. + +## Project overview + +The file structure is as follows (might be slightly outdated): + +``` +music-downloader +├── assets +│   └── database_structure.sql +├── LICENSE +├── notes.md +├── README.md +├── requirements.txt +└── src + ├── audio + │   └── song.py + ├── download_links.py + ├── download.py + ├── lyrics + │   ├── genius.py + │   └── lyrics.py + ├── __main__.py + ├── metadata + │   ├── database.py + │   ├── download.py + │   ├── object_handeling.py + │   └── search.py + ├── scraping + │   ├── file_system.py + │   ├── musify.py + │   ├── phonetic_compares.py + │   └── youtube_music.py + ├── url_to_path.py + └── utils + ├── object_handeling.py + ├── phonetic_compares.py + └── shared.py + +``` + +You can obviously find the source code in the folder src. The two "most important" files are `__main__.py` and `utils/shared.py`. + +In the first one is the code gluing everything together and providing the cli. + +### utils + +The constants like the global database object can be found in `shared.py`. diff --git a/src/music_kraken/music_kraken.egg-info/SOURCES.txt b/src/music_kraken/music_kraken.egg-info/SOURCES.txt new file mode 100644 index 0000000..c6a17be --- /dev/null +++ b/src/music_kraken/music_kraken.egg-info/SOURCES.txt @@ -0,0 +1,32 @@ +README.md +pyproject.toml +setup.py +music_kraken/__init__.py +music_kraken/__main__.py +music_kraken/download.py +music_kraken/download_links.py +music_kraken/url_to_path.py +music_kraken/audio/__init__.py +music_kraken/audio/song.py +music_kraken/lyrics/__init__.py +music_kraken/lyrics/genius.py +music_kraken/lyrics/lyrics.py +music_kraken/metadata/__init__.py +music_kraken/metadata/database.py +music_kraken/metadata/download.py +music_kraken/metadata/object_handeling.py +music_kraken/metadata/search.py +music_kraken/music_kraken.egg-info/PKG-INFO +music_kraken/music_kraken.egg-info/SOURCES.txt +music_kraken/music_kraken.egg-info/dependency_links.txt +music_kraken/music_kraken.egg-info/requires.txt +music_kraken/music_kraken.egg-info/top_level.txt +music_kraken/scraping/__init__.py +music_kraken/scraping/file_system.py +music_kraken/scraping/musify.py +music_kraken/scraping/phonetic_compares.py +music_kraken/scraping/youtube_music.py +music_kraken/utils/__init__.py +music_kraken/utils/object_handeling.py +music_kraken/utils/phonetic_compares.py +music_kraken/utils/shared.py \ No newline at end of file diff --git a/src/music_kraken/music_kraken.egg-info/dependency_links.txt b/src/music_kraken/music_kraken.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/src/music_kraken/music_kraken.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/src/music_kraken/music_kraken.egg-info/requires.txt b/src/music_kraken/music_kraken.egg-info/requires.txt new file mode 100644 index 0000000..9a5a0b9 --- /dev/null +++ b/src/music_kraken/music_kraken.egg-info/requires.txt @@ -0,0 +1,8 @@ +requests~=2.28.1 +mutagen~=1.46.0 +musicbrainzngs~=0.7.1 +jellyfish~=0.9.0 +pydub~=0.25.1 +youtube_dl +beautifulsoup4~=4.11.1 +pycountry~=22.3.5 diff --git a/src/music_kraken/music_kraken.egg-info/top_level.txt b/src/music_kraken/music_kraken.egg-info/top_level.txt new file mode 100644 index 0000000..eaebb54 --- /dev/null +++ b/src/music_kraken/music_kraken.egg-info/top_level.txt @@ -0,0 +1,10 @@ +__init__ +__main__ +audio +download +download_links +lyrics +metadata +scraping +url_to_path +utils diff --git a/src/scraping/__init__.py b/src/music_kraken/scraping/__init__.py similarity index 100% rename from src/scraping/__init__.py rename to src/music_kraken/scraping/__init__.py diff --git a/src/scraping/file_system.py b/src/music_kraken/scraping/file_system.py similarity index 100% rename from src/scraping/file_system.py rename to src/music_kraken/scraping/file_system.py diff --git a/src/scraping/musify.py b/src/music_kraken/scraping/musify.py similarity index 100% rename from src/scraping/musify.py rename to src/music_kraken/scraping/musify.py diff --git a/src/scraping/phonetic_compares.py b/src/music_kraken/scraping/phonetic_compares.py similarity index 100% rename from src/scraping/phonetic_compares.py rename to src/music_kraken/scraping/phonetic_compares.py diff --git a/src/scraping/youtube_music.py b/src/music_kraken/scraping/youtube_music.py similarity index 100% rename from src/scraping/youtube_music.py rename to src/music_kraken/scraping/youtube_music.py diff --git a/src/url_to_path.py b/src/music_kraken/url_to_path.py similarity index 100% rename from src/url_to_path.py rename to src/music_kraken/url_to_path.py diff --git a/src/utils/__init__.py b/src/music_kraken/utils/__init__.py similarity index 100% rename from src/utils/__init__.py rename to src/music_kraken/utils/__init__.py diff --git a/src/utils/object_handeling.py b/src/music_kraken/utils/object_handeling.py similarity index 100% rename from src/utils/object_handeling.py rename to src/music_kraken/utils/object_handeling.py diff --git a/src/utils/phonetic_compares.py b/src/music_kraken/utils/phonetic_compares.py similarity index 100% rename from src/utils/phonetic_compares.py rename to src/music_kraken/utils/phonetic_compares.py diff --git a/src/utils/shared.py b/src/music_kraken/utils/shared.py similarity index 100% rename from src/utils/shared.py rename to src/music_kraken/utils/shared.py diff --git a/src/pyproject.toml b/src/pyproject.toml new file mode 100644 index 0000000..99ca2b8 --- /dev/null +++ b/src/pyproject.toml @@ -0,0 +1,20 @@ +[build-system] +requires = ["setuptools>=61.0", "requests~=2.28.1", "mutagen~=1.46.0", "musicbrainzngs~=0.7.1", "jellyfish~=0.9.0", "pydub~=0.25.1", "youtube_dl", "beautifulsoup4~=4.11.1", "pycountry~=22.3.5"] +build-backend = "setuptools.build_meta" + +[project] +name = "music_kraken" +version = "0.0.1" +authors = [ + { name="Hellow2", email="Hellow2@outlook.de" }, +] +description = "An extensive music downloader crawling the internet. It gets its metadata from a couple metadata provider, and it scrapes the audiofiles." +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://github.com/HeIIow2/music-downloader" \ No newline at end of file diff --git a/src/setup.py b/src/setup.py new file mode 100644 index 0000000..3188e14 --- /dev/null +++ b/src/setup.py @@ -0,0 +1,13 @@ +from distutils.core import setup + +setup( + name='music-kraken', + version='1.0', + description='An extensive music downloader crawling the internet. It gets its metadata from a couple metadata ' + 'provider, and it scrapes the audiofiles.', + author='Hellow2', + author_email='Hellow2@outlook.de', + url='https://github.com/HeIIow2/music-downloader', + package_dir={'': 'music_kraken'}, + install_requires=["requests~=2.28.1", "mutagen~=1.46.0", "musicbrainzngs~=0.7.1", "jellyfish~=0.9.0", "pydub~=0.25.1", "youtube_dl", "beautifulsoup4~=4.11.1", "pycountry~=22.3.5"] +) diff --git a/src/try-programming-interface.py b/src/try-programming-interface.py new file mode 100644 index 0000000..e075326 --- /dev/null +++ b/src/try-programming-interface.py @@ -0,0 +1,4 @@ +from music_kraken.__main__ import cli + +if __name__ == "__main__": + cli()