From 3cba909c05dad1bb04b06d72d469fb71071bec9f Mon Sep 17 00:00:00 2001 From: Hellow <74311245+HeIIow2@users.noreply.github.com> Date: Fri, 16 Jun 2023 23:27:08 +0200 Subject: [PATCH] made streaming more stable --- documentation/html/youtube/channel_api.md | 471 ++++++++++++++++++ .../html/youtube/channel_api_request.md | 204 ++++++++ src/actual_donwload.py | 4 +- src/music_kraken/audio/codec.py | 4 +- src/music_kraken/connection/connection.py | 93 ++-- src/music_kraken/pages/youtube.py | 3 - src/music_kraken/utils/config/audio.py | 2 +- .../utils/support_classes/download_result.py | 10 +- 8 files changed, 738 insertions(+), 53 deletions(-) create mode 100644 documentation/html/youtube/channel_api.md create mode 100644 documentation/html/youtube/channel_api_request.md diff --git a/documentation/html/youtube/channel_api.md b/documentation/html/youtube/channel_api.md new file mode 100644 index 0000000..a51f1b8 --- /dev/null +++ b/documentation/html/youtube/channel_api.md @@ -0,0 +1,471 @@ +# Zombiez + + +https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false + +```json +{ + "responseContext": { + ... + }, + "contents": { + "twoColumnBrowseResultsRenderer": { + "tabs": [ + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CBAQ8JMBGAUiEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/featured", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCV0Ntl3lVR7xDXKoCU6uUXA", + "params": "EghmZWF0dXJlZPIGBAoCMgA%3D", + "canonicalBaseUrl": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + }, + "title": "Home", + "selected": true, + "content": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + { + "shelfRenderer": { + "title": { + "runs": [ + { + "text": "Albums & Singles", + "navigationEndpoint": { + "clickTrackingParams": "CBMQ3BwYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/playlists?view=50&sort=dd&shelf_id=17666223384013636040", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCVQK_XpXPuE8xcIpWp_JtlA", + "params": "EglwbGF5bGlzdHMYAyAycMiD1PaWksKV9QHyBgkKB0IAogECCAE%3D", + "canonicalBaseUrl": "/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + } + } + ] + }, + "endpoint": { + ... + }, + "content": { + "horizontalListRenderer": { + "items": [ + + /* first playlist */ + { + "gridPlaylistRenderer": { + "playlistId": "OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/-OPkZtrBlVM/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLASssT7fLkqB10cML8k_Tv5qoF4hg", + "width": 480, + "height": 270 + } + ], + "sampledThumbnailColor": { + "red": 62, + "green": 21, + "blue": 114 + } + }, + "title": { + "runs": [ + { + "text": "KATHARZIZ", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiDIGZy1oaWdoWhhVQ1YwTnRsM2xWUjd4RFhLb0NVNnVVWEGaAQYQ8jgYlwE=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=-OPkZtrBlVM&list=OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832 + } + }, + "watchEndpoint": { + "videoId": "-OPkZtrBlVM", + "playlistId": "OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "params": "OAI%3D", + "loggingContext": { + "vssLoggingContext": { + "serializedContextData": "GilPTEFLNXV5X25idlFlc2tyOG5iSXV6ZUx4b2NlTkx1Q0xfS2pBbXpWdw%3D%3D" + } + }, + "watchEndpointSupportedOnesieConfig": { + "html5PlaybackOnesieConfig": { + "commonConfig": { + "url": "https://rr6---sn-8xgn5uxa-4g5l.googlevideo.com/initplayback?source=youtube&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&beids=24350018&msp=1&odepv=1&id=f8e3e466dac19553&ip=87.123.241.91&initcwndbps=2165000&mt=1686834329&oweuc=" + } + } + } + } + } + } + ] + }, + "shortBylineText": { + ... + }, + "videoCountText": { + "runs": [ + { + "text": "16" + }, + { + "text": " videos" + } + ] + }, + "navigationEndpoint": { + ... + }, + "publishedTimeText": { + "simpleText": "Updated today" + }, + "videoCountShortText": { + "simpleText": "16" + }, + "trackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "sidebarThumbnails": [ + ... + ], + "thumbnailText": { + "runs": [ + { + "text": "16", + "bold": true + }, + { + "text": " videos" + } + ] + }, + "longBylineText": { + "runs": [ + { + "text": "Album" + }, + { + "text": " ยท " + }, + { + "text": "ZOMBIEZ", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiA==", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCVQK_XpXPuE8xcIpWp_JtlA", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCVQK_XpXPuE8xcIpWp_JtlA", + "canonicalBaseUrl": "/channel/UCVQK_XpXPuE8xcIpWp_JtlA" + } + } + } + ] + }, + "thumbnailOverlays": [ + ... + ], + "viewPlaylistText": { + "runs": [ + { + "text": "View full playlist", + "navigationEndpoint": { + "clickTrackingParams": "CCIQljUYACITCP_c2oWtxf8CFULHEQgdXmwFiDIGZy1oaWdo", + "commandMetadata": { + "webCommandMetadata": { + "url": "/playlist?list=OLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw", + "webPageType": "WEB_PAGE_TYPE_PLAYLIST", + "rootVe": 5754, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "VLOLAK5uy_nbvQeskr8nbIuzeLxoceNLuCL_KjAmzVw" + } + } + } + ] + } + } + }, + + /* next playlist (same format as first one) */ + { + ... + }, + + /* many more playlists */ + ... + + + ], + ... + } + }, + "trackingParams": "CBMQ3BwYACITCP_c2oWtxf8CFULHEQgdXmwFiA==" + } + } + ], + "trackingParams": "CBIQuy8YACITCP_c2oWtxf8CFULHEQgdXmwFiA==" + } + } + ], + "trackingParams": "CBEQui8iEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "targetId": "browse-feedUCV0Ntl3lVR7xDXKoCU6uUXA", + "disablePullToRefresh": true + } + }, + "trackingParams": "CBAQ8JMBGAUiEwj_3NqFrcX_AhVCxxEIHV5sBYg=" + } + }, + ... + ] + } + }, + + /* Some probably irrelevant stuff */ + + "header": { + ... + }, + "metadata": { + "channelMetadataRenderer": { + "title": "ZOMBIEZ - Topic", + "description": "", + "rssUrl": "https://www.youtube.com/feeds/videos.xml?channel_id=UCV0Ntl3lVR7xDXKoCU6uUXA", + "externalId": "UCV0Ntl3lVR7xDXKoCU6uUXA", + "keywords": "", + "ownerUrls": [ + "http://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + ], + "avatar": { + "thumbnails": [ + { + "url": "https://yt3.googleusercontent.com/nGpgytnZ-gQ8bcmUvqQ4UQruCRqWiQiujnAaHD6jSlLMMPGbl2J7V-j8iD1ZGj2iUEylvsFkNQ=s900-c-k-c0x00ffffff-no-rj", + "width": 900, + "height": 900 + } + ] + }, + "channelUrl": "https://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "isFamilySafe": true, + "availableCountryCodes": [ + ], + "musicArtistName": "ZOMBIEZ", + "androidDeepLink": "android-app://com.google.android.youtube/http/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "androidAppindexingLink": "android-app://com.google.android.youtube/http/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "iosAppindexingLink": "ios-app://544007664/vnd.youtube/www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA", + "vanityChannelUrl": "http://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA" + } + }, + "trackingParams": "CAAQhGciEwj_3NqFrcX_AhVCxxEIHV5sBYg=", + "topbar": { + ... + }, + "microformat": { + .... + } +} +``` + +# Eminem + +https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false + +```json +{ + "responseContext": { + ... + }, + "contents": { + "twoColumnBrowseResultsRenderer": { + "tabs": [ + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CDEQ8JMBGAUiEwiSlomurMX_AhWhiTgKHeKqA0o=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCedvOgsKFzcK3hA5taf3KoQ/featured", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCedvOgsKFzcK3hA5taf3KoQ", + "params": "EghmZWF0dXJlZPIGBAoCMgA%3D", + "canonicalBaseUrl": "/channel/UCedvOgsKFzcK3hA5taf3KoQ" + } + }, + "title": "Home", + "trackingParams": "CDEQ8JMBGAUiEwiSlomurMX_AhWhiTgKHeKqA0o=" + } + }, + { + "tabRenderer": { + "endpoint": { + "clickTrackingParams": "CA8Q8JMBGAYiEwiSlomurMX_AhWhiTgKHeKqA0o=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/channel/UCedvOgsKFzcK3hA5taf3KoQ/playlists", + "webPageType": "WEB_PAGE_TYPE_CHANNEL", + "rootVe": 3611, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "UCedvOgsKFzcK3hA5taf3KoQ", + "params": "EglwbGF5bGlzdHPyBgQKAkIA", + "canonicalBaseUrl": "/channel/UCedvOgsKFzcK3hA5taf3KoQ" + } + }, + "title": "Playlists", + "selected": true, + "content": { + "sectionListRenderer": { + "contents": [ + { + "itemSectionRenderer": { + "contents": [ + { + "gridRenderer": { + "items": [ + { + "gridPlaylistRenderer": { + "playlistId": "OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "thumbnail": { + "thumbnails": [ + { + "url": "https://i.ytimg.com/vi/r_0JjYUe5jo/hqdefault.jpg?sqp=-oaymwEXCOADEI4CSFryq4qpAwkIARUAAIhCGAE=&rs=AOn4CLBDE5zUUu7bOlXU1uOhaNxJl5zIbQ", + "width": 480, + "height": 270 + } + ], + "sampledThumbnailColor": { + "red": 74, + "green": 89, + "blue": 88 + } + }, + "title": { + "runs": [ + { + "text": "Curtain Call 2", + "navigationEndpoint": { + "clickTrackingParams": "CDAQljUYACITCJKWia6sxf8CFaGJOAod4qoDSjIGZy1oaWdoWhhVQ2Vkdk9nc0tGemNLM2hBNXRhZjNLb1GaAQYQ8jgYlwE=", + "commandMetadata": { + "webCommandMetadata": { + "url": "/watch?v=r_0JjYUe5jo&list=OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "webPageType": "WEB_PAGE_TYPE_WATCH", + "rootVe": 3832 + } + }, + "watchEndpoint": { + "videoId": "r_0JjYUe5jo", + "playlistId": "OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "params": "OAI%3D", + "loggingContext": { + "vssLoggingContext": { + "serializedContextData": "GilPTEFLNXV5X21IbVF0QW9qZE80UFJNeGh5R185RlIyY1NjcTUyWl93dw%3D%3D" + } + }, + "watchEndpointSupportedOnesieConfig": { + "html5PlaybackOnesieConfig": { + "commonConfig": { + "url": "https://rr2---sn-8xgn5uxa-4g5s.googlevideo.com/initplayback?source=youtube&oeis=1&c=WEB&oad=3200&ovd=3200&oaad=11000&oavd=11000&ocs=700&oewis=1&oputc=1&ofpcc=1&beids=24350018&msp=1&odepv=1&id=affd098d851ee63a&ip=87.123.241.91&initcwndbps=1772500&mt=1686834084&oweuc=" + } + } + } + } + } + } + ] + }, + "shortBylineText": { + "runs": [ + + ] + }, + "videoCountText": { + "runs": [ + { + "text": "35" + }, + { + "text": " videos" + } + ] + }, + "navigationEndpoint": { + ... + } + }, + "publishedTimeText": { + "simpleText": "Updated today" + }, + "videoCountShortText": { + "simpleText": "35" + }, + ... + ], + "thumbnailText": { + "runs": [ + { + "text": "35", + "bold": true + }, + { + "text": " videos" + } + ] + }, + ... + "viewPlaylistText": { + "runs": [ + { + "text": "View full playlist", + "navigationEndpoint": { + "clickTrackingParams": "CDAQljUYACITCJKWia6sxf8CFaGJOAod4qoDSjIGZy1oaWdo", + "commandMetadata": { + "webCommandMetadata": { + "url": "/playlist?list=OLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww", + "webPageType": "WEB_PAGE_TYPE_PLAYLIST", + "rootVe": 5754, + "apiUrl": "/youtubei/v1/browse" + } + }, + "browseEndpoint": { + "browseId": "VLOLAK5uy_mHmQtAojdO4PRMxhyG_9FR2cScq52Z_ww" + } + } + } + ] + } + } + }, +``` diff --git a/documentation/html/youtube/channel_api_request.md b/documentation/html/youtube/channel_api_request.md new file mode 100644 index 0000000..a2f3ffa --- /dev/null +++ b/documentation/html/youtube/channel_api_request.md @@ -0,0 +1,204 @@ +# Results + +## Request + +```curl +curl 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false' \ + -H 'authority: www.youtube.com' \ + -H 'accept: */*' \ + -H 'accept-language: en-US,en;q=0.7' \ + -H 'authorization: SAPISIDHASH 1686916263_e46803ac72f6036ba3b55e6edb0e45218c2d9222' \ + -H 'cache-control: no-cache' \ + -H 'content-type: application/json' \ + -H 'cookie: CONSENT=PENDING+359; SOCS=CAESEwgDEgk1MDQzNzQ2NDAaAmVuIAEaBgiAhceeBg; DEVICE_INFO=ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ+2kJ8GGJ+2kJ8G; HSID=ACaGQ6WFrIz4Ip9MQ; SSID=AkEQqNjJYkdquRRu6; APISID=c9DHm8ianHgc4LdY/AQMhE_0ECWj0r_orU; SAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-1PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-3PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; LOGIN_INFO=AFmmF2swRQIhANli-SWeYk6WvtIgBzsX4XMeuzlvdXMVtbfJTdhUJPRAAiAlG7PiNiEuUynQLUz5y-t7IvHPtpbmp0sU7_1B8ebQBw:QUQ3MjNmejhuWW5EZV9OVklIOTNxYXpEMXYwQWVvM1ptYTc2OWtObE9TcllyOEc4VndOR0tXVVRUOUhmT2hRbFRkNVZjeVhDRUJ0bmwxcTJTU3ZKWWhvX1hKMFdFbDVBYTdOS2tNMU41ZDI1Y3pIQ2NTVjNrdzJNVGFENDROT2IwdDBTTlV6cmh2RGFWaXpCWndGTHUxMzZBM1NGNk5FbVZn; VISITOR_INFO1_LIVE=swm2xaU6T4s; SID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6N69ULSrjryLwj4ZSvzfy6qA.; __Secure-1PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NcKwuUDKgPPeuOmwXW3uZUg.; __Secure-3PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NIh3P5codR8w_2O6jDK4PVw.; PREF=tz=Europe.Berlin&f6=40000000&f7=100; YSC=nVbKyjjk8dw; __Secure-YEC=CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D; SIDCC=AP8dLty32FdokPf_5Qg2eA8TWwebRgwg1n55TG2yQUd-mJkLvroZJj9QGMOIbE33Q2ovqK10H_uA; __Secure-1PSIDCC=AP8dLtz5c8aDk1Gw5VFhcM6EuUcQiZP50kH2XnUIx0PF4bDPWTPwLEq1LhVvadHnHwagqV8DXJI; __Secure-3PSIDCC=AP8dLtzfSXJb4bJhF8czo12Tuc90ONVeuOVXtinx8-_tXE7D4HW1XwHwDfspIBZItbz0aYCjgYDO; ST-txiubb=itct=CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw%3D&csn=MC45MTAyMTY0NjM3ODQ2MTUz&endpoint=%7B%22clickTrackingParams%22%3A%22CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw%3D%22%2C%22commandMetadata%22%3A%7B%22webCommandMetadata%22%3A%7B%22url%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%2Fplaylists%22%2C%22webPageType%22%3A%22WEB_PAGE_TYPE_CHANNEL%22%2C%22rootVe%22%3A3611%2C%22apiUrl%22%3A%22%2Fyoutubei%2Fv1%2Fbrowse%22%7D%7D%2C%22browseEndpoint%22%3A%7B%22browseId%22%3A%22UC8JxlMZ9VPddCuQGwB49fYg%22%2C%22params%22%3A%22EglwbGF5bGlzdHPyBgQKAkIA%22%2C%22canonicalBaseUrl%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%22%7D%7D; ST-plcuz1=' \ + -H 'origin: https://www.youtube.com' \ + -H 'pragma: no-cache' \ + -H 'referer: https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists' \ + -H 'sec-fetch-dest: empty' \ + -H 'sec-fetch-mode: same-origin' \ + -H 'sec-fetch-site: same-origin' \ + -H 'sec-gpc: 1' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' \ + -H 'x-goog-authuser: 0' \ + -H 'x-goog-visitor-id: CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D' \ + -H 'x-origin: https://www.youtube.com' \ + -H 'x-youtube-bootstrap-logged-in: true' \ + -H 'x-youtube-client-name: 1' \ + -H 'x-youtube-client-version: 2.20230613.01.00' \ + --data-raw '{ + "context":{ + "client":{ + "hl":"de", + "gl":"DE", + "remoteHost":"87.123.241.79", + "deviceMake":"", + "deviceModel":"", + "visitorData":"CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D", + "userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36,gzip(gfe)", + "clientName":"WEB", + "clientVersion":"2.20230613.01.00", + "osName":"X11", + "osVersion":"", + "originalUrl":"https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists", + "platform":"DESKTOP", + "clientFormFactor":"UNKNOWN_FORM_FACTOR", + "configInfo":{ + "appInstallData":"CJ2ZsaQGEKXC_hIQzLf-EhDftq8FELq0rwUQvbauBRDbr68FEPOorwUQ4tSuBRDj0f4SEMzfrgUQ7NH-EhDrk64FEI_DrwUQorSvBRDUoa8FEKqy_hIQpZmvBRC4i64FEMyu_hIQscavBRDnuq8FEN62rwUQouyuBRCCna8FEMO3_hIQ1bavBRCMt68FEJCjrwUQ7qKvBRDpw68FEInorgUQl9L-EhD4ta8FEOf3rgUQrLevBRD-ta8FEOSz_hIQ_u6uBQ%3D%3D" + }, + "userInterfaceTheme":"USER_INTERFACE_THEME_DARK", + "timeZone":"Europe/Berlin", + "browserName":"Chrome", + "browserVersion":"114.0.0.0", + "acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "deviceExperimentId":"ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ2ZsaQGGJ-2kJ8G", + "screenWidthPoints":1090, + "screenHeightPoints":980, + "screenPixelDensity":1, + "screenDensityFloat":1, + "utcOffsetMinutes":120, + "memoryTotalKbytes":"500000", + "mainAppWebInfo":{ + "graftUrl":"/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists", + "pwaInstallabilityStatus":"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED", + "webDisplayMode":"WEB_DISPLAY_MODE_BROWSER", + "isWebNativeShareAvailable":false + } + }, + "user":{ + "lockedSafetyMode":false + }, + "request":{ + "useSsl":true, + "internalExperimentFlags":[ + + ], + "consistencyTokenJars":[ + + ] + }, + "clickTracking":{ + "clickTrackingParams":"CCgQ8JMBGAciEwjq7bzg3Mf_AhWjRHoFHezzDzw=" + }, + "adSignalsInfo":{ + "params":[{"key":"dt","value":"1686916254647"},{"key":"flash","value":"0"},{"key":"frm","value":"0"},{"key":"u_tz","value":"120"},{"key":"u_his","value":"14"},{"key":"u_h","value":"1080"},{"key":"u_w","value":"1920"},{"key":"u_ah","value":"1049"},{"key":"u_aw","value":"1866"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"980"},{"key":"biw","value":"1075"},{"key":"brdim","value":"1280,31,1280,31,1866,31,1866,1049,1090,980"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}] + } + }, + "browseId":"UC8JxlMZ9VPddCuQGwB49fYg", + "params":"EglwbGF5bGlzdHPyBgQKAkIA" +}' \ + --compressed +``` + + +# No Results + +## Request + +```curl +curl 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8&prettyPrint=false' \ + -H 'authority: www.youtube.com' \ + -H 'accept: */*' \ + -H 'accept-language: en-US,en;q=0.7' \ + -H 'authorization: SAPISIDHASH 1686916270_9e68abaf319d6994e84c3b64f46a2f6b91258f26' \ + -H 'cache-control: no-cache' \ + -H 'content-type: application/json' \ + -H 'cookie: CONSENT=PENDING+359; SOCS=CAESEwgDEgk1MDQzNzQ2NDAaAmVuIAEaBgiAhceeBg; DEVICE_INFO=ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ+2kJ8GGJ+2kJ8G; HSID=ACaGQ6WFrIz4Ip9MQ; SSID=AkEQqNjJYkdquRRu6; APISID=c9DHm8ianHgc4LdY/AQMhE_0ECWj0r_orU; SAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-1PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; __Secure-3PAPISID=a_vNFJhvrDylguw8/Ann5G_jsanpFtW0Pu; LOGIN_INFO=AFmmF2swRQIhANli-SWeYk6WvtIgBzsX4XMeuzlvdXMVtbfJTdhUJPRAAiAlG7PiNiEuUynQLUz5y-t7IvHPtpbmp0sU7_1B8ebQBw:QUQ3MjNmejhuWW5EZV9OVklIOTNxYXpEMXYwQWVvM1ptYTc2OWtObE9TcllyOEc4VndOR0tXVVRUOUhmT2hRbFRkNVZjeVhDRUJ0bmwxcTJTU3ZKWWhvX1hKMFdFbDVBYTdOS2tNMU41ZDI1Y3pIQ2NTVjNrdzJNVGFENDROT2IwdDBTTlV6cmh2RGFWaXpCWndGTHUxMzZBM1NGNk5FbVZn; VISITOR_INFO1_LIVE=swm2xaU6T4s; SID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6N69ULSrjryLwj4ZSvzfy6qA.; __Secure-1PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NcKwuUDKgPPeuOmwXW3uZUg.; __Secure-3PSID=WwivRnpt9CjuCydglBkAPlpoMnou9Djmu9MIggo5NnkT1-6NIh3P5codR8w_2O6jDK4PVw.; PREF=tz=Europe.Berlin&f6=40000000&f7=100; YSC=nVbKyjjk8dw; __Secure-YEC=CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D; SIDCC=AP8dLtxamg1oLrM-gw2tCtJfMUighwYfCtCfwGT6G39nrxXXhtRoBcacfSPMOQwAOIN0xyOUSLyb; __Secure-1PSIDCC=AP8dLtxZ3YHCdW5cYDDaNKO8wvAGqqy4Kp7_xZtEYQW5bryvlEuU2iuJArAJskwLMwqiCznn038; __Secure-3PSIDCC=AP8dLtywHnuXOD7p59fWeUsQXp6MpPsJgZiUgW5ydTzajVvM6w64k4u7aE5WIuB5InMlTNJzKGGK; ST-plcuz1=; ST-ximm1t=csn=MC43NjY5NTkyNzczNjg1NjUy&itct=CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o%3D&endpoint=%7B%22clickTrackingParams%22%3A%22CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o%3D%22%2C%22commandMetadata%22%3A%7B%22webCommandMetadata%22%3A%7B%22url%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%2Fplaylists%3Fview%3D1%22%2C%22webPageType%22%3A%22WEB_PAGE_TYPE_CHANNEL%22%2C%22rootVe%22%3A3611%2C%22apiUrl%22%3A%22%2Fyoutubei%2Fv1%2Fbrowse%22%7D%7D%2C%22browseEndpoint%22%3A%7B%22browseId%22%3A%22UC8JxlMZ9VPddCuQGwB49fYg%22%2C%22params%22%3A%22EglwbGF5bGlzdHMgAQ%253D%253D%22%2C%22canonicalBaseUrl%22%3A%22%2Fchannel%2FUC8JxlMZ9VPddCuQGwB49fYg%22%7D%7D' \ + -H 'origin: https://www.youtube.com' \ + -H 'pragma: no-cache' \ + -H 'referer: https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1' \ + -H 'sec-fetch-dest: empty' \ + -H 'sec-fetch-mode: same-origin' \ + -H 'sec-fetch-site: same-origin' \ + -H 'sec-gpc: 1' \ + -H 'user-agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' \ + -H 'x-goog-authuser: 0' \ + -H 'x-goog-visitor-id: CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D' \ + -H 'x-origin: https://www.youtube.com' \ + -H 'x-youtube-bootstrap-logged-in: true' \ + -H 'x-youtube-client-name: 1' \ + -H 'x-youtube-client-version: 2.20230613.01.00' \ + --data-raw '{ + "context":{ + "client":{ + "hl":"de", + "gl":"DE", + "remoteHost":"87.123.241.79", + "deviceMake":"", + "deviceModel":"", + "visitorData":"CgtuQ2lKWWg3NFhTVSidmbGkBg%3D%3D", + "userAgent":"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36,gzip(gfe)", + "clientName":"WEB", + "clientVersion":"2.20230613.01.00", + "osName":"X11", + "osVersion":"", + "originalUrl":"https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1", + "platform":"DESKTOP", + "clientFormFactor":"UNKNOWN_FORM_FACTOR", + "configInfo":{ + "appInstallData":"CJ2ZsaQGEKXC_hIQzLf-EhDftq8FELq0rwUQvbauBRDbr68FEPOorwUQ4tSuBRDj0f4SEMzfrgUQ7NH-EhDrk64FEI_DrwUQorSvBRDUoa8FEKqy_hIQpZmvBRC4i64FEMyu_hIQscavBRDnuq8FEN62rwUQouyuBRCCna8FEMO3_hIQ1bavBRCMt68FEJCjrwUQ7qKvBRDpw68FEInorgUQl9L-EhD4ta8FEOf3rgUQrLevBRD-ta8FEOSz_hIQ_u6uBQ%3D%3D" + }, + "userInterfaceTheme":"USER_INTERFACE_THEME_DARK", + "timeZone":"Europe/Berlin", + "browserName":"Chrome", + "browserVersion":"114.0.0.0", + "acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8", + "deviceExperimentId":"ChxOekU1Tnprd056a3lOalUxTURJd01EQTNOdz09EJ2ZsaQGGJ-2kJ8G", + "screenWidthPoints":1090, + "screenHeightPoints":980, + "screenPixelDensity":1, + "screenDensityFloat":1, + "utcOffsetMinutes":120, + "memoryTotalKbytes":"500000", + "mainAppWebInfo":{ + "graftUrl":"/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1", + "pwaInstallabilityStatus":"PWA_INSTALLABILITY_STATUS_CAN_BE_INSTALLED", + "webDisplayMode":"WEB_DISPLAY_MODE_BROWSER", + "isWebNativeShareAvailable":false + } + }, + "user":{ + "lockedSafetyMode":false + }, + "request":{ + "useSsl":true, + "internalExperimentFlags":[ + + ], + "consistencyTokenJars":[ + + ] + }, + "clickTracking":{ + "clickTrackingParams":"CCkQui8iEwi3hYnl3Mf_AhU7zBEIHQ_pA8o=" + }, + "adSignalsInfo":{ + "params":[{"key":"dt","value":"1686916254647"},{"key":"flash","value":"0"},{"key":"frm","value":"0"},{"key":"u_tz","value":"120"},{"key":"u_his","value":"15"},{"key":"u_h","value":"1080"},{"key":"u_w","value":"1920"},{"key":"u_ah","value":"1049"},{"key":"u_aw","value":"1866"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"980"},{"key":"biw","value":"1075"},{"key":"brdim","value":"1280,31,1280,31,1866,31,1866,1049,1090,980"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}] + } + }, + "browseId":"UC8JxlMZ9VPddCuQGwB49fYg", + "params":"EglwbGF5bGlzdHMgAQ%3D%3D" +}' \ + --compressed + ``` + +# Potentially relevant difference + +difference | Result | No Result +referer | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1 +`"originalURL"` | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists | https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists?view=1 + +## What I think the problem is + +I think the problem arises, because the request is made in the wrong manner. Refering to the by the endpoint returned data from earlier, the data is given in pretty much the same format. + +Now the thing is, that there are some Topic Channels, where you can't switch the playlist type. + +- https://www.youtube.com/channel/UCPOUrPpYMpxQU_gKtBQZroQ/playlists +- https://www.youtube.com/channel/UCV0Ntl3lVR7xDXKoCU6uUXA/playlists + +But then there are those Topic Channels, where you actually can switch the playlist type. These are usually the bigger ones. I haven't found why that difference exists though. + +- https://www.youtube.com/channel/UC8JxlMZ9VPddCuQGwB49fYg/playlists +- https://www.youtube.com/channel/UCedvOgsKFzcK3hA5taf3KoQ/playlists + +So modifying the requests from one of those channels, that currently do yield results, to have the `?view=1` parameter in the url, they still yield results. So my bet right now would be, that the reasons some channels do yield results and some don't is, because invidious makes the requests with `?view=1`, or does something simmilar, which causes the same behaviour. diff --git a/src/actual_donwload.py b/src/actual_donwload.py index 360fbea..9e8bd89 100644 --- a/src/actual_donwload.py +++ b/src/actual_donwload.py @@ -15,7 +15,7 @@ if __name__ == "__main__": youtube_search = [ "s: #a Zombiez", "10", - "8" + "d: 5" ] - music_kraken.cli(genre="test", command_list=direct_download) + music_kraken.cli(genre="test", command_list=youtube_search) diff --git a/src/music_kraken/audio/codec.py b/src/music_kraken/audio/codec.py index f8ba952..42518a9 100644 --- a/src/music_kraken/audio/codec.py +++ b/src/music_kraken/audio/codec.py @@ -45,9 +45,9 @@ def correct_codec(target: Target, bitrate_kb: int = BITRATE, audio_format: str = # run the ffmpeg command with a progressbar ff = FfmpegProgress(ffmpeg_command) - with tqdm(total=100, desc="ffmpeg") as pbar: + with tqdm(total=100, desc=f"removing {len(interval_list)} segments") as pbar: for progress in ff.run_command_with_progress(): - pbar.update(int(progress)-pbar.n) + pbar.update(progress-pbar.n) LOGGER.debug(ff.stderr) diff --git a/src/music_kraken/connection/connection.py b/src/music_kraken/connection/connection.py index 81dc21a..cd739c4 100644 --- a/src/music_kraken/connection/connection.py +++ b/src/music_kraken/connection/connection.py @@ -17,7 +17,7 @@ class Connection: self, host: str, proxies: List[dict] = None, - tries: int = (len(PROXIES_LIST) + 1) * 2, + tries: int = (len(PROXIES_LIST) + 1) * 4, timeout: int = 7, logger: logging.Logger = logging.getLogger("connection"), header_values: Dict[str, str] = None, @@ -80,7 +80,7 @@ class Connection: self, request: Callable, try_count: int, - accepted_response_code: set, + accepted_response_codes: set, url: str, timeout: float, headers: dict, @@ -109,7 +109,7 @@ class Connection: try: r: requests.Response = request(request_url, timeout=timeout, headers=headers, **kwargs) - if r.status_code in accepted_response_code: + if r.status_code in accepted_response_codes: return r if self.SEMANTIC_NOT_FOUND and r.status_code == 404: @@ -136,7 +136,7 @@ class Connection: return self._request( request=request, try_count=try_count+1, - accepted_response_code=accepted_response_code, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -154,10 +154,13 @@ class Connection: raw_url: bool = False, **kwargs ) -> Optional[requests.Response]: + if accepted_response_codes is None: + accepted_response_codes = self.ACCEPTED_RESPONSE_CODES + r = self._request( request=self.session.get, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -185,7 +188,7 @@ class Connection: r = self._request( request=self.session.post, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, url=url, timeout=timeout, headers=headers, @@ -215,11 +218,19 @@ class Connection: progress: int = 0, **kwargs ) -> DownloadResult: + + if progress > 0: + if headers is None: + headers = dict() + headers["Range"] = f"bytes={target.size}-" + + if accepted_response_codes is None: + accepted_response_codes = self.ACCEPTED_RESPONSE_CODES r = self._request( request=self.session.get, try_count=0, - accepted_response_code=accepted_response_codes or self.ACCEPTED_RESPONSE_CODES, + accepted_response_codes=accepted_response_codes, url=url, timeout=timeout, headers=headers, @@ -238,45 +249,47 @@ class Connection: retry = False - with target.open("wb") as f: - try: - """ - https://en.wikipedia.org/wiki/Kilobyte - > The internationally recommended unit symbol for the kilobyte is kB. - """ + with target.open("ab") as f: + """ + https://en.wikipedia.org/wiki/Kilobyte + > The internationally recommended unit symbol for the kilobyte is kB. + """ - with tqdm(total=total_size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: - + with tqdm(total=total_size-target.size, unit='B', unit_scale=True, unit_divisor=1024, desc=description) as t: + try: for chunk in r.iter_content(chunk_size=chunk_size): size = f.write(chunk) progress += size t.update(size) - return True - except requests.exceptions.ConnectionError: - if try_count >= self.TRIES: - self.LOGGER.warning(f"Stream timed out at \"{url}\": to many retries, aborting.") - return DownloadResult(error_message=f"Stream timed out from {url}, reducing the chunksize might help.") - - self.LOGGER.warning(f"Stream timed out at \"{url}\": ({try_count}-{self.TRIES})") + except requests.exceptions.ConnectionError: + if try_count >= self.TRIES: + self.LOGGER.warning(f"Stream timed out at \"{url}\": to many retries, aborting.") + return DownloadResult(error_message=f"Stream timed out from {url}, reducing the chunksize might help.") + + self.LOGGER.warning(f"Stream timed out at \"{url}\": ({try_count}-{self.TRIES})") + retry = True + + if total_size > progress: retry = True - finally: - if total_size > progress or retry: - return self.stream_into( - url = url, - target = target, - description = description, - try_count=try_count+1, - progress=progress, - accepted_response_code=accepted_response_codes, - timeout=timeout, - headers=headers, - raw_url=raw_url, - refer_from_origin=refer_from_origin, - chunk_size=chunk_size, - **kwargs - ) - - return DownloadResult() + if retry: + self.LOGGER.warning(f"Retrying stream...") + accepted_response_codes.add(206) + return self.stream_into( + url = url, + target = target, + description = description, + try_count=try_count+1, + progress=progress, + accepted_response_codes=accepted_response_codes, + timeout=timeout, + headers=headers, + raw_url=raw_url, + refer_from_origin=refer_from_origin, + chunk_size=chunk_size, + **kwargs + ) + + return DownloadResult() diff --git a/src/music_kraken/pages/youtube.py b/src/music_kraken/pages/youtube.py index 797480e..40c62b5 100644 --- a/src/music_kraken/pages/youtube.py +++ b/src/music_kraken/pages/youtube.py @@ -394,7 +394,4 @@ class YouTube(Page): except HTTPException as e: self.LOGGER.warning(f"{e}") - if len(segments) > 0: - print(f"Removing {len(segments)} interruptions in the audio...") - return [(segment.start, segment.end) for segment in segments] diff --git a/src/music_kraken/utils/config/audio.py b/src/music_kraken/utils/config/audio.py index 969bb67..ea85d84 100644 --- a/src/music_kraken/utils/config/audio.py +++ b/src/music_kraken/utils/config/audio.py @@ -106,7 +106,7 @@ ID3.1: {', '.join(_sorted_id3_1_formats)} self.DOWNLOAD_PATH = StringAttribute( name="download_path", - value="{genre}/{artist}/{album_type}/{album}", + value="{genre}/{artist}/{album}", description="The folder music kraken should put the songs into." ) diff --git a/src/music_kraken/utils/support_classes/download_result.py b/src/music_kraken/utils/support_classes/download_result.py index a47a054..2f54111 100644 --- a/src/music_kraken/utils/support_classes/download_result.py +++ b/src/music_kraken/utils/support_classes/download_result.py @@ -80,11 +80,11 @@ class DownloadResult: def __str__(self): if self.is_fatal_error: return self.error_message - head = f"{self.fail} from {self.total} downloads failed:\n" \ - f"successrate:\t{int(self.success_percentage * 100)}%\n" \ - f"failrate:\t{int(self.failure_percentage * 100)}%\n" \ - f"total size:\t{self.formated_size}\n" \ - f"skipped segments:\t{self.sponsor_segments}" \ + head = f"{self.fail} from {self.total} downloads failed:\n" \ + f"successrate:\t{int(self.success_percentage * 100)}%\n" \ + f"failrate:\t{int(self.failure_percentage * 100)}%\n" \ + f"total size:\t{self.formated_size}\n" \ + f"skipped segments:\t{self.sponsor_segments}\n" \ f"found on disc:\t{self.found_on_disk}" if not self.is_mild_failure: