layed out api request

documentation/html/youtube-music/search/01-search-request.json

@@ -1,3 +1,4 @@
+// https://music.youtube.com/youtubei/v1/search?key=AIzaSyC9XL3ZjWddXya6X74dJoCTL-WEYFDNX30&prettyPrint=false
 // ctoken could be short for continue token 
 {
 	"POST": {
@@ -147,3 +148,64 @@
     }
   }
   
+
+-----
+
+
+{
+  "context":
+  {
+    "client":
+      {
+        "hl":"en",
+        "gl":"DE",
+        "remoteHost":"87.123.241.85",
+        "deviceMake":"",
+        "deviceModel":"",
+        "visitorData":"CgtucS1ibEdPa045ZyiT4YWmBg%3D%3D",
+        "userAgent":"Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/115.0,gzip(gfe)",
+        "clientName":"WEB_REMIX",
+        "clientVersion":"1.20230724.00.00-canary_experiment",
+        "osName":"X11","osVersion":"","originalUrl":"https://music.youtube.com/?cbrd=1","platform":"DESKTOP","clientFormFactor":"UNKNOWN_FORM_FACTOR","configInfo":
+        {"appInstallData":"CJPhhaYGEJ3b_hIQsdWvBRC41a8FEL22rgUQ3ravBRD-ta8FEOe6rwUQw7f-EhDgtq8FEKnErwUQ6sOvBRCst68FEIXZ_hIQ5LP-EhDMrv4SELiLrgUQ65OuBRCMt68FEOPO_hIQwt7-EhDbz68FELTJrwUQ8qivBRD4ta8FEJbOrwUQzN-uBRCPw68FEP24_RIQhLavBRC1pq8FEKqy_hIQksuvBRCa0a8FEMy3_hIQjMuvBRCj1K8FEKXC_hIQ_eeoGBD51a8F","coldConfigData":"CJPhhaYGEOy6rQUQ65OuBRC9tq4FEKT-rgUQ6KivBRDyqK8FEIy3rwUQ4bqvBRDDxq8FEJ7HrwUQ88yvBRDbz68FEMDQrwUQmtGvBRDK068FENTTrwUQo9SvBRCx1a8FELjVrwUQ-dWvBRDZ168FEI7YrwUQ0NmvBRoyQU53R2I4V013TDV5bTJ1S0hPZndFWFZqcFB4b0l6MVRxcllyNFo2dDdKVGRTQjFFS3ciMkFOd0diOFdNd0w1eW0ydUtIT2Z3RVhWanBQeG9JejFUcXJZcjRaNnQ3SlRkU0IxRUt3KkhDQU1TTUEwVGdwYW9Bc2dXX2dXZkJJOFNuUXEwQW9FRWxnTVZINUtDMEF5elI4bVVCdDhhdmxLQ0F0NWluUy1KSjQtNUJBPT0%3D","coldHashData":"CJPhhaYGEhM0OTUzOTkxMTAyODE4MjI5NTY3GJPhhaYGMjJBTndHYjhXTXdMNXltMnVLSE9md0VYVmpwUHhvSXoxVHFyWXI0WjZ0N0pUZFNCMUVLdzoyQU53R2I4V013TDV5bTJ1S0hPZndFWFZqcFB4b0l6MVRxcllyNFo2dDdKVGRTQjFFS3dCSENBTVNNQTBUZ3Bhb0FzZ1dfZ1dmQkk4U25RcTBBb0VFbGdNVkg1S0MwQXl6UjhtVUJ0OGF2bEtDQXQ1aW5TLUpKNC01QkE9PQ%3D%3D","hotHashData":"CJPhhaYGEhQxMjc1MzUxNTg3MDYwNDg5NzEwMRiT4YWmBiiU5PwSKNuT_RIoxrL9EiiqtP0SKJ6R_hIomq3-EiiUzf4SKN3O_hIo487-EiiF2f4SKJfZ_hIondv-EijI3P4SKNjd_hIovt7-EjIyQU53R2I4V013TDV5bTJ1S0hPZndFWFZqcFB4b0l6MVRxcllyNFo2dDdKVGRTQjFFS3c6MkFOd0diOFdNd0w1eW0ydUtIT2Z3RVhWanBQeG9JejFUcXJZcjRaNnQ3SlRkU0IxRUt3QihDQU1TR1EwUDJJXzVGY29BcURrVkNvM2l6UXlMN2dIRmtBRGgwQUk9"},
+        "browserName":"Firefox",
+        "browserVersion":"115.0",
+        "acceptHeader":"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "deviceExperimentId":"ChxOekkyTURJd056ZzBPRFF3TWpVME5EVTRNUT09EJPhhaYGGJLhhaYG",
+        "screenWidthPoints":923,
+        "screenHeightPoints":964,
+        "screenPixelDensity":1,
+        "screenDensityFloat":1,
+        "utcOffsetMinutes":120,
+        "userInterfaceTheme":"USER_INTERFACE_THEME_DARK",
+        "timeZone":"Atlantic/Jan_Mayen",
+        "musicAppInfo":{"pwaInstallabilityStatus":"PWA_INSTALLABILITY_STATUS_UNKNOWN","webDisplayMode":"WEB_DISPLAY_MODE_BROWSER","storeDigitalGoodsApiSupportStatus":{"playStoreDigitalGoodsApiSupportStatus":"DIGITAL_GOODS_API_SUPPORT_STATUS_UNSUPPORTED"}}
+      },
+      "user":{"lockedSafetyMode":false},
+      "request":{"useSsl":true,"internalExperimentFlags":[],"consistencyTokenJars":[]
+    },
+      "adSignalsInfo":{
+          "params":[
+            {"key":"dt","value":"1690398867909"},
+            {"key":"flash","value":"0"},
+            {"key":"frm","value":"0"},
+            {"key":"u_tz","value":"120"},{"key":"u_his","value":"5"},{"key":"u_h","value":"1080"},{"key":"u_w","value":"1920"},{"key":"u_ah","value":"1049"},{"key":"u_aw","value":"1866"},{"key":"u_cd","value":"24"},{"key":"bc","value":"31"},{"key":"bih","value":"964"},{"key":"biw","value":"923"},{"key":"brdim","value":"1280,31,1280,31,1866,31,1866,1049,923,964"},{"key":"vis","value":"1"},{"key":"wgl","value":"true"},{"key":"ca_type","value":"image"}
+            ]
+          }
+        },
+        "query":"psychonaut 4",
+        "suggestStats":{
+          "validationStatus":"VALID",
+          "parameterValidationStatus":"VALID_PARAMETERS",
+          "clientName":"youtube-music",
+          "searchMethod":"ENTER_KEY",
+          "inputMethod":"KEYBOARD",
+          "originalQuery":"psychonaut 4",
+          "availableSuggestions":[{"index":0,"suggestionType":0},{"index":1,"suggestionType":0},{"index":2,"suggestionType":0},{"index":3,"suggestionType":0},{"index":4,"suggestionType":0},{"index":5,"suggestionType":0},{"index":6,"suggestionType":0}],
+          "zeroPrefixEnabled":true,
+          "firstEditTimeMsec":1329258,
+          "lastEditTimeMsec":1330993
+          }
+        }
+
+

src/actual_donwload.py

@@ -28,4 +28,8 @@ if __name__ == "__main__":
         "d: 5"
     ]
 
-    music_kraken.cli.download(genre="test", command_list=download_youtube_playlist, process_metadata_anyway=True)
+    youtube_music_test = [
+        "s: psychonaut 4"
+    ]
+
+    music_kraken.cli.download(genre="test", command_list=youtube_music_test, process_metadata_anyway=True)

src/music_kraken/connection/connection.py

@@ -53,6 +53,10 @@ class Connection:
         self.hearthbeat_thread = None
         self.hearthbeat_interval = hearthbeat_interval
 
+    @property
+    def user_agent(self) -> str:
+        return self.session.headers.get("user-agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/112.0.0.0 Safari/537.36")
+
 
     def start_hearthbeat(self):
         if self.hearthbeat_interval <= 0:

src/music_kraken/pages/youtube_music.py

@@ -1,5 +1,5 @@
 from typing import Dict, List, Optional, Set, Type
-from urllib.parse import urlparse
+from urllib.parse import urlparse, urlunparse, quote
 import logging
 import random
 import json
@@ -7,8 +7,9 @@ from dataclasses import dataclass
 import re
 
 from ..utils.exception.config import SettingValueError
-from ..utils.shared import PROXIES_LIST, YOUTUBE_MUSIC_LOGGER
+from ..utils.shared import PROXIES_LIST, YOUTUBE_MUSIC_LOGGER, DEBUG
 from ..utils.config import CONNECTION_SECTION, write_config
+from ..utils.functions import get_current_millis
 
 from ..objects import Source, DatabaseObject
 from .abstract import Page
@@ -24,6 +25,11 @@ from ..objects import (
 from ..connection import Connection
 from ..utils.support_classes import DownloadResult
 
+
+def get_youtube_url(path: str = "", params: str = "", query: str = "", fragment: str = "") -> str:
+    return urlunparse(("https", "music.youtube.com", path, params, query, fragment))
+
+
 class YoutubeMusicConnection(Connection):
     """
     ===Hearthbeat=timings=for=YOUTUBEMUSIC===
@@ -72,20 +78,76 @@ class YouTubeMusicCredentials:
     # It is probably not strictly necessary, but hey :))
     ctoken: str
 
+    # the context in requests
+    context: dict
+
 
 class YoutubeMusic(Page):
     # CHANGE
-    SOURCE_TYPE = SourcePages.PRESET
+    SOURCE_TYPE = SourcePages.YOUTUBE
     LOGGER = YOUTUBE_MUSIC_LOGGER
 
     def __init__(self, *args, **kwargs):
         self.connection: YoutubeMusicConnection = YoutubeMusicConnection(logger=self.LOGGER)
         self.credentials: YouTubeMusicCredentials = YouTubeMusicCredentials(
             api_key=CONNECTION_SECTION.YOUTUBE_MUSIC_API_KEY.object_from_value,
-            ctoken=""
+            ctoken="",
+            context= {
+                "client": {
+                    "hl": "en",
+                    "gl": "DE",
+                    "remoteHost": "87.123.241.77",
+                    "deviceMake": "",
+                    "deviceModel": "",
+                    "visitorData": "CgtiTUxaTHpoXzk1Zyia59WlBg%3D%3D",
+                    "userAgent": self.connection.user_agent,
+                    "clientName": "WEB_REMIX",
+                    "clientVersion": "1.20230710.01.00",
+                    "osName": "X11",
+                    "osVersion": "",
+                    "originalUrl": "https://music.youtube.com/",
+                    "platform": "DESKTOP",
+                    "clientFormFactor": "UNKNOWN_FORM_FACTOR",
+                    "configInfo": {
+                        "appInstallData": "",
+                        "coldConfigData": "",
+                        "coldHashData": "",
+                        "hotHashData": ""
+                    },
+                    "userInterfaceTheme": "USER_INTERFACE_THEME_DARK",
+                    "timeZone": "Atlantic/Jan_Mayen",
+                    "browserName": "Firefox",
+                    "browserVersion": "115.0",
+                    "acceptHeader": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                    "deviceExperimentId": "ChxOekkxTmpnek16UTRNVFl4TkRrek1ETTVOdz09EJrn1aUGGJrn1aUG",
+                    "screenWidthPoints": 584,
+                    "screenHeightPoints": 939,
+                    "screenPixelDensity": 1,
+                    "screenDensityFloat": 1,
+                    "utcOffsetMinutes": 120,
+                    "musicAppInfo": {
+                        "pwaInstallabilityStatus": "PWA_INSTALLABILITY_STATUS_UNKNOWN",
+                        "webDisplayMode": "WEB_DISPLAY_MODE_BROWSER",
+                        "storeDigitalGoodsApiSupportStatus": {
+                            "playStoreDigitalGoodsApiSupportStatus": "DIGITAL_GOODS_API_SUPPORT_STATUS_UNSUPPORTED"
+                        }
+                    }
+                },
+                "user": { "lockedSafetyMode": False },
+                "request": {
+                    "useSsl": True,
+                    "internalExperimentFlags": [],
+                    "consistencyTokenJars": []
+                },
+                "adSignalsInfo": {
+                    "params": []
+                }
+            }
         )
 
-        if self.credentials.api_key == "":
+        self.start_millis = get_current_millis()
+
+        if self.credentials.api_key == "" or DEBUG:
             self._fetch_from_main_page()
         
         super().__init__(*args, **kwargs)
@@ -132,12 +194,64 @@ class YoutubeMusic(Page):
         else:
             self.LOGGER.error(f"Couldn't find an API-KEY for {type(self).__name__}. :((")
 
+        # context
+        context_pattern = r"(?<=\"INNERTUBE_CONTEXT\":{)(.*?)(?=},\"INNERTUBE_CONTEXT_CLIENT_NAME\":)"
+        found_context = False
+        for context_string in re.findall(context_pattern, content):
+            try:
+                context = json.loads("{" + context_string + "}")
+                found_context
+            except json.decoder.JSONDecodeError:
+                continue
+
+            self.credentials.context = context
+            break
+
+        if not found_context:
+            self.LOGGER.warning(f"Couldn't find a context for {type(self).__name__}.")
 
     def get_source_type(self, source: Source) -> Optional[Type[DatabaseObject]]:
         return super().get_source_type(source)
     
     def general_search(self, search_query: str) -> List[DatabaseObject]:
-        return []
+        self.LOGGER.info(f"general search for {search_query}")
+        print(self.credentials)
+
+        urlescaped_query: str = quote(search_query.strip().replace(" ", "+"))
+
+        LAST_EDITED_TIME = get_current_millis() - random.randint(0, 20)
+        _estimated_time = sum(len(search_query) * random.randint(50, 100) for _ in search_query.strip())
+        FIRST_EDITED_TIME = LAST_EDITED_TIME - _estimated_time if LAST_EDITED_TIME - self.start_millis > _estimated_time else random.randint(50, 100)
+
+        # construct the request
+        r = self.connection.post(
+            url=get_youtube_url(path="/youtubei/v1/search", query=f"key={self.credentials.api_key}&prettyPrint=false"),
+            json={
+                "context": {**self.credentials.context, "adSignalsInfo":{"params":[]}},
+                "query": search_query,
+                "suggestStats": {
+                    "clientName": "youtube-music",
+                    "firstEditTimeMsec": FIRST_EDITED_TIME,
+                    "inputMethod": "KEYBOARD",
+                    "lastEditTimeMsec":	LAST_EDITED_TIME,
+                    "originalQuery": search_query,
+                    "parameterValidationStatus": "VALID_PARAMETERS",
+                    "searchMethod": "ENTER_KEY",
+                    "validationStatus":	"VALID",
+                    "zeroPrefixEnabled": True,
+                    "availableSuggestions": []
+                }
+            },
+            headers={
+                "Referer": get_youtube_url(path=f"/search", query=f"q={urlescaped_query}")
+            }
+        )
+
+        print(r)
+
+        return [
+            Song(title="Lore Ipsum")
+        ]
     
     def label_search(self, label: Label) -> List[Label]:
         return []

src/music_kraken/utils/enums/source.py

@@ -11,6 +11,7 @@ class SourceTypes(Enum):
 class SourcePages(Enum):
     YOUTUBE = "youtube"
     MUSIFY = "musify"
+    YOUTUBE_MUSIC = "youtube music"
     GENIUS = "genius"
     MUSICBRAINZ = "musicbrainz"
     ENCYCLOPAEDIA_METALLUM = "encyclopaedia metallum"

src/music_kraken/utils/functions.py

@@ -1,4 +1,11 @@
 import os
+from datetime import datetime
+
 
 def clear_console():
-    os.system('cls' if os.name in ('nt', 'dos') else 'clear')
+    os.system('cls' if os.name in ('nt', 'dos') else 'clear')
+
+
+def get_current_millis() -> int:
+    dt = datetime.now()
+    return int(dt.microsecond / 1_000)

src/music_kraken/utils/shared.py

@@ -126,3 +126,7 @@ FFMPEG_BINARY: Path = PATHS_SECTION.FFMPEG_BINARY.object_from_value
 
 HASNT_YET_STARTED: bool = MISC_SECTION.HASNT_YET_STARTED.object_from_value
 SLEEP_AFTER_YOUTUBE_403: float = CONNECTION_SECTION.SLEEP_AFTER_YOUTUBE_403.object_from_value
+
+DEBUG = True
+if DEBUG:
+    print("DEBUG ACTIVE")