From 289f1d8c63523eb298f50a648ae2eb53d11a528c Mon Sep 17 00:00:00 2001 From: imsamuka Date: Mon, 27 Dec 2021 01:14:39 -0300 Subject: [PATCH 1/7] fix video selection offset --- src/search/youtube.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 5dd58c5..8d629f4 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -107,7 +107,7 @@ module Youtube end end - return yt_metadata[input]["href"] + return yt_metadata[input-1]["href"] end From bdc63b4c352f129d3c05c060e2a81b9578d01b76 Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 16:57:10 -0300 Subject: [PATCH 2/7] fix --url ignoring argument on song.cr --- src/bottle/cli.cr | 6 +++--- src/glue/song.cr | 20 ++++++++++++-------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/bottle/cli.cr b/src/bottle/cli.cr index 7b2b7d3..2ff974a 100755 --- a/src/bottle/cli.cr +++ b/src/bottle/cli.cr @@ -50,9 +50,9 @@ class CLI #{Style.blue "-s, --song "} Specify song name to download #{Style.blue "-A, --album "} Specify the album name to download #{Style.blue "-p, --playlist "} Specify the playlist name to download - #{Style.blue "-u, --url []"} Specify the youtube url to download from - #{Style.blue " "} (for single songs, include as an command-line - #{Style.blue " "} argument, for albums or playlists do not) + #{Style.blue "-u, --url "} Specify the youtube url to download from + #{Style.blue " "} (for albums and playlists, the command-line + #{Style.blue " "} argument is ignored, and it should be '') #{Style.blue "-S, --select"} Use a menu to choose each song's video source #{Style.bold "Examples:"} diff --git a/src/glue/song.cr b/src/glue/song.cr index c7acdad..89ee737 100755 --- a/src/glue/song.cr +++ b/src/glue/song.cr @@ -53,12 +53,12 @@ class Song # Find, downloads, and tags the mp3 song that this class represents. # Optionally takes a youtube URL to download from - # + # # ``` # Song.new("Bohemian Rhapsody", "Queen").grab_it # ``` def grab_it(url : (String | Nil) = nil, flags = {} of String => String) - ask_url = flags["url"]? + passed_url : (String | Nil) = flags["url"] select_link = flags["select"]? outputter("intro", 0) @@ -92,11 +92,15 @@ class Song @artist_name = data["artists"][0]["name"].as_s @filename = "#{Pattern.parse(Config.filename_pattern, data)}.mp3" - if ask_url - outputter("url", 4) - url = gets - if !url.nil? && url.strip == "" - url = nil + if passed_url + if passed_url.strip != "" + url = passed_url + else + outputter("url", 4) + url = gets + if !url.nil? && url.strip == "" + url = nil + end end end @@ -130,7 +134,7 @@ class Song outputter("albumart", 0) # check if song's metadata has been modded in playlist, update artist accordingly - if data["artists"][-1]["owner"]? + if data["artists"][-1]["owner"]? @artist = data["artists"][-1]["name"].as_s else @artist = data["artists"][0]["name"].as_s From ac7bc02ec5e9374f0fd9d36cec13afe6f91d3edb Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 17:20:37 -0300 Subject: [PATCH 3/7] fix youtube urls validation --- src/search/youtube.cr | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 8d629f4..051adca 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -57,7 +57,7 @@ module Youtube root = "https://youtube.com" if download_first - return root + yt_metadata[0]["href"] + return root + yt_metadata[0]["href"] end ranked = Ranker.rank_videos(spotify_metadata, yt_metadata, human_query) @@ -75,12 +75,12 @@ module Youtube exit 1 end - + # Presents a menu with song info for the user to choose which url they want to download - private def select_link_menu(spotify_metadata : JSON::Any, + private def select_link_menu(spotify_metadata : JSON::Any, yt_metadata : YT_METADATA_CLASS) : String - puts Style.dim(" Spotify info: ") + - Style.bold("\"" + spotify_metadata["name"].to_s) + "\" by \"" + + puts Style.dim(" Spotify info: ") + + Style.bold("\"" + spotify_metadata["name"].to_s) + "\" by \"" + Style.bold(spotify_metadata["artists"][0]["name"].to_s + "\"") + " @ " + Style.blue((spotify_metadata["duration_ms"].as_i / 1000).to_i.to_s) + "s" puts " Choose video to download:" @@ -111,7 +111,7 @@ module Youtube end - # Finds valid video links from a `HTTP::Client.get` request + # Finds valid video links from a `HTTP::Client.get` request # Returns an `Array` of `NODES_CLASS` containing additional metadata from Youtube private def get_yt_search_metadata(response_body : String) : YT_METADATA_CLASS yt_initial_data : JSON::Any = JSON.parse("{}") @@ -181,7 +181,9 @@ module Youtube # is it a video on youtube, with a query query = uri.query - if uri.host != "www.youtube.com" || uri.path != "/watch" || !query + + if !uri || !query || !uri.host || uri.path != "/watch" || + !uri.host.not_nil!.ends_with?("youtube.com") return false end @@ -204,8 +206,9 @@ module Youtube # this is an internal endpoint to validate the video ID - response = HTTP::Client.get "https://www.youtube.com/get_video_info?video_id=#{vID}" - return response.body.includes?("status=ok") + response = HTTP::Client.get "https://www.youtube.com/oembed?format=json&url=#{url}" + + return response.success? end -end \ No newline at end of file +end From f962a0ab757ce558253667bac0761861a294f48a Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 18:04:05 -0300 Subject: [PATCH 4/7] make youtube url validation safer --- src/glue/song.cr | 5 +++-- src/search/youtube.cr | 38 +++++++++++++------------------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/src/glue/song.cr b/src/glue/song.cr index 89ee737..008759e 100755 --- a/src/glue/song.cr +++ b/src/glue/song.cr @@ -115,8 +115,9 @@ class Song outputter("url", 1) else outputter("url", 2) - if !Youtube.is_valid_url(url) - raise("The url '#{url}' is an invalid youtube URL " + + url = Youtube.validate_url(url) + if !url + raise("The url is an invalid youtube URL " + "Check the URL and try again") end outputter("url", 3) diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 051adca..5fd83ba 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -170,45 +170,33 @@ module Youtube return video_metadata end - # Checks if the given URL is a valid youtube URL + # Returns as a valid URL if possible # # ``` - # Youtube.is_valid_url("https://www.youtube.com/watch?v=NOTANACTUALVIDEOID") - # => false + # Youtube.validate_url("https://www.youtube.com/watch?v=NOTANACTUALVIDEOID") + # => nil # ``` - def is_valid_url(url : String) : Bool + def validate_url(url : String) : String | Nil uri = URI.parse url + return nil if !uri - # is it a video on youtube, with a query query = uri.query - - if !uri || !query || !uri.host || uri.path != "/watch" || - !uri.host.not_nil!.ends_with?("youtube.com") - return false - end - - - queries = query.split('&') + return nil if !query # find the video ID - i = 0 - while i < queries.size - if queries[i].starts_with?("v=") - vID = queries[i][2..-1] - break + vID = nil + query.split('&').each do |q| + if q.starts_with?("v=") + vID = q[2..-1] end - i += 1 - end - - if !vID - return false end + return nil if !vID + url = "https://www.youtube.com/watch?v=#{vID}" # this is an internal endpoint to validate the video ID - response = HTTP::Client.get "https://www.youtube.com/oembed?format=json&url=#{url}" - return response.success? + return response.success? ? url : nil end end From 72938a9b6a277d49e738a138878c100b6e12418b Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 19:24:54 -0300 Subject: [PATCH 5/7] show video title from url --- src/glue/song.cr | 2 +- src/search/youtube.cr | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/glue/song.cr b/src/glue/song.cr index 008759e..77fbd66 100755 --- a/src/glue/song.cr +++ b/src/glue/song.cr @@ -58,7 +58,7 @@ class Song # Song.new("Bohemian Rhapsody", "Queen").grab_it # ``` def grab_it(url : (String | Nil) = nil, flags = {} of String => String) - passed_url : (String | Nil) = flags["url"] + passed_url : (String | Nil) = flags["url"]? select_link = flags["select"]? outputter("intro", 0) diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 5fd83ba..554459e 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -34,7 +34,6 @@ module Youtube search_terms = Config.search_terms - download_first = flags["dl_first"]? select_link = flags["select"]? song_name = spotify_metadata["name"].as_s @@ -43,9 +42,9 @@ module Youtube human_query = song_name + " " + artist_name + " " + search_terms.strip url_query = human_query.gsub(" ", "+") - url = "https://www.youtube.com/results?search_query=" + url_query + search_url = "https://www.youtube.com/results?search_query=" + url_query - response = HTTP::Client.get(url) + response = HTTP::Client.get(search_url) yt_metadata = get_yt_search_metadata(response.body) @@ -55,19 +54,14 @@ module Youtube end root = "https://youtube.com" - - if download_first - return root + yt_metadata[0]["href"] - end - ranked = Ranker.rank_videos(spotify_metadata, yt_metadata, human_query) if select_link return root + select_link_menu(spotify_metadata, yt_metadata) end - begin + puts Style.dim(" Video: ") + yt_metadata[ranked[0]["index"]]["title"] return root + yt_metadata[ranked[0]["index"]]["href"] rescue IndexError return nil @@ -196,7 +190,12 @@ module Youtube # this is an internal endpoint to validate the video ID response = HTTP::Client.get "https://www.youtube.com/oembed?format=json&url=#{url}" + return nil unless response.success? - return response.success? ? url : nil + res_json = JSON.parse(response.body) + title = res_json["title"].as_s + puts Style.dim(" Video: ") + title + + return url end end From 3d4acdeaea581f9087e5b963ee1f3fa03293964a Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 20:25:47 -0300 Subject: [PATCH 6/7] add option to skip tracks on albums/playlists --- src/bottle/cli.cr | 4 +++- src/glue/list.cr | 27 +++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/src/bottle/cli.cr b/src/bottle/cli.cr index 2ff974a..c0d22bb 100755 --- a/src/bottle/cli.cr +++ b/src/bottle/cli.cr @@ -21,7 +21,8 @@ class CLI [["-A", "--album"], "album", "string"], [["-p", "--playlist"], "playlist", "string"], [["-u", "--url"], "url", "string"], - [["-S", "--select"], "select", "bool"] + [["-S", "--select"], "select", "bool"], + [["--ask-skip"], "ask_skip", "bool"] ] @args : Hash(String, String) @@ -54,6 +55,7 @@ class CLI #{Style.blue " "} (for albums and playlists, the command-line #{Style.blue " "} argument is ignored, and it should be '') #{Style.blue "-S, --select"} Use a menu to choose each song's video source + #{Style.blue "--ask-skip"} Before every playlist/album song, ask to skip #{Style.bold "Examples:"} $ #{Style.green %(irs --song "Bohemian Rhapsody" --artist "Queen")} diff --git a/src/glue/list.cr b/src/glue/list.cr index 717b0ed..6cfc11c 100755 --- a/src/glue/list.cr +++ b/src/glue/list.cr @@ -29,6 +29,8 @@ abstract class SpotifyList # Finds the list, and downloads all of the songs using the `Song` class def grab_it(flags = {} of String => String) ask_url = flags["url"]? + ask_skip = flags["ask_skip"]? + is_playlist = flags["playlist"]? if !@spotify_searcher.authorized? raise("Need to call provide_client_keys on Album or Playlist class.") @@ -45,22 +47,28 @@ abstract class SpotifyList i = 0 contents.each do |datum| + i += 1 if datum["track"]? datum = datum["track"] end data = organize_song_metadata(list, datum) - song = Song.new(data["name"].to_s, data["artists"][0]["name"].to_s) + s_name = data["name"].to_s + s_artist = data["artists"][0]["name"].to_s + + song = Song.new(s_name, s_artist) song.provide_spotify(@spotify_searcher) song.provide_metadata(data) - puts Style.bold("[#{data["track_number"]}/#{contents.size}]") - song.grab_it(flags: flags) + puts Style.bold("[#{i}/#{contents.size}]") - organize(song) - - i += 1 + unless ask_skip && skip?(s_name, s_artist, is_playlist) + song.grab_it(flags: flags) + organize(song) + else + puts "Skipping..." + end end end @@ -69,6 +77,13 @@ abstract class SpotifyList @spotify_searcher.authorize(client_key, client_secret) end + private def skip?(name, artist, is_playlist) + print "Skip #{Style.blue name}" + + (is_playlist ? " (by #{Style.green artist})": "") + "? " + response = gets + return response && response.lstrip.downcase.starts_with? "y" + end + private def outputter(key : String, index : Int32) text = @outputs[key][index] .gsub("%l", @list_name) From 3263ff4e07e033095b4ac4cda625d64a437cabc3 Mon Sep 17 00:00:00 2001 From: imsamuka Date: Sun, 2 Jan 2022 22:58:03 -0300 Subject: [PATCH 7/7] fix GET requests url encoding --- src/search/spotify.cr | 36 +++++++++++++++++------------------- src/search/youtube.cr | 11 +++++------ 2 files changed, 22 insertions(+), 25 deletions(-) diff --git a/src/search/spotify.cr b/src/search/spotify.cr index 45e9262..ae95492 100755 --- a/src/search/spotify.cr +++ b/src/search/spotify.cr @@ -60,9 +60,10 @@ class SpotifySearcher # ``` def find_item(item_type : String, item_parameters : Hash, offset = 0, limit = 20) : JSON::Any? - query = generate_query(item_type, item_parameters, offset, limit) + query = generate_query(item_type, item_parameters) - url = @root_url.join("search?q=#{query}").to_s + url = "search?q=#{query}&type=#{item_type}&limit=#{limit}&offset=#{offset}" + url = @root_url.join(url).to_s response = HTTP::Client.get(url, headers: @access_header) error_check(response) @@ -228,8 +229,7 @@ class SpotifySearcher # Generates url to run a GET request against to the Spotify open API # Returns a `String.` - private def generate_query(item_type : String, item_parameters : Hash, - offset : Int32, limit : Int32) : String + private def generate_query(item_type : String, item_parameters : Hash) : String query = "" # parameter keys to exclude in the api request. These values will be put @@ -241,9 +241,9 @@ class SpotifySearcher if k == "name" # will remove the "name:" param from the query if item_type == "playlist" - query += item_parameters[k].gsub(" ", "+") + "+" + query += item_parameters[k] + "+" else - query += param_encode(item_type, item_parameters[k]) + query += as_field(item_type, item_parameters[k]) end # check if the key is to be excluded @@ -254,14 +254,21 @@ class SpotifySearcher # NOTE: playlist names will be inserted into the query normally, without # a parameter. else - query += param_encode(k, item_parameters[k]) + query += as_field(k, item_parameters[k]) end end - # extra api info - query += "&type=#{item_type}&limit=#{limit}&offset=#{offset}" + return URI.encode(query.rchop("+")) + end - return query + # Returns a `String` encoded for the spotify api + # + # ``` + # query_encode("album", "A Night At The Opera") + # => "album:A Night At The Opera+" + # ``` + private def as_field(key, value) : String + return "#{key}:#{value}+" end # Ranks the given items based off of the info from parameters. @@ -327,15 +334,6 @@ class SpotifySearcher end end - # Returns a `String` encoded for the spotify api - # - # ``` - # query_encode("album", "A Night At The Opera") - # => "album:A+Night+At+The+Opera" - # ``` - private def param_encode(key : String, value : String) : String - return key.gsub(" ", "+") + ":" + value.gsub(" ", "+") + "+" - end end # puts SpotifySearcher.new() diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 554459e..3bdd2a4 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -39,12 +39,10 @@ module Youtube song_name = spotify_metadata["name"].as_s artist_name = spotify_metadata["artists"][0]["name"].as_s - human_query = song_name + " " + artist_name + " " + search_terms.strip - url_query = human_query.gsub(" ", "+") + human_query = "#{song_name} #{artist_name} #{search_terms.strip}" + params = HTTP::Params.encode({"search_query" => human_query}) - search_url = "https://www.youtube.com/results?search_query=" + url_query - - response = HTTP::Client.get(search_url) + response = HTTP::Client.get("https://www.youtube.com/results?#{params}") yt_metadata = get_yt_search_metadata(response.body) @@ -189,7 +187,8 @@ module Youtube url = "https://www.youtube.com/watch?v=#{vID}" # this is an internal endpoint to validate the video ID - response = HTTP::Client.get "https://www.youtube.com/oembed?format=json&url=#{url}" + params = HTTP::Params.encode({"format" => "json", "url" => url}) + response = HTTP::Client.get "https://www.youtube.com/oembed?#{params}" return nil unless response.success? res_json = JSON.parse(response.body)