diff --git a/shard.lock b/shard.lock index fefdfa4..ac36c2c 100755 --- a/shard.lock +++ b/shard.lock @@ -1,6 +1,10 @@ -version: 1.0 +version: 2.0 shards: - ydl_binaries: - github: cooperhammond/ydl-binaries - commit: c82e3937fee20fd076b1c73e24b2d0205e2cf0da + json_mapping: + git: https://github.com/crystal-lang/json_mapping.cr.git + version: 0.1.0 + + ydl_binaries: + git: https://github.com/cooperhammond/ydl-binaries.git + version: 1.1.1+git.commit.c82e3937fee20fd076b1c73e24b2d0205e2cf0da diff --git a/shard.yml b/shard.yml index 9782be4..f3f95ea 100755 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: irs -version: 1.0.0 +version: 1.0.1 authors: - Cooper Hammond @@ -12,4 +12,6 @@ license: MIT dependencies: ydl_binaries: - github: cooperhammond/ydl-binaries \ No newline at end of file + github: cooperhammond/ydl-binaries + json_mapping: + github: crystal-lang/json_mapping.cr diff --git a/src/glue/album.cr b/src/glue/album.cr index 1f5e3ee..7f8d019 100755 --- a/src/glue/album.cr +++ b/src/glue/album.cr @@ -9,7 +9,7 @@ class Album < SpotifyList # Uses the `spotify_searcher` defined in parent `SpotifyList` to find the # correct metadata of the list - def find_it + def find_it : JSON::Any album = @spotify_searcher.find_item("album", { "name" => @list_name.as(String), "artist" => @list_author.as(String), diff --git a/src/glue/mapper.cr b/src/glue/mapper.cr index 5083a37..5a38221 100755 --- a/src/glue/mapper.cr +++ b/src/glue/mapper.cr @@ -1,4 +1,5 @@ require "json" +require "json_mapping" class PlaylistExtensionMapper JSON.mapping( diff --git a/src/glue/playlist.cr b/src/glue/playlist.cr index 0a8dd68..069b599 100755 --- a/src/glue/playlist.cr +++ b/src/glue/playlist.cr @@ -13,7 +13,7 @@ class Playlist < SpotifyList # Uses the `spotify_searcher` defined in parent `SpotifyList` to find the # correct metadata of the list - def find_it + def find_it : JSON::Any @playlist = @spotify_searcher.find_item("playlist", { "name" => @list_name.as(String), "username" => @list_author.as(String), diff --git a/src/glue/song.cr b/src/glue/song.cr index e512ba9..2cc175a 100755 --- a/src/glue/song.cr +++ b/src/glue/song.cr @@ -146,7 +146,8 @@ class Song FileUtils.mkdir_p(strpath) end safe_filename = @filename.gsub(/[\/]/, "").gsub(" ", " ") - File.rename("./" + @filename, (path / safe_filename).to_s) + FileUtils.cp("./" + @filename, (path / safe_filename).to_s) + FileUtils.rm("./" + @filename) end # Provide metadata so that it doesn't have to find it. Useful for overwriting diff --git a/src/interact/future.cr b/src/interact/future.cr new file mode 100644 index 0000000..fa6a492 --- /dev/null +++ b/src/interact/future.cr @@ -0,0 +1,157 @@ +# copy and pasted from crystal 0.33.1 +# https://github.com/crystal-lang/crystal/blob/18e76172444c7bd07f58bf360bc21981b667668d/src/concurrent/future.cr#L138 + + +# :nodoc: +class Concurrent::Future(R) + enum State + Idle + Delayed + Running + Completed + Canceled + end + + @value : R? + @error : Exception? + @delay : Float64 + + def initialize(run_immediately = true, delay = 0.0, &@block : -> R) + @state = State::Idle + @value = nil + @error = nil + @channel = Channel(Nil).new + @delay = delay.to_f + @cancel_msg = nil + + spawn_compute if run_immediately + end + + def get + wait + value_or_raise + end + + def success? + completed? && !@error + end + + def failure? + completed? && @error + end + + def canceled? + @state == State::Canceled + end + + def completed? + @state == State::Completed + end + + def running? + @state == State::Running + end + + def delayed? + @state == State::Delayed + end + + def idle? + @state == State::Idle + end + + def cancel(msg = "Future canceled, you reached the [End of Time]") + return if @state >= State::Completed + @state = State::Canceled + @cancel_msg = msg + @channel.close + nil + end + + private def compute + return if @state >= State::Delayed + run_compute + end + + private def spawn_compute + return if @state >= State::Delayed + + @state = @delay > 0 ? State::Delayed : State::Running + + spawn { run_compute } + end + + private def run_compute + delay = @delay + + if delay > 0 + sleep delay + return if @state >= State::Canceled + @state = State::Running + end + + begin + @value = @block.call + rescue ex + @error = ex + ensure + @channel.close + @state = State::Completed + end + end + + private def wait + return if @state >= State::Completed + compute + @channel.receive? + end + + private def value_or_raise + raise Exception.new(@cancel_msg) if @state == State::Canceled + + value = @value + if value.is_a?(R) + value + elsif error = @error + raise error + else + raise "compiler bug" + end + end +end + +# Spawns a `Fiber` to compute *&block* in the background after *delay* has elapsed. +# Access to get is synchronized between fibers. *&block* is only called once. +# May be canceled before *&block* is called by calling `cancel`. +# ``` +# d = delay(1) { Process.kill(Process.pid) } +# long_operation +# d.cancel +# ``` +def delay(delay, &block : -> _) + Concurrent::Future.new delay: delay, &block +end + +# Spawns a `Fiber` to compute *&block* in the background. +# Access to get is synchronized between fibers. *&block* is only called once. +# ``` +# f = future { http_request } +# ... other actions ... +# f.get #=> String +# ``` +def future(&exp : -> _) + Concurrent::Future.new &exp +end + +# Conditionally spawns a `Fiber` to run *&block* in the background. +# Access to get is synchronized between fibers. *&block* is only called once. +# *&block* doesn't run by default, only when `get` is called. +# ``` +# l = lazy { expensive_computation } +# spawn { maybe_use_computation(l) } +# spawn { maybe_use_computation(l) } +# ``` +def lazy(&block : -> _) + Concurrent::Future.new run_immediately: false, &block +end + diff --git a/src/interact/logger.cr b/src/interact/logger.cr index 3f21acd..e1fb4ba 100755 --- a/src/interact/logger.cr +++ b/src/interact/logger.cr @@ -1,3 +1,5 @@ +require "./future" + class Logger @done_signal = "---DONE---" diff --git a/src/search/youtube.cr b/src/search/youtube.cr index 5742852..7242ea3 100755 --- a/src/search/youtube.cr +++ b/src/search/youtube.cr @@ -1,5 +1,7 @@ require "http" require "xml" +require "json" + module Youtube extend self @@ -19,6 +21,8 @@ module Youtube "official video", "official music video", ] + alias NODES_CLASS = Array(Hash(String, String)) + # Finds a youtube url based off of the given information. # The query to youtube is constructed like this: # " " @@ -63,7 +67,7 @@ module Youtube # ... # ] private def rank_videos(song_name : String, artist_name : String, - query : String, nodes : Array(XML::Node)) : Array(Hash(String, Int32)) + query : String, nodes : Array(Hash(String, String))) : Array(Hash(String, Int32)) points = [] of Hash(String, Int32) index = 0 @@ -149,32 +153,45 @@ module Youtube # Finds valid video links from a `HTTP::Client.get` request # Returns an `Array` of `XML::Node` - private def get_video_link_nodes(doc : String) : Array(XML::Node) - nodes = XML.parse(doc).xpath_nodes("//a") - valid_nodes = [] of XML::Node + private def get_video_link_nodes(response_body : String) : NODES_CLASS + yt_initial_data : JSON::Any = JSON.parse("{}") - nodes.each do |node| - if video_link_node?(node) - valid_nodes.push(node) + response_body.each_line do |line| + if line.includes?("window[\"ytInitialData\"]") + yt_initial_data = JSON.parse(line.split(" = ")[1][0..-2]) end end - return valid_nodes - end - - # Tests if the provided `XML::Node` has a valid link to a video - # Returns a `Bool` - private def video_link_node?(node : XML::Node) : Bool - # If this passes, then the node links to a playlist, not a video - if node["href"]? - return false if node["href"].includes?("&list=") + if yt_initial_data == JSON.parse("{}") + puts "Youtube has changed the way it organizes its webpage, submit a bug" + puts "on https://github.com/cooperhammond/irs" + exit(1) end - VALID_LINK_CLASSES.each do |valid_class| - if node["class"]? - return true if node["class"].includes?(valid_class) + # where the vid metadata lives + yt_initial_data = yt_initial_data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"]["contents"] + + video_metadata = [] of Hash(String, String) + + i = 0 + while true + begin + # video title + raw_metadata = yt_initial_data[0]["itemSectionRenderer"]["contents"][i]["videoRenderer"] + + metadata = {} of String => String + + metadata["title"] = raw_metadata["title"]["runs"][0]["text"].as_s + metadata["href"] = raw_metadata["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"].as_s + + video_metadata.push(metadata) + rescue IndexError + break + rescue Exception end + i += 1 end - return false + + return video_metadata end end