diff --git a/.gitignore b/.gitignore index b31c519..dbde338 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ update_pypi_and_github.py # Temporarily built binaries ffmpeg binaries/ + +# vscode work space +.vscode/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f98d2a6..0000000 --- a/.travis.yml +++ /dev/null @@ -1,15 +0,0 @@ -language: python -python: - - "2.7" - - "3.5" - - "3.6" - -install: - - python setup.py install - - irs --setup - -script: - - python tests/album.py - - python tests/playlist.py - - python tests/post_processors.py - - python tests/song.py diff --git a/README.md b/README.md index 8f0c2a4..7818c0f 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,3 @@ -
- # Ironic Redistribution System [![License: GNU](https://img.shields.io/badge/license-gnu-yellow.svg?style=flat-square)](http://www.gnu.org/licenses/gpl.html) @@ -7,8 +5,6 @@ [![Say Thanks](https://img.shields.io/badge/say-thanks-ff69b4.svg?style=flat-square)](https://saythanks.io/to/kepoorhampond) [![PyPI](https://img.shields.io/badge/pypi-irs-blue.svg?style=flat-square)](https://pypi.python.org/pypi/irs) -(Shields: Gotta Catch Em All) - > A music downloader that understands your metadata needs. A tool to download your music with metadata. It uses [Spotify](https://www.spotify.com/) for finding metadata and [Youtube](https://www.youtube.com/) for the actual audio source. You will need to have some Spotify tokens, the instructions to set them up are [here](https://github.com/kepoorhampond/irs#spotify-tokens). @@ -19,7 +15,6 @@ Works with Python 2 and 3. ``` $ sudo pip install irs $ irs --setup -$ pip install youtube_dl # Only if on windows ``` **You will need to have some Spotify tokens, the instructions to set them up are [here](https://github.com/kepoorhampond/irs#spotify-tokens).** @@ -27,30 +22,29 @@ $ pip install youtube_dl # Only if on windows ## Demo and Usages -This is a demo of the CLI displayling its features: -[![demo](https://asciinema.org/a/105993.png)](https://asciinema.org/a/105993?autoplay=1) - The usages can be found with the `-h` or `--help` flag: ``` -usage: irs [-h] [-a ARTIST -s SONG] [-A ALBUM [-a ARTIST]] - [-u USERNAME -p PLAYLIST] [-l LOCATION] [-o] [-c] +usage: irs [-h] [-S] [-a ARTIST] [-s SONG] [-A ALBUM] [-p PLAYLIST] + [-u USERNAME] [-o ORGANIZATION] optional arguments: -h, --help show this help message and exit + -S, --setup Run this by itself to setup config files and folder + for irs and download the ffmpeg binaries -a ARTIST, --artist ARTIST Specify artist name. Must be used with -s/--song or -A/--album -s SONG, --song SONG Specify song name. Must be used with -a/--artist -A ALBUM, --album ALBUM - Specify album name - -u USERNAME, --username USERNAME - Specify username. Must be used with -p/--playlist + Specify album name. Can be used by itself. -p PLAYLIST, --playlist PLAYLIST - Specify playlist name. Must be used with -u/--username - -l LOCATION, --location LOCATION - Specify a directory to place files in. - -o, --organize Organize downloaded files. - -c, --config Display path to config file. + Specify playlist name. Must be used with -A/--album + -u USERNAME, --username USERNAME + Specify user name for playlist. Must be used with + -A/--album + -o ORGANIZATION, --organization ORGANIZATION + Specify type of organization for list. Used when + downloading spotify playlist/album ``` So all of these are valid commands: @@ -79,19 +73,13 @@ Currently, the program attaches the following metadata to the downloaded files: - Genre - Track Number - Disc Number - - Compilation (iTunes only) - -## Philosophy - -When I made this program I was pretty much broke and my music addiction wasn't really helping that problem. So, I did the obvious thing: make an uber-complicated program to ~~steal~~ download music for me! As for the name, its acronym spells IRS, which I found amusing, seeing as the IRS ~~takes~~ steals money while my program ~~gives~~ reimburses you with music. - -The design/style inspiration of the CLI goes to [k4m4](https://github.com/k4m4). ## Wishlist - [x] Full album downloading - [x] Album art metadata correctly displayed - [x] Spotify playlist downloading + - [ ] Comment metadata + - [ ] Compilation metadata - [ ] GUI/Console interactive version - *in progress* - [ ] Lyric metadata - - [ ] 99% success rate for automatic song choosing diff --git a/irs/__init__.py b/irs/__init__.py index d97e88c..e69de29 100644 --- a/irs/__init__.py +++ b/irs/__init__.py @@ -1,21 +0,0 @@ -import os -import argparse -from .setup_binaries import setup - -parser = argparse.ArgumentParser() - -parser.add_argument("-S", "--setup", dest="setup", help="Setup IRS", - action="store_true") - -args, unknown = parser.parse_known_args() - -if args.setup: - setup() - exit(0) -elif not os.path.isdir(os.path.expanduser("~/.irs")): - print("Please run `irs --setup` to install the youtube-dl and \ -ffmpeg binaries.") - exit(1) -else: - from .ripper import Ripper - Ripper diff --git a/irs/cli.py b/irs/cli.py deleted file mode 100644 index 6454a92..0000000 --- a/irs/cli.py +++ /dev/null @@ -1,86 +0,0 @@ -# Arguments -import argparse - -# System -import sys -import os - -# Powered by: -from .ripper import Ripper -from .utils import Config, console - - -def main(): - parser = argparse.ArgumentParser() - - # Setup - parser.add_argument("-S", "--setup", dest="setup", help="Setup IRS", - action="store_true") - - # Single Song - parser.add_argument("-a", "--artist", dest="artist", help="Specify artist \ -name. Must be used with -s/--song or -A/--album") - parser.add_argument("-s", "--song", dest="song", help="Specify song name.\ - Must be used with -a/--artist") - - # Album - parser.add_argument("-A", "--album", dest="album", help="Specify album \ -name") - parser.add_argument("-e", "--exact", dest="exact", action="store_true", - help="The list will only be chosen if it equals the \ -user input.") - - # Playlist - parser.add_argument("-u", "--username", dest="username", help="Specify \ -username. Must be used with -p/--playlist") - parser.add_argument("-p", "--playlist", dest="playlist", help="Specify \ -playlist name. Must be used with -u/--username") - - # Post-Processors - parser.add_argument("-l", "--location", dest="location", help="Specify a \ -directory to place files in.") - parser.add_argument("-o", "--organize", dest="organize", - action="store_true", help="Organize downloaded files.") - - # Config - parser.add_argument("-c", "--config", dest="config", action="store_true", - help="Display path to config file.") - - args = parser.parse_args(Config.parse_default_flags()) - - if args.config: - import irs - print(os.path.dirname(irs.__file__) + "/config.py") - sys.exit() - - ripper_args = { - "post_processors": { - "custom_directory": args.location, - "organize": args.organize, - } - } - - # Combine args from argparse and the ripper_args as above and then - # remove all keys with the value of "None" - ripper_args.update(vars(args)) - - # Python 2 and below uses list.iteritems() while Python 3 uses list.items() - if sys.version_info[0] >= 3: - ripper_args = dict((k, v) for k, v in ripper_args.items() if v) - elif sys.version_info[0] < 3: - ripper_args = dict((k, v) for k, v in ripper_args.iteritems() if v) - - ripper = Ripper(ripper_args) - - if args.artist and args.song: - ripper.song(args.song, args.artist) - elif args.album: - ripper.spotify_list("album", args.album, artist=args.artist) - elif args.username and args.playlist: - ripper.spotify_list("playlist", args.playlist, args.username) - else: - console(ripper) - - -if __name__ == "__main__": - main() diff --git a/irs/cli/__init__.py b/irs/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/irs/cli/cli.py b/irs/cli/cli.py new file mode 100644 index 0000000..8079625 --- /dev/null +++ b/irs/cli/cli.py @@ -0,0 +1,51 @@ +import argparse + +from ..glue.song import Song +from ..glue.album import Album +from ..glue.playlist import Playlist +from ..install.setup import set_it_up +from .config_parser import parse_config + +def main(): + """The main cli method. Parses arguments from the command line.""" + + parser = argparse.ArgumentParser() + + parser.add_argument("-S", "--setup", dest="setup", action='store_true', + help="Run this by itself to setup config files " + "and folder for irs and download the ffmpeg binaries") + + parser.add_argument("-a", "--artist", dest="artist", + help="Specify artist name. Must be used with -s/--song or -A/--album") + + parser.add_argument("-s", "--song", dest="song", + help="Specify song name. Must be used with -a/--artist") + + parser.add_argument("-A", "--album", dest="album", + help="Specify album name. Can be used by itself.") + + parser.add_argument("-p", "--playlist", dest="playlist", + help="Specify playlist name. Must be used with -A/--album") + + parser.add_argument("-u", "--username", dest="username", + help="Specify user name for playlist. Must be used with -A/--album") + + parser.add_argument("-o", "--organization", dest="organization", + default="single-folder", help="Specify type of organization for list." + " Used when downloading spotify playlist/album") + + args = parser.parse_args() + + if args.setup: + set_it_up() + + parse_config() + + if args.song and args.artist: # single song + Song(args.song, args.artist).grab_it() + elif args.album and args.artist: # album with an artist + Album(args.album, args.artist).grab_it() + elif args.album: # album without artist + Album(args.album).grab_it() + elif args.playlist and args.username: # playlist + Playlist(args.playlist, args.username, args.organization).grab_it() diff --git a/irs/cli/config_parser.py b/irs/cli/config_parser.py new file mode 100644 index 0000000..2f5a4a1 --- /dev/null +++ b/irs/cli/config_parser.py @@ -0,0 +1,51 @@ +import os +import sys + +import yaml + + +def parse_config(): + """Parses config using environment variables.""" + + check_for_and_set("irs_config_dir", os.environ["HOME"] + "/.irs", None) + + home = os.environ["HOME"] + check_for = [home + "/.irs/config.yml", home + "/.irs/bin/ffmpeg", + home + "/.irs/bin/ffprobe"] + + for path in check_for: + if not os.path.exists(path): + print("There's no config set up. Set up a configuration folder by " + "running `irs --setup`") + sys.exit(1) + + config = {} + + with open(os.environ["irs_config_dir"] + "/config.yml", "r") as stream: + try: + config = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + check_for_and_set("SPOTIFY_CLIENT_ID", config.get( + "SPOTIFY_KEYS").get("CLIENT_ID"), None) + check_for_and_set("SPOTIFY_CLIENT_SECRET", config.get( + "SPOTIFY_KEYS").get("CLIENT_SECRET"), None) + + check_for_and_set("irs_music_dir", os.path.expanduser(config.get("music_directory")), + os.environ["HOME"] + "/Music") + check_for_and_set("irs_ffmpeg_dir", os.environ["irs_config_dir"] + "/bin", None) + + +def check_for_and_set(key, val, else_): + """Checks for an environment variable and if it doesn't exist, then set it + equal to the val given. + :param key: string, key to check for existence + :param val: value to replace key value with if it doesn't exists + :param else_: if val doesn't exist, use else_ instead + """ + if not os.environ.get(key): + if key: + os.environ[key] = val + else: + os.environ[key] = else_ diff --git a/irs/config.py b/irs/config.py deleted file mode 100644 index ac09b1d..0000000 --- a/irs/config.py +++ /dev/null @@ -1,12 +0,0 @@ -import sys -from os import path - -if path.isfile(path.expanduser("~/.irs/config_.py")): - sys.path.append(path.expanduser("~/.irs")) # Add config to path - - import config_ # from "~/.irs/config_.py" - - CONFIG = config_.CONFIG -else: - config = open("irs/config_preset.py", "r").read() - CONFIG = eval(config) diff --git a/irs/config_preset.py b/irs/config_preset.py deleted file mode 100644 index 9e2c159..0000000 --- a/irs/config_preset.py +++ /dev/null @@ -1,24 +0,0 @@ -CONFIG = dict( - - default_flags = ['-o'], - # For default flags. Right now, it organizes your files into an - # artist/album/song structure. - # To add a flag or argument, add an element to the index: - # default_flags = ['-o', '-l', '~/Music'] - - SPOTIFY_CLIENT_ID = '', - SPOTIFY_CLIENT_SECRET = '', - # You can either specify Spotify keys here, or in environment variables. - - additional_search_terms = 'lyrics', - # Search terms for youtube - - organize = True, - # True always forces organization. - # False always forces non-organization. - # None allows options and flags to determine if the files - # will be organized. - - custom_directory = "", - # When blank, defaults to '~/Music' -) diff --git a/irs/glue/__init__.py b/irs/glue/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/irs/glue/album.py b/irs/glue/album.py new file mode 100644 index 0000000..8f6af7a --- /dev/null +++ b/irs/glue/album.py @@ -0,0 +1,14 @@ +from .list import SpotifyList + +class Album(SpotifyList): + """A class for downloading albums as a whole.""" + + def _SpotifyList__find_it(self): + album = self.spotify_searcher.find_album( + self.list_title, + self.list_author + ) + return album + + def _SpotifyList__set_organization(self, song_index, song): + pass \ No newline at end of file diff --git a/irs/glue/list.py b/irs/glue/list.py new file mode 100644 index 0000000..1c730c0 --- /dev/null +++ b/irs/glue/list.py @@ -0,0 +1,50 @@ +import abc + +from ..search import spotify +from .song import Song + +class SpotifyList(object): + """A parent class for downloading spotify albums and playlists""" + + def __init__(self, list_title, list_author=None): + self.spotify_searcher = spotify.SpotifySearcher().authorize() + self.list_title = list_title + self.list_author = list_author + self.file_names = [] + + def grab_it(self): + """Downloads the songs! + """ + spotify_list = self.__find_it() + list_contents = spotify_list["tracks"]["items"] + + for index, s in enumerate(list_contents): + # if it's a playlist, get the actual track, not the metadata of + # the playlist + if s.get("track"): + s = s["track"] + + song = Song(s["name"], s["artists"][0]["name"]) + song.provide_spotify(self.spotify_searcher) + song.provide_metadata(self.spotify_searcher.song(s["uri"])) + song.get_relevant_tags() + self.__set_organization(index, song) + song.grab_it() + + # These following functions are named weird b/c PEP8 and python are in + # conflict. An error gets raised when private methods + # (prefix = __ due to PEP8) are overriden by child class with the same + # name b/c python is dumb with how it concatenates class names and method + # names with underscores + + def __find_it(self): + """Finds the list and return it""" + raise NotImplementedError("Must override __find_it with" + "_SpotifyList__find_it") + + def __set_organization(self, song_index, song): + """Post processing method for a single song + :param song: Song class + """ + raise NotImplementedError("Must override __post_process with" + "_SpotifyList__set_organization") \ No newline at end of file diff --git a/irs/glue/playlist.py b/irs/glue/playlist.py new file mode 100644 index 0000000..1c2695a --- /dev/null +++ b/irs/glue/playlist.py @@ -0,0 +1,39 @@ +import os + +from .list import SpotifyList + +class Playlist(SpotifyList): + """A class for downloading albums as a whole. + The majority of specs for methods can be found in the SpotifyList file""" + + def __init__(self, playlist_name, username, organization="single-folder"): + """ + :param playlist_name: a string, the name of the playlist + :param username: a string, the username of the creator of the playlist + :param organization: a string, following options: + "single-folder": All of the songs downloaded will be put into a + single folder. root-music-dir>playlist-name + "standard": All of the songs downloaded will be + organized by root-music-dir>artist>album + """ + super(Playlist, self).__init__(playlist_name, username) + self.organization = organization + + def _SpotifyList__find_it(self): + playlist = self.spotify_searcher.find_playlist( + self.list_title, + self.list_author + ) + return playlist + + def _SpotifyList__set_organization(self, song_index, song): + if self.organization == "standard": + song.set_standard_organization() + elif self.organization == "single-folder": + # reindex the file names in order to keep them in alphabetical order + song.provide_new_file_name("{} - {}.mp3".format( + song_index + 1, song.tags["title"] + )) + song.provide_new_location(os.path.join( + os.getcwd(), self.list_title + )) \ No newline at end of file diff --git a/irs/glue/song.py b/irs/glue/song.py new file mode 100644 index 0000000..ca8f0e6 --- /dev/null +++ b/irs/glue/song.py @@ -0,0 +1,192 @@ +import os +import errno +import string + +from ..search import spotify, youtube +from ..interact import ripper, tagger + + +class Song(object): + """A grabber for a single song. Unless provided, gets metadata, finds url, + downloads, converts, tags, and moves it.""" + def __init__(self, song_title, artist_name): + self.song_title = song_title + self.artist_name = artist_name + + self.spotify_searcher = None + self.spotify_authenticated = False + + self.metadata = None + self.tags = {} + self.parsed_tags = False + + self.file_name = song_title + ".mp3" + self.end_file_name = None + + self.current_location = os.getcwd() + self.end_location = None + + def grab_it(self, post_process=True): + """The main method to call for this class. Unless provided, grabs + metadata, finds the url, downloads the video, converts it, and + tags it. + :param post_process: boolean, + :rtype: a string, the file_name + """ + + self.metadata = self.__parse_data() + + self.tags = self.get_relevant_tags() + + print("'{}' by {}:".format(self.tags["title"], self.tags["artist"])) + + print("Searching youtube ...") + song_url = youtube.find_url(self.tags["title"], self.tags["artist"]) + + if self.metadata: + self.file_name = '{} - {}.mp3'.format( + self.tags["tracknumber"], + self.tags["title"] + ) + + print("Downloading ...") + ripper.rip_from_url(song_url, self.file_name) + print("Converting to mp3 ...") # TODO: add this into a hook for ydl + + print("Tagging ...") + song_tags = tagger.Tagger(self.file_name) + + for tag in self.tags: + if tag is "albumart": + song_tags.add_album_art(self.tags[tag]) + else: + song_tags.add_tag(tag, self.tags[tag]) + + if post_process: + self.__organize() + + def provide_spotify(self, spotify_searcher): + """This function will set this class's spotify searcher to the one + provided to prevent the need to authenticate twice + :param spotify_searcher: an instance of + irs.searcher.spotify.SpotifySearcher, the spotify searcher to use + :rtype: self class + """ + self.spotify_searcher = spotify_searcher + self.spotify_authenticated = True + return self + + def provide_metadata(self, metadata): + """Provides metadata for the song so searches don't have to be + performed twice. If this is called with new metadata, + other metadata won't be searched. + :param metadata: a dict, the new metadata from a spotipy track search + :rtype: self class + """ + self.metadata = metadata + return self + + def provide_tag(self, key, value): + """Provides tags for the song. Tags will still be parsed, but will not + overwrite these provided tags. + :param: a dict, the tags that will overwrite the metadata provided tags + :rtype: self class + """ + self.tags[key] = value + return self + + def provide_new_location(self, new_loc): + """Provides a new, non-default location for the song. + :param new_loc: a string, the path of the new location WITHOUT filename + :rtype: self class + """ + self.end_location = new_loc + return self + + def provide_new_file_name(self, new_name): + """Provides a new file name for the song file. DOESNT append .mp3 + :param new_name: string + :rtype: self class + """ + self.end_file_name = new_name + return self + + def set_standard_organization(self): + """Sets standard organization for the file, which is + root-music-dir>artist-folder>album-folder>song + """ + if not self.parsed_tags: + self.tags = self.get_relevant_tags() + self.end_location = os.path.join( + os.environ.get("irs_music_dir"), self.tags["artist"], + self.tags["album"] + ) + self.end_file_name = "{} - {}.mp3".format( + self.tags["tracknumber"], self.tags["title"] + ) + + def get_relevant_tags(self): + """Sorts relevant info from the spotipy metadata. Merges with any + provided tags from provide_tags method. + :rtype: a dict, parsed tags + """ + # TODO: come up with fallback solution if there's no metadata found + # follows this pattern: + # if this does not exist: + # set the thing that doesn't exist to a + # specific value from the metadata dict + tags = self.tags + metadata = self.metadata + + if not tags.get("title"): + tags["title"] = metadata["name"] + if not tags.get("artist"): + tags["artist"] = metadata["artists"][0]["name"] + if not tags.get("album"): + tags["album"] = metadata["album"]["name"] + if not tags.get("tracknumber"): + tags["tracknumber"] = str(metadata["track_number"]) + if not tags.get("albumart"): + tags["albumart"] = metadata["album"]["images"][0]["url"] + if not tags.get("genre") and self.spotify_searcher: + tags["genre"] = string.capwords(self.spotify_searcher.artist( + metadata["artists"][0]["uri"])["genres"][0]) + + self.tags = tags + return self.tags + + def __organize(self): + """Based off of self.current_location, self.end_location, and self. + file_name, this function creates folders for the end location and moves + the file there. + """ + if not self.end_location: + self.set_standard_organization() + + if not os.path.exists(self.end_location): + # try loop to prevent against race conditions with os.path.exists + # and os.makedirs + try: + os.makedirs(self.end_location) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + os.rename( + self.current_location + "/" + self.file_name, + self.end_location + "/" + self.end_file_name, + ) + + def __parse_data(self): + """If a spotify searcher has not been provided, create one.""" + if not self.spotify_authenticated and not self.metadata: + self.spotify_searcher = spotify.SpotifySearcher().authorize() + self.spotify_authenticated = True + + """If metadata has not been provided, search for it.""" + if not self.metadata: + self.metadata = self.spotify_searcher.find_song( + self.song_title, self.artist_name + ) + + return self.metadata diff --git a/irs/install b/irs/install new file mode 160000 index 0000000..3a8323f --- /dev/null +++ b/irs/install @@ -0,0 +1 @@ +Subproject commit 3a8323f39eda597ac5c4b4e2559577ca757aedc0 diff --git a/irs/interact/__init__.py b/irs/interact/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/irs/interact/ripper.py b/irs/interact/ripper.py new file mode 100644 index 0000000..b6dd57e --- /dev/null +++ b/irs/interact/ripper.py @@ -0,0 +1,50 @@ +import os +import sys +import glob +import shutil + +import youtube_dl + + +def rip_from_url(video_url, output_name): + """This method downloads a video, converts it to an MP3 and renames it + :param video_url: a string, youtube url of the video you want to download + :param output_name: a string, the name of the output file + """ + + ydl_opts = { + 'format': 'bestaudio/best', + 'postprocessors': [{ + 'key': 'FFmpegExtractAudio', + 'preferredcodec': 'mp3', + 'preferredquality': '192', + }], + 'logger': _DownloadLogger(), + 'progress_hooks': [_download_hook], + 'output': "tmp_file", + 'prefer-ffmpeg': True, + 'ffmpeg_location': os.environ["irs_ffmpeg_dir"], + } + + with youtube_dl.YoutubeDL(ydl_opts) as ydl: + ydl.download([video_url]) + + for f in glob.glob("./*%s*" % video_url.split("/watch?v=")[-1]): + shutil.move(f, output_name) + + +class _DownloadLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + print(msg) + + +# TODO: update the download log +def _download_hook(d): + if d['status'] == 'finished': + print("Done!") \ No newline at end of file diff --git a/irs/interact/tagger.py b/irs/interact/tagger.py new file mode 100644 index 0000000..26a6409 --- /dev/null +++ b/irs/interact/tagger.py @@ -0,0 +1,67 @@ +import sys + +if sys.version_info[0] >= 3: + from urllib.request import urlopen +elif sys.version_info[0] < 3: + from urllib import quote_plus, quote + from urllib import urlopen + +from mutagen.mp3 import EasyMP3 +from mutagen.easyid3 import EasyID3, EasyID3KeyError +from mutagen.id3 import APIC, ID3 + + +class Tagger(object): + """Attaches ID3 tags to MP3 files.""" + + def __init__(self, location): + """Initializes the class and generates ID3 tags for the mp3 + :param location: a string, the location of the mp3 that you want ID3 + tags on + """ + EasyID3.RegisterTextKey("comment", "COMM") + self.location = location + self.mp3 = EasyID3(self.location) + + def add_tag(self, tag, data): + """Adds a tag to the mp3 file you specified in __init__ and saves it + :param tag: a string, the name of the tag you want to add to the mp3 + valid tag names: + "title", "artist", "album", "genre", "tracknumber" (string), + "discnumber" (string), + "compilation" ("1" for true, "" for false) + :param data: a string, the data that you want to attach to the mp3 + under the specified tag name + """ + # For valid tags: `EasyID3.valid_keys.keys()` + self.mp3[tag] = data + self.mp3.save() + + def read_tag(self, tag): + """Tries to read a tag from the initialized mp3 file + :param tag: a string, the name of the tag you want to read + :rtype: an array with a string inside. The string inside the array is + the data you're requesting. If there's no tag associated or no data + attached with your requested tag, a blank array will be returned. + """ + try: + return self.mp3[tag] + except EasyID3KeyError or KeyError: + return [] + + def add_album_art(self, image_url): + """Adds album art to the initialized mp3 file + :param image_url: a string, the url of the image you want to attach to + the mp3 + """ + mp3 = EasyMP3(self.location, ID3=ID3) + mp3.tags.add( + APIC( + encoding = 3, + mime = 'image/png', + type = 3, + desc = 'cover', + data = urlopen(image_url).read() + ) + ) + mp3.save() \ No newline at end of file diff --git a/irs/metadata.py b/irs/metadata.py deleted file mode 100644 index a26fc5d..0000000 --- a/irs/metadata.py +++ /dev/null @@ -1,76 +0,0 @@ -# MP3 Metadata editing -from mutagen.mp3 import EasyMP3 -from mutagen.easyid3 import EasyID3 -from mutagen.id3 import * # There's A LOT of stuff to import, forgive me. -from mutagen.id3 import APIC, ID3 - -# System -import sys - -# Powered by... -import spotipy - -# Local utils -from .utils import ObjManip -om = ObjManip - -# Info finding -if sys.version_info[0] >= 3: - from urllib.parse import quote_plus, quote - from urllib.request import urlopen, Request -elif sys.version_info[0] < 3: - from urllib import quote_plus, quote - from urllib import urlopen - from urllib2 import Request - - -class Metadata: - def __init__(self, location): - self.spotify = spotipy.Spotify() - self.location = location - self.mp3 = EasyID3(self.location) - EasyID3.RegisterTextKey("comment", "COMM") - - def add_tag(self, tag, data): - # For valid tags: `EasyID3.valid_keys.keys()` - self.mp3[tag] = data - self.mp3.save() - - def read_tag(self, tag): - try: - return self.mp3[tag] - except Exception: - return [] - - def add_album_art(self, image_url): - mp3 = EasyMP3(self.location, ID3=ID3) - mp3.tags.add( - APIC( - encoding = 3, - mime = 'image/png', - type = 3, - desc = 'cover', - data = urlopen(image_url).read() - ) - ) - mp3.save() - - -def find_album_and_track(song, artist, spotify=spotipy.Spotify()): - tracks = spotify.search(q=song, type="track")["tracks"]["items"] - - for track in tracks: - if om.blank_include(track["name"], song): - if om.blank_include(track["artists"][0]["name"], artist): - return track["album"], track - return False, False - - -def parse_genre(genres): - if genres != []: - genres.reverse() - genres = list(map(lambda x: x.replace("-", " "), genres)) - genres.sort(key=lambda x: len(x.split())) - return genres[0].title() - else: - return "" diff --git a/irs/ripper.py b/irs/ripper.py deleted file mode 100644 index 8aa480b..0000000 --- a/irs/ripper.py +++ /dev/null @@ -1,484 +0,0 @@ -# _*_ coding:utf-8 _*_ - -# System -import sys -import os -import glob -import shutil - - -# Add youtube-dl binary to path -sys.path.append(os.path.expanduser("~/.irs/bin/youtube-dl")) - -# Powered by: -import youtube_dl # Locally imported from the binary - -import spotipy -from spotipy.oauth2 import SpotifyClientCredentials - - -# Local utilities -from .utils import YdlUtils, ObjManip, Config, CaptchaCheat -from .metadata import Metadata -from .metadata import find_album_and_track, parse_genre - -# Config File and Flags -from .config import CONFIG - -# Parsing -from bs4 import BeautifulSoup -if sys.version_info[0] >= 3: - from urllib.parse import urlencode - from urllib.request import urlopen -elif sys.version_info[0] < 3: - from urllib import urlencode - from urllib import urlopen -else: - print("Must be using Python 2 or 3") - sys.exit(1) - - -class Ripper: - def __init__(self, args={}): - self.args = args - if self.args.get("hook-text") is None: - self.args["hook-text"] = { - "youtube": "Finding Youtube link ...", - "list": '{0}: "{1}" by "{2}"', - "song": 'Downloading "{0}" by "{1}"', - "converting": "Converting to mp3 ...", - } - if self.args["hook-text"].get("converting") is not None: - CONFIG["converting"] = self.args["hook-text"]["converting"] - - self.locations = [] - self.type = None - try: - CLIENT_ID, CLIENT_SECRET = Config.parse_spotify_creds(self) - client_credentials_manager = SpotifyClientCredentials(CLIENT_ID, - CLIENT_SECRET - # Stupid lint - # and stupid - # long var - # names - ) - - self.spotify = spotipy.Spotify( - client_credentials_manager=client_credentials_manager) - - self.authorized = True - except Exception: - self.spotify = spotipy.Spotify() - self.authorized = False - - def post_processing(self, locations): - post_processors = self.args.get("post_processors") - directory_option = Config.parse_directory(self) - if post_processors: - if directory_option is not None: - for index, loc in enumerate(locations): - new_file_name = directory_option + "/" + loc - if not os.path.exists(directory_option): - os.makedirs(directory_option) - shutil.move(loc, new_file_name) - locations[index] = new_file_name - # I'd just go on believing that code this terrible doesn't exist. - # You can just close your eyes and scroll by. I'd encourage it. - # It's okay if you need to cry though. - # The rest of the code is here for you. - # It's like loving someone, - # Everyone has some flaws, but you still appreciate and embrace - # those flaws for being exclusive to them. - # And if those flaws are really enough to turn you off of them, - # then you *probably* don't really want to be with them anyways. - # Either way, it's up to you. (I'd just ignore this) - - if Config.parse_organize(self): - if self.type in ("album", "song"): - for index, loc in enumerate(locations): - mp3 = Metadata(loc) - new_loc = "" - if len(loc.split("/")) >= 2: - new_loc = "/".join(loc.split("/")[0:-1]) + "/" - file_name = loc.split("/")[-1] - else: - file_name = loc - artist = mp3.read_tag("artist")[0] - album = mp3.read_tag("album") - new_loc += ObjManip.blank(artist, False) - if album != []: - new_loc += "/" + ObjManip.blank(album[0], False) - if not os.path.exists(new_loc): - os.makedirs(new_loc) - new_loc += "/" + file_name - loc = loc.replace("//", "/") - new_loc = new_loc.replace("//", "/") - shutil.move(loc, new_loc) - locations[index] = new_loc - elif self.type == "playlist": - for index, loc in enumerate(locations): - new_loc = "" - if len(loc.split("/")) > 1: - new_loc = "/".join(loc.split("/")[0:-1]) - file_name = loc.split("/")[-1] - else: - file_name = loc - new_loc += ObjManip.blank(self.playlist_title, False) - if not os.path.exists(new_loc): - os.makedirs(new_loc) - loc = loc.replace("//", "/") - new_loc = (new_loc + "/" + file_name).replace("//", "/") - shutil.move(loc, new_loc) - - return locations - - def find_yt_url(self, song=None, artist=None, additional_search=None, caught_by_google=False, first=False, tries=0): - if additional_search is None: - additional_search = Config.parse_search_terms(self) - print(str(self.args["hook-text"].get("youtube"))) - - try: - if not song: - song = self.args["song_title"] - if not artist: - artist = self.args["artist"] - except KeyError: - raise ValueError("Must specify song_title/artist in `args` with \ -init, or in method arguments.") - - search_terms = song + " " + artist + " " + additional_search - - query_string = urlencode({"search_query": ( - search_terms.encode('utf-8'))}) - link = "http://www.youtube.com/results?" + query_string - - if not caught_by_google: - html_content = urlopen(link).read() - soup = BeautifulSoup(html_content, 'html.parser') - else: - soup = BeautifulSoup(CaptchaCheat.cheat_it(link), 'html.parser') - - # print(soup.prettify()) - # with open("index.html", "w") as f: - # f.write(soup.prettify().encode('utf-8')) - - def find_link(link): - try: - if "yt-simple-endpoint style-scope ytd-video-renderer" in str(" ".join(link["class"])) or \ - "yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 yt-uix-sessionlink spf-link " in str(" ".join(link["class"])): - if "&list=" not in link["href"]: - return link - except KeyError: - pass - - results = list(filter(None, map(find_link, soup.find_all("a")))) - - garbage_phrases = "cover album live clean rare version full full \ -album row at @ session how to npr music reimagined hr version".split(" ") - - self.code = None - counter = 0 - - while self.code is None and counter <= 10: - counter += 1 - for link in results: - if first == True and tries >= 10: - self.code = link - break - if ObjManip.check_garbage_phrases(garbage_phrases, - link["title"], song): - continue - if first == True: - self.code = link - break - if ObjManip.blank_include(link["title"], song) and \ - ObjManip.blank_include(link["title"], artist): - self.code = link - break - - if self.code is None: - for link in results: - if ObjManip.check_garbage_phrases(garbage_phrases, - link["title"], song): - continue - if ObjManip.individual_word_match(song, link["title"]) \ - >= 0.8 and ObjManip.blank_include(link["title"], - artist): - self.code = link - break - - if self.code is None: - for link in results: - if ObjManip.check_garbage_phrases(garbage_phrases, - link["title"], song): - continue - if ObjManip.blank_include(link["title"], song): - self.code = link - break - - if self.code is None: - song = ObjManip.limit_song_name(song) - - if self.code is None and first is not True: - if tries >= 5: - return self.find_yt_url(song, artist, additional_search, caught_by_google, first=True, tries=tries + 1) - elif additional_search == "lyrics": - return self.find_yt_url(song, artist, additional_search, caught_by_google, first, tries=tries + 1) - - try: - return ("https://youtube.com" + self.code["href"], self.code["title"]) - except TypeError: - if caught_by_google is not True: - # Assuming Google catches you trying to search youtube for music ;) - print("Trying to bypass google captcha.") - return self.find_yt_url(song=song, artist=artist, additional_search=additional_search, caught_by_google=True, tries=tries + 1) - elif caught_by_google is True and first is not True: - return self.find_yt_url(song, artist, additional_search, caught_by_google, first=True, tries=tries + 1) - - def album(self, title, artist=None): # Alias for spotify_list("album", ..) - return self.spotify_list("album", title=title, artist=artist) - - def playlist(self, title, username): - # Alias for `spotify_list("playlist", ...)` - return self.spotify_list("playlist", title=title, username=username) - - def spotify_list(self, type=None, title=None, username=None, artist=None): - try: - if not type: - type = self.args["type"] - if not title: - title = self.args["list_title"] - if not username and type == "playlist": - username = self.args["username"] - except KeyError: - raise ValueError("Must specify type/title/username in `args` \ -with init, or in method arguments.") - - if not self.type: - self.type = type - - if type == "album": - search = title - if "artist" in self.args: - search += " " + self.args["artist"] - list_of_lists = self.spotify.search(q=search, type="album") - list_of_lists = list_of_lists["albums"]["items"] - elif type == "playlist": - try: - list_of_lists = self.spotify.user_playlists(username)["items"] - except spotipy.client.SpotifyException: - print("No user was found by that name.") - return False - - if len(list_of_lists) > 0: - the_list = None - for list_ in list_of_lists: - if Config.parse_exact(self) == True: - if list_["name"].encode("utf-8") == title.encode("utf-8"): - if Config.parse_artist(self): - if list_["artists"][0]["name"].encode("utf-8") == \ - Config.parse_artist(self).encode('utf-8'): - the_list = self.spotify.album(list_["uri"]) - break - else: - if type == "album": - the_list = self.spotify.album(list_["uri"]) - else: - the_list = self.spotify.user_playlist( - list_["owner"]["id"], list_["uri"]) - the_list["artists"] = [{"name": username}] - break - - else: - if ObjManip.blank_include(list_["name"], title): - if Config.parse_artist(self): - if ObjManip.blank_include(list_["artists"][0]["name"], - Config.parse_artist(self)): - the_list = self.spotify.album(list_["uri"]) - break - else: - if type == "album": - the_list = self.spotify.album(list_["uri"]) - else: - the_list = self.spotify.user_playlist( - list_["owner"]["id"], list_["uri"]) - the_list["artists"] = [{"name": username}] - break - if the_list is not None: - YdlUtils.clear_line() - - print(self.args["hook-text"].get("list") - .format(type.title(), the_list["name"].encode("utf-8"), - the_list["artists"][0]["name"].encode("utf-8"))) - - compilation = "" - if type == "album": - tmp_artists = [] - - for track in the_list["tracks"]["items"]: - tmp_artists.append(track["artists"][0]["name"]) - tmp_artists = list(set(tmp_artists)) - if len(tmp_artists) > 1: - compilation = "1" - - tracks = [] - file_prefix = "" - - for track in the_list["tracks"]["items"]: - if type == "playlist": - # For post-processors - self.playlist_title = the_list["name"] - - file_prefix = str(len(tracks) + 1) + " - " - track = track["track"] - album = self.spotify.album(track["album"]["uri"]) - elif type == "album": - file_prefix = str(track["track_number"]) + " - " - track = self.spotify.track(track["uri"]) - album = the_list - - data = { - "name": track["name"], - "artist": track["artists"][0]["name"], - "album": album["name"], - "genre": parse_genre( - self.spotify.artist(track["artists"][0]["uri"] - )["genres"]), - "track_number": track["track_number"], - "disc_number": track["disc_number"], - "album_art": album["images"][0]["url"], - "compilation": compilation, - "file_prefix": file_prefix, - } - - tracks.append(data) - - locations = self.list(tracks) - return locations - # return self.post_processing(locations) - - print("Could not find any lists.") - return False - - def list(self, list_data): - locations = [] - # with open(".irs-download-log", "w+") as file: - # file.write(format_download_log_data(list_data)) - - for track in list_data: - loc = self.song(track["name"], track["artist"], track) - - if loc is not False: - # update_download_log_line_status(track, "downloaded") - locations.append(loc) - - if self.type in ("album", "playlist"): - return self.post_processing(locations) - - # os.remove(".irs-download-log") - return locations - - def parse_song_data(self, song, artist): - album, track = find_album_and_track(song, artist, self.spotify) - if album is False: - return {} - - album = self.spotify.album(album["uri"]) - track = self.spotify.track(track["uri"]) - genre = self.spotify.artist(album["artists"][0]["uri"])["genres"] - - return { - "name": track["name"], - "artist": track["artists"][0]["name"], - "album": album["name"], - "album_art": album["images"][0]["url"], - "genre": parse_genre(genre), - "track_number": track["track_number"], - "disc_number": track["disc_number"], - - # If this method is being called, it's not a compilation - "compilation": "", - # And therefore, won't have a prefix - "file_prefix": "" - } - - def song(self, song, artist, data={}): - # "data" comes from "self.parse_song_data"'s layout - - if not self.type: - self.type = "song" - - try: - if not song: - song = self.args["song_title"] - if not artist: - artist = self.args["artist"] - except KeyError: - raise ValueError("Must specify song_title/artist in `args` with \ -init, or in method arguments.") - - if data == {}: - data = self.parse_song_data(song, artist) - if data != {}: - song = data["name"] - artist = data["artist"] - - if "file_prefix" not in data: - data["file_prefix"] = "" - - video_url, video_title = self.find_yt_url(song, artist) - - if sys.version_info[0] == 2: - print(self.args["hook-text"].get("song").decode().format(song, - artist)) - else: - print(self.args["hook-text"].get("song").format(song, artist)) - - file_name = data["file_prefix"] + ObjManip.blank(song, False) + ".mp3" - ydl_opts = { - 'format': 'bestaudio/best', - 'postprocessors': [{ - 'key': 'FFmpegExtractAudio', - 'preferredcodec': 'mp3', - 'preferredquality': '192', - }], - 'logger': YdlUtils.MyLogger(), - 'progress_hooks': [YdlUtils.my_hook], - 'output': "tmp_file", - 'prefer-ffmpeg': True, - 'ffmpeg_location': os.path.expanduser("~/.irs/bin/"), - } - - with youtube_dl.YoutubeDL(ydl_opts) as ydl: - ydl.download([video_url]) - - for file in glob.glob("./*%s*" % video_url.split("/watch?v=")[-1]): - shutil.move(file, file_name) - - # Ease of Variables (C) (patent pending) (git yer filthy hands off) - # [CENSORED BY THE BAD CODE ACT] - # *5 Minutes Later* - # Deprecated. It won't be the next big thing. :( - - m = Metadata(file_name) - - m.add_tag("comment", 'URL: "%s"\nVideo Title: "%s"' % - (video_url, video_title)) - if len(data.keys()) > 1: - m.add_tag("title", data["name"]) - m.add_tag("artist", data["artist"]) - m.add_tag("album", data["album"]) - m.add_tag("genre", data["genre"]) - m.add_tag("tracknumber", str(data["track_number"])) - m.add_tag("discnumber", str(data["disc_number"])) - m.add_tag("compilation", data["compilation"]) - m.add_album_art(str(data["album_art"])) - else: - print("Could not find metadata.") - m.add_tag("title", song) - m.add_tag("artist", artist) - - if self.type == "song": - return self.post_processing([file_name]) - - return file_name diff --git a/irs/search/__init__.py b/irs/search/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/irs/search/spotify.py b/irs/search/spotify.py new file mode 100644 index 0000000..8628c53 --- /dev/null +++ b/irs/search/spotify.py @@ -0,0 +1,115 @@ +import os +import re + +import spotipy +from spotipy.oauth2 import SpotifyClientCredentials + + +class SpotifySearcher(object): + """Searches spotify for song, album, and playlist metadata.""" + + def authorize(self, client_id=None, client_secret=None): + """Authorizes this class with spotify using client ids + :rtype: returns self class + """ + + # TODO: remove these when you finish config files + if not client_id: + client_id = os.environ["SPOTIFY_CLIENT_ID"] + if not client_secret: + client_secret = os.environ["SPOTIFY_CLIENT_SECRET"] + + try: + creds = SpotifyClientCredentials(client_id, client_secret) + self.authorized = True + self.spotify = spotipy.Spotify(client_credentials_manager=creds) + except Exception: + self.authorized = False + self.spotify = spotipy.Spotify() + + return self + + def find_song(self, song_title, artist_name, limit=50, offset=0): + """Searches spotify for a song and grabs its metadata + :param song_title: a string, the title of the song you're looking for + :param artist_name: a string, the artist of the above song + :rtype: a dictionary of metadata about the song + """ + songs = self.spotify.search(q=song_title, type="track")["tracks"] + + for song in songs["items"]: + if _simplify(song_title) in _simplify(song["name"]) and \ + _simplify(artist_name) in _simplify(song["artists"][0]["name"]): + return song + + if songs['next']: + return self.find_song(song_title, artist_name, + offset=offset + limit) + else: + print("There were no songs found by that name with that artist") + + def find_album(self, album_title, artist_name=None, limit=50, offset=0): + """Searches spotify for an album and grabs its contents and metadata + :param album_title: a string, the title of the album + :param artist_name: a string, the name of the artist of the album + :rtype: a dictionary of metadata about the album + """ + query = album_title + if artist_name: + query += " " + artist_name + albums = self.spotify.search(q=query, type="album")['albums'] + + for album in albums['items']: + if _simplify(album_title) in _simplify(album["name"]): + return self.spotify.album(album['uri']) + + if albums['next']: + return self.find_album(album_title, artist_name, + offset=offset + limit) + else: + print("There were no albums found by that name with that artist") + + def find_playlist(self, playlist_title, username, limit=50, offset=0): + """Searches spotify for a playlist and grabs its contents and metadata + :param playlist_title: a string, the title of the playlist + :param username: a string, the username of the playlist creator/owner + :rtype: a dictionary of metadata about the playlist + """ + playlists = [] + playlists = self.spotify.user_playlists(username, limit, offset) + + for playlist in playlists['items']: + if _simplify(playlist_title) in _simplify(playlist['name']): + return self.spotify.user_playlist(username, playlist['id']) + + if playlists['next']: + return self.find_playlist(playlist_title, username, + offset=offset + limit) + else: + print("There were no playlists by that name found.") + + def artist(self, artist_uri): + """Gets artist metadata from uri + :param artist_uri: the spotify uri for the artist + :rtype: a dict of info about the artist + """ + return self.spotify.artist(artist_uri) + + def song(self, song_uri): + """Gets song metadata from uri + :param song_uri: the spotify uri for the artist + :rtype: a dict of info about the artist + """ + return self.spotify.track(song_uri) + + + +# TODO: export this function to a utilities file +def _simplify(string): + """Lowercases and strips all non alphanumeric characters from the string + :param string: a string to be modified + :rtype: the modified string + """ + if type(string) == bytes: + string = string.decode() + return re.sub(r'[^a-zA-Z0-9]+', '', string.lower()) \ No newline at end of file diff --git a/irs/search/youtube.py b/irs/search/youtube.py new file mode 100644 index 0000000..3cecc33 --- /dev/null +++ b/irs/search/youtube.py @@ -0,0 +1,191 @@ +import sys +import re + +if sys.version_info[0] >= 3: + from urllib.parse import urlencode + from urllib.request import urlopen +elif sys.version_info[0] < 3: + from urllib import urlencode + from urllib import urlopen +else: + print("Must be using Python 2 or 3") + sys.exit(1) + +from bs4 import BeautifulSoup + + +def find_url(song_title, artist_name, search_terms=None, caught_by_google=False, download_first=False): + """Finds the youtube video url for the requested song. The youtube + query is constructed like this: + " " + so plugging in "Bohemian Rhapsody", "Queen", and "lyrics" would end + up with a search for "Bohemian Rhapsody Queen lyrics" on youtube + :param-required song: song name + :param-required artist: artist name + :param search_terms: any additional search terms you may want to + add to the search query + :param caught_by_google: a boolean, if not false or none, turns on + the captcha catcher + :param download_first: a boolean, if true, downloads first video + that youtube returns + :rtype: A string of the youtube url for the song + """ + + query = artist_name + " " + song_title + if search_terms: + query += " " + search_terms + + encoded_query = urlencode({"search_query": query}) + + url = "http://www.youtube.com/results?" + encoded_query + + soup = _get_url_data(url, caught_by_google) + + # if you want to inspect the html being requested + # print(soup.prettify()) + # with open("index.html", "wb") as f: + # f.write(soup.prettify().encode('utf-8')) + + # Each of the tags in the results list have the following relevant + # attributes: + # "title": the title of the youtube video + # "href": the youtube video code, namely the X's of + # https://www.youtube.com/watch?v=XXXXXXXXXXX + # "class": the classes of the link, used to identify the youtube title + results = _find_links(soup) + + best_guess = None + total_tries_counter = 0 + + if len(results) <= 0: + raise Exception('There were no search results for "{}"'.format(query)) + + if download_first == True: + return "https://youtube.com" + results[0]["href"] + + scores = [] + + for index, link in enumerate(results): + scores.append([ + index, + _score_song(song_title, artist_name, link["title"]), + link["href"] + ]) + + # sort by the score of the song + sorted(scores, key=lambda x: x[1]) + + return "https://youtube.com" + results[scores[0][0]]["href"] + + +def _score_song(song_title, artist_name, video_title): + """Scores the likelihood of the song audio being in the video based off of + the video title. + :param song_title: a string, the title of the song that you're looking for + :param video_title: a string, the title of the video you're analyzing + :rtype: an integer, the score of the song + """ + points = 0 + + song_title = _simplify(song_title) + artist_name = _simplify(artist_name) + video_title = _simplify(video_title) + + if song_title in video_title: + points += 3 + + if artist_name in video_title: + points += 3 + + points -= _count_garbage_phrases(video_title, song_title) + + return points + + +def _simplify(string): + """Lowercases and strips all non alphanumeric characters from the string + :param string: a string to be modified + :rtype: the modified string + """ + if type(string) == bytes: + string = string.decode() + return re.sub(r'[^a-zA-Z0-9]+', '', string.lower()) + + +def _count_garbage_phrases(video_title, song_title): + """Checks if there are any phrases in the title of the video that would + indicate it doesn't have the audio we want + :param string: a string, the youtube video title + :param title: a string, the actual title of the song we're looking for + :rtype: an integer, of the number of bad phrases in the song + """ + + # Garbage phrases found through experiences of downloading the wrong song + # TODO: add this into the config so the user can mess with it if they want + garbage_phrases = ( + "cover album live clean rare version full full album row at " + "@ session how to npr music reimagined hr version" + ).split(" ") + + bad_phrases = 0 + + for gphrase in garbage_phrases: + # make sure we're not invalidating part of the title of the song + if gphrase in song_title.lower(): + continue + + # check if the garbage phrase is not in the video title + if gphrase in video_title.lower(): + bad_phrases += 1 + + return bad_phrases + + +def _find_links(soup): + """Finds youtube video links in the html soup + :param soup: a BeautifulSoup(...) element + :rtype: returns a list of valid youtube video links + """ + return list(filter(None, map(_find_link, soup.find_all("a")))) + + +def _find_link(link): + """Tests html tags to see if they contain a youtube video link. + Should be used only with the find_links function in a map func. + :param link: accepts an element from BeautifulSoup(...).find_all(...) + :rtype: returns the link if it's an actual video link, otherwise, None + """ + try: + class_to_check = str(" ".join(link["class"])) + except KeyError: + return + + # these classes are found by inspecting the html soup of a youtube search. + valid_classes = [ + "yt-simple-endpoint style-scope ytd-video-renderer", + ("yt-uix-tile-link yt-ui-ellipsis yt-ui-ellipsis-2 " + "yt-uix-sessionlink spf-link ") + ] + + try: + # Make sure it's not a playlist + if "&list=" in link["href"]: + return + + for valid_class in valid_classes: + if valid_class in class_to_check: + return link + except KeyError: + pass + + +# TODO: build in the captcha cheater if the user is "caught" by google +def _get_url_data(url, caught_by_google): + """Gets parsed html from the specified url + :param url: A string, the url to request and parse. + :param caught_by_google: A boolean, will open and use the captcha + cheat to get around google's captcha. + :rtype: A BeautifulSoup class + """ + html_content = urlopen(url).read() + return BeautifulSoup(html_content, 'html.parser') \ No newline at end of file diff --git a/irs/setup_binaries.py b/irs/setup_binaries.py deleted file mode 100644 index 23992b3..0000000 --- a/irs/setup_binaries.py +++ /dev/null @@ -1,17 +0,0 @@ -import ydl_binaries -from shutil import copyfile -import os -import inspect -import irs - - -def setup(): - bin_path = os.path.expanduser("~/.irs/bin/") - - ydl_binaries.download_ffmpeg(bin_path) - ydl_binaries.update_ydl(bin_path) - - config_file = os.path.expanduser("~/.irs/config_.py") - if not os.path.isfile(config_file): - copyfile(os.path.dirname(inspect.getfile(irs)) + "/config_preset.py", - config_file) diff --git a/irs/utils.py b/irs/utils.py deleted file mode 100644 index 32830fa..0000000 --- a/irs/utils.py +++ /dev/null @@ -1,480 +0,0 @@ -# _*_ coding:utf-8 _*_ - - -# ======= -# Imports -# ======= - -# Static Method Hook -import inspect - -# And Now For Something Completely Different -import os -import sys -import re -from time import sleep -import pkg_resources - -# Config File and Flags -if sys.version_info[0] == 2: - import config - CONFIG = config.CONFIG -else: - from irs.config import CONFIG - - -# ================== -# Static Method Hook -# ================== - -def staticmethods(cls): - for name, method in inspect.getmembers(cls, inspect.ismethod): - setattr(cls, name, staticmethod(method.__func__)) - return cls - - -# ========================= -# Youtube-DL Logs and Hooks -# ========================= - -@staticmethods -class YdlUtils: - def clear_line(): - sys.stdout.write("\x1b[2K\r") - - class MyLogger(object): - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - print(msg) - - def my_hook(d): - if d['status'] == 'finished': - print(CONFIG["converting"]) - - -# ================================ -# Object Manipulation and Checking -# ================================ - -def set_encoding(ld, encoding): # ld => list or dictionary with strings in it - if type(ld) == dict: - for k in ld: - if type(ld[k]) == dict or type(ld[k]) == list: - ld[k] = set_encoding(ld[k], encoding) - elif type(ld[k]) == str: - ld[k] = encoding(ld[k]) - elif type(ld) == list: - for index, datum in enumerate(ld): - if type(datum) == str: - ld[index] = encoding(datum) - elif type(ld[k]) == dict or type(ld[k]) == list: - ld[k] = set_encoding(ld[k], encoding) - return ld - - -@staticmethods -class ObjManip: # Object Manipulation - def limit_song_name(song): - bad_phrases = "remaster remastered master".split(" ") - # I have "master" here because Spotify actually sometimes mispells - # stuff and it is hella annoying, so this was my solution - for phrase in bad_phrases: - if ObjManip.blank_include(song.split(" - ")[-1], phrase): - return song.split(" - ")[0] - return song - - def check_garbage_phrases(phrases, string, title): - for phrase in phrases: - if phrase in string.lower(): - if phrase not in title.lower(): - return True - return False - - def blank(string, downcase=True, remove_and=True): - if downcase: - string = string.lower() - if remove_and: - string = string.replace("and", "") - import re - regex = re.compile('[^a-zA-Z0-9\ ]') - if sys.version_info == 2: - string = regex.sub('', string.decode("utf8")) - return ' '.join(string.decode().split()) - else: - string = regex.sub('', string) - return ' '.join(string.split()) - - def blank_include(this, includes_this): - this = ObjManip.blank(this) - includes_this = ObjManip.blank(includes_this) - if includes_this in this: - return True - return False - - def individual_word_match(match_against, match): - match_against = ObjManip.blank(match_against).split(" ") - match = ObjManip.blank(match).split(" ") - matched = [] - for match_ag in match_against: - for word in match: - if match_ag == word: - matched.append(word) - return (float(len(set(matched))) / float(len(match_against))) - - def flatten(l): - flattened_list = [] - for x in l: - if type(x) != str: - for y in x: - flattened_list.append(y) - else: - flattened_list.append(x) - return flattened_list - - def remove_none_values(d): - new_d = d - for x in list(d.keys()): - if type(new_d[x]) is list: - new_d[x] = ObjManip.remove_none_values(d[x]) - elif new_d[x] is None: - del new_d[x] - return new_d - - # ld => a list or dictionary with strings in it - def set_utf8_encoding(ld): - return set_encoding(ld, lambda x: x.encode('utf-8')) - - def set_encoding(*args): - return set_encoding(*args) - -# ======================================== -# Download Log Reading/Updating/Formatting -# ======================================== - - -@staticmethods -class DLog: - def format_download_log_line(t, download_status="not downloaded"): - return (" @@ ".join([t["name"], t["artist"], t["album"]["id"], - str(t["genre"]), t["track_number"], t["disc_number"], - t["compilation"], t["file_prefix"], download_status])) - - def format_download_log_data(data): - lines = [] - for track in data: - lines.append(DLog.format_download_log_line(track)) - return "\n".join(lines) - - def read_download_log(spotify): - data = [] - with open(".irs-download-log", "r") as file: - for line in file: - line = line.split(" @@ ") - data.append({ - "name": line[0], - "artist": line[1], - "album": spotify.album(line[2]), - "genre": eval(line[3]), - "track_number": line[4], - "disc_number": line[5], - "compilation": bool(line[6]), - "file_prefix": line[7], - }) - return data - - def update_download_log_line_status(track, status="downloaded"): - line_to_find = DLog.format_download_log_line(track) - with open(".irs-download-log", "r") as input_file: - with open(".irs-download-log", "w") as output_file: - for line in input_file: - if line == line_to_find: - output_file.write( - DLog.format_download_log_line(track, status)) - else: - output_file.write(line) - - -# =========================================== -# And Now, For Something Completely Different -# =========================================== -# (It's for the CLI) - -try: - COLS = int(os.popen('tput cols').read().strip("\n")) -except: - COLS = 80 - -if sys.version_info[0] == 2: - def input(string): - return raw_input(string) - - -def code(code1): - return "\x1b[%sm" % str(code1) - - -def no_colors(string): - return re.sub("\x1b\[\d+m", "", string) - - -def center_colors(string, cols): - return no_colors(string).center(cols).replace(no_colors(string), string) - - -def decode_utf8(string): - if sys.version_info[0] == 3: - return string.encode("utf8", "strict").decode() - elif sys.version_info[0] == 2: - return string.decode("utf8") - - -def center_unicode(string, cols): - tmp_chars = "X" * len(decode_utf8(string)) - chars = center_colors(tmp_chars, cols) - return chars.replace(tmp_chars, string) - - -def center_lines(string, cols, end="\n"): - lines = [] - for line in string.split("\n"): - lines.append(center_unicode(line, cols)) - return end.join(lines) - - -def flush_puts(msg, time=0.01): - # For slow *burrrp* scroll text, Morty. They-They just love it, Morty. - # When they see this text. Just slowwwly extending across the page. Mmm, - # mmm. You just give the time for how *buurp* slow you wa-want it, Morty. - # It works with colors and escape characters too, Morty. - # Your grandpa's a genius *burrrp* Morty - def check_color(s): - if "\x1b" not in s: - new = list(s) - else: - new = s - return new - msg = re.split("(\x1b\[\d+m)", msg) - msg = list(filter(None, map(check_color, msg))) - msg = ObjManip.flatten(msg) - for char in msg: - if char not in (" ", "", "\n") and "\x1b" not in char: - sleep(time) - sys.stdout.write(char) - sys.stdout.flush() - print("") - - -BOLD = code(1) -END = code(0) -RED = code(31) -GREEN = code(32) -YELLOW = code(33) -BLUE = code(34) -PURPLE = code(35) -CYAN = code(36) -GRAY = code(37) -BRED = RED + BOLD -BGREEN = GREEN + BOLD -BYELLOW = YELLOW + BOLD -BBLUE = BLUE + BOLD -BPURPLE = PURPLE + BOLD -BCYAN = CYAN + BOLD -BGRAY = GRAY + BOLD - - -def banner(): - title = (BCYAN + center_lines("""\ -██╗██████╗ ███████╗ -██║██╔══██╗██╔════╝ -██║██████╔╝███████╗ -██║██╔══██╗╚════██║ -██║██║ ██║███████║ -╚═╝╚═╝ ╚═╝╚══════╝\ -""", COLS) + END) - for num in range(0, 6): - os.system("clear || cls") - if num % 2 == 1: - print(BRED + center_unicode("🚨 🚨 🚨 🚨 🚨 \r", COLS)) - else: - print("") - print(title) - sleep(0.3) - flush_puts(center_colors("{0}Ironic Redistribution System ({1}IRS{2})" - .format(BYELLOW, BRED, BYELLOW), COLS)) - - flush_puts(center_colors("{0}Made with 😈 by: {1}Kepoor Hampond \ -({2}kepoorhampond{3})".format(BBLUE, BYELLOW, BRED, BYELLOW) + END, COLS)) - - flush_puts(center_colors("{0}Version: {1}".format(BBLUE, BYELLOW) + - pkg_resources.get_distribution("irs").version, COLS)) - - -def menu(unicode, time=0.01): - flush_puts("Choose option from menu:", time) - flush_puts("\t[{0}song{1}] Download Song".format(BGREEN, END), time) - flush_puts("\t[{0}album{1}] Download Album".format(BGREEN, END), time) - flush_puts("\t[{0}{1}{2}] Download Playlist" - .format(BGREEN, unicode[-1], END), time) - flush_puts("\t[{0}help{1}] Print This Menu".format(BGREEN, END), time) - flush_puts("\t[{0}exit{1}] Exit IRS".format(BGREEN, END), time) - print("") - - -def console(ripper): - banner() - print(END) - if ripper.authorized is True: - unicode = [BGREEN + "✔" + END, "list"] - elif ripper.authorized is False: - unicode = [BRED + "✘" + END] - flush_puts("[{0}] Authenticated with Spotify".format(unicode[0])) - print("") - menu(unicode) - while True: - try: - choice = input("{0}irs{1}>{2} ".format(BBLUE, BGRAY, END)) - - if choice in ("exit", "e"): - raise KeyboardInterrupt - - try: - if choice in ("song", "s"): - song_name = input("Song name{0}:{1} ".format(BBLUE, END)) - artist_name = input("Artist name{0}:{1} " - .format(BBLUE, END)) - ripper.song(song_name, artist_name) - - elif choice in ("album", "a"): - album_name = input("Album name{0}:{1} ".format(BBLUE, END)) - ripper.spotify_list("album", album_name) - - elif choice in ("list", "l") and ripper.authorized is True: - username = input("Spotify Username{0}:{1} " - .format(BBLUE, END)) - list_name = input("Playlist Name{0}:{1} " - .format(BBLUE, END)) - ripper.spotify_list("playlist", list_name, username) - - elif choice in ("help", "h", "?"): - menu(unicode, 0) - except (KeyboardInterrupt, EOFError): - print("") - pass - - except (KeyboardInterrupt, EOFError): - sys.exit(0) - - -""" -# ===================== -# Config File and Flags -# ===================== - -def check_sources(ripper, key, default=None, environment=False, where=None): - if where is not None: - tmp_args = ripper.args.get(where) - else: - tmp_args = ripper.args - - if tmp_args.get(key): - return tmp_args.get(key) -""" - - -# =========== -# CONFIG FILE -# =========== - -def check_sources(ripper, key, default=None, environment=False, where=None): - # tmp_args = ripper.args - # if where is not None and ripper.args.get(where): - # tmp_args = ripper.args.get("where") - - if ripper.args.get(key): - return ripper.args.get(key) - elif CONFIG.get(key): - return CONFIG.get(key) - elif os.environ.get(key) and environment is True: - return os.environ.get(key) - else: - return default - - -@staticmethods -class Config: - - def parse_spotify_creds(ripper): - CLIENT_ID = check_sources(ripper, "SPOTIFY_CLIENT_ID", - environment=True) - CLIENT_SECRET = check_sources(ripper, "SPOTIFY_CLIENT_SECRET", - environment=True) - return CLIENT_ID, CLIENT_SECRET - - def parse_search_terms(ripper): - search_terms = check_sources(ripper, "additional_search_terms", - "lyrics") - return search_terms - - def parse_artist(ripper): - artist = check_sources(ripper, "artist") - return artist - - def parse_directory(ripper): - directory = check_sources(ripper, "custom_directory", - where="post_processors") - if directory is None: - directory = check_sources(ripper, "custom_directory", "~/Music") - return directory.replace("~", os.path.expanduser("~")) - - def parse_default_flags(default=""): - if CONFIG.get("default_flags"): - args = sys.argv[1:] + CONFIG.get("default_flags") - else: - args = default - return args - - def parse_organize(ripper): - organize = check_sources(ripper, "organize") - if organize is None: - return check_sources(ripper, "organize", False, - where="post_processors") - else: - return True - - def parse_exact(ripper): - exact = check_sources(ripper, "exact") - if exact in (True, False): - return exact - - - -#============== -# Captcha Cheat -#============== -# I basically consider myself a genius for this snippet. - -from splinter import Browser -from time import sleep - - -@staticmethods -class CaptchaCheat: - def cheat_it(url, t=1): - executable_path = {'executable_path': '/usr/local/bin/chromedriver'} - with Browser('chrome', **executable_path) as b: - b.visit(url) - sleep(t) - while CaptchaCheat.strip_it(b.evaluate_script("document.URL")) != CaptchaCheat.strip_it(url): - sleep(t) - return b.evaluate_script("document.getElementsByTagName('html')[0].innerHTML") - - def strip_it(s): - s = s.encode("utf-8") - s = s.strip("http://") - s = s.strip("https://") - return s diff --git a/setup.py b/setup.py index 8a547bb..96b4e66 100644 --- a/setup.py +++ b/setup.py @@ -2,22 +2,23 @@ from setuptools import setup setup( name = 'irs', - version = '6.7.7', - description = 'A music downloader that just gets metadata.', + version = '7.0.0', + description = 'A music downloader that gets metadata too.', url = 'https://github.com/kepoorhampond/irs', author = 'Kepoor Hampond', author_email = 'kepoorh@gmail.com', license = 'GPL', - packages = ['irs'], + packages = ['irs', 'irs.search', 'irs.interact', 'irs.glue', + 'irs.install', 'irs.cli'], install_requires = [ - 'bs4', - 'mutagen', - 'requests', - 'spotipy', - 'ydl-binaries', - 'splinter' + 'bs4', # HTML parsing + 'mutagen', # MP3 tags + 'argparse', # CLI arg parsing + 'spotipy', # Interfacing w/ Spotify API + 'ydl-binaries', # Downloading ffmpeg/ffprobe binaries + 'pyyaml' # Config files done simply ], entry_points = { - 'console_scripts': ['irs = irs.cli:main'], + 'console_scripts': ['irs = irs.cli.cli:main'], }, ) diff --git a/tests/README.md b/tests/README.md deleted file mode 100644 index c8e0231..0000000 --- a/tests/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# Travis CI tests are wierd -Tests with `ffmpeg` are wierd on Travis. It's a fact that I've had to deal with for 25 commits in total. So, the only songs I've found *so far* that work with ffmpeg are from the album `Da Frame 2R / Matador` by `Arctic Monkeys`. It's really lame, but don't mess around with them. diff --git a/tests/album.py b/tests/album.py deleted file mode 100644 index 271da69..0000000 --- a/tests/album.py +++ /dev/null @@ -1,5 +0,0 @@ -from irs.ripper import Ripper - -print ("[*] Testing `album.py`") -Ripper().spotify_list("album", "Da Frame 2R / Matador") -print ("[+] Passed!") \ No newline at end of file diff --git a/tests/playlist.py b/tests/playlist.py deleted file mode 100644 index d870544..0000000 --- a/tests/playlist.py +++ /dev/null @@ -1,5 +0,0 @@ -from irs.ripper import Ripper - -print ("[*] Testing `playlist.py`") -Ripper().spotify_list("playlist", "IRS Testing", "prakkillian") -print ("[+] Passed!") \ No newline at end of file diff --git a/tests/post_processors.py b/tests/post_processors.py deleted file mode 100644 index 5aacc7d..0000000 --- a/tests/post_processors.py +++ /dev/null @@ -1,22 +0,0 @@ -from irs.ripper import Ripper -import os - -print ("[*] Testing `post_processors.py`") - -if not os.path.exists("test_dir"): - os.makedirs("test_dir") -Ripper({ - "post_processors": { - "custom_directory": "test_dir/", - "organize": True, - } -}).album("Da Frame 2R / Matador") - -Ripper({ - "post_processors": { - "custom_directory": "test_dir/", - "organize": True, - } -}).playlist("IRS Testing", "prakkillian") - -print ("[+] Passed!") \ No newline at end of file diff --git a/tests/song.py b/tests/song.py deleted file mode 100644 index 1950ba3..0000000 --- a/tests/song.py +++ /dev/null @@ -1,5 +0,0 @@ -from irs.ripper import Ripper - -print ("[*] Testing `song.py`") -Ripper().song("Da Frame 2R", "Arctic Monkeys") -print ("[+] Passed!")