Merge pull request #1 from connormason/connor_first_pass

Add .aac -> .mp3 conversion, re-work some logging

youtube_dl/YoutubeDL.py

@@ -12,6 +12,7 @@ import io
 import itertools
 import json
 import locale
+import logging
 import operator
 import os
 import platform
@@ -24,6 +25,8 @@ import time
 import tokenize
 import traceback
 import random
+from typing import Any
+from typing import cast
 
 try:
     from ssl import OPENSSL_VERSION
@@ -130,6 +133,10 @@ from .version import __version__
 if compat_os_name == 'nt':
     import ctypes
 
+logger = logging.getLogger('soundcloudutil.downloader')
+
+TAGGED_LOG_MSG_REGEX = re.compile(r'^\[(?P<tag>\w+)(:(?P<subtag>\w+))?\]\s*(?P<msg>.+)$')
+
 
 def _catch_unsafe_file_extension(func):
     @functools.wraps(func)
@@ -536,37 +543,53 @@ class YoutubeDL(object):
                       for _ in range(line_count))
         return res[:-len('\n')]
 
-    def to_screen(self, message, skip_eol=False):
+    def to_screen(self, message, skip_eol: bool = False):
         """Print message to stdout if not in quiet mode."""
         return self.to_stdout(message, skip_eol, check_quiet=True)
 
-    def _write_string(self, s, out=None, only_once=False, _cache=set()):
+    @property
-        if only_once and s in _cache:
+    def user_logger(self) -> logging.Logger | None:
-            return
+        return cast(logging.Logger | None, self.params.get('logger'))
+
+    def _write_string(self, s: str, out: io.TextIOWrapper | None = None) -> None:
         write_string(s, out=out, encoding=self.params.get('encoding'))
-        if only_once:
-            _cache.add(s)
 
-    def to_stdout(self, message, skip_eol=False, check_quiet=False, only_once=False):
+    def to_stdout(self, message, skip_eol: bool = False, check_quiet: bool = False):
         """Print message to stdout if not in quiet mode."""
-        if self.params.get('logger'):
+        quiet = check_quiet and self.params.get('quiet', False)
-            self.params['logger'].debug(message)
-        elif not check_quiet or not self.params.get('quiet', False):
-            message = self._bidi_workaround(message)
-            terminator = ['\n', ''][skip_eol]
-            output = message + terminator
 
-            self._write_string(output, self._screen_file, only_once=only_once)
+        debug: bool
-
+        if message.startswith(f'[debug]'):
-    def to_stderr(self, message, only_once=False):
+            debug = True
-        """Print message to stderr."""
+            message = message.removeprefix('[debug]').lstrip()
-        assert isinstance(message, compat_str)
+        elif message.startswith('[info]'):
-        if self.params.get('logger'):
+            debug = False
-            self.params['logger'].error(message)
+            message = message.removeprefix('[info]').lstrip()
+        elif quiet:
+            debug = True
         else:
-            message = self._bidi_workaround(message)
+            debug = False
-            output = message + '\n'
+
-            self._write_string(output, self._err_file, only_once=only_once)
+        _logger = logger
+        if m := TAGGED_LOG_MSG_REGEX.match(message):
+            tag = m.group('tag')
+            subtag = m.group('subtag')
+            _logger_name = f'youtube_dl.{tag}'
+            if m.group('subtag'):
+                _logger_name += f'.{subtag}'
+            _logger = logging.getLogger(_logger_name)
+            message = m.group('msg')
+
+        if debug:
+            _logger.debug(message)
+        else:
+            _logger.info(message)
+
+    def to_stderr(self, message: str) -> None:
+        if self.user_logger is not None:
+            self.user_logger.error(message)
+        else:
+            logger.error(message)
 
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
@@ -645,33 +668,26 @@ class YoutubeDL(object):
             raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
-    def report_warning(self, message, only_once=False):
+    def report_warning(self, message: str, only_once: bool = False, _cache: dict[int, int] | None = None) -> None:
-        '''
+        _cache = _cache or {}
-        Print the message to stderr, it will be prefixed with 'WARNING:'
+        if only_once:
-        If stderr is a tty file the 'WARNING:' will be colored
+            m_hash = hash((self, message))
-        '''
+            m_cnt = _cache.setdefault(m_hash, 0)
-        if self.params.get('logger') is not None:
+            _cache[m_hash] = m_cnt + 1
-            self.params['logger'].warning(message)
+            if m_cnt > 0:
+                return
+
+        if self.user_logger is not None:
+            self.user_logger.warning(message)
         else:
             if self.params.get('no_warnings'):
                 return
-            if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
+            logger.warning(message)
-                _msg_header = '\033[0;33mWARNING:\033[0m'
-            else:
-                _msg_header = 'WARNING:'
-            warning_message = '%s %s' % (_msg_header, message)
-            self.to_stderr(warning_message, only_once=only_once)
 
-    def report_error(self, message, *args, **kwargs):
+    # TODO: re-implement :meth:`trouble` to output tracebacks with RichHandler
-        '''
+    def report_error(self, message: str, *args: Any, **kwargs: Any) -> None:
-        Do the same as trouble, but prefixes the message with 'ERROR:', colored
+        logger.error(message)
-        in red if stderr is a tty file.
+        kwargs['message'] = f'ERROR: {message}'
-        '''
-        if not self.params.get('no_color') and self._err_file.isatty() and compat_os_name != 'nt':
-            _msg_header = '\033[0;31mERROR:\033[0m'
-        else:
-            _msg_header = 'ERROR:'
-        kwargs['message'] = '%s %s' % (_msg_header, message)
         self.trouble(*args, **kwargs)
 
     def write_debug(self, message, only_once=False):
@@ -2663,7 +2679,13 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
-    def _write_info_json(self, label, info_dict, infofn, overwrite=None):
+    def _write_info_json(
+        self,
+        label: str,
+        info_dict: dict[str, Any],
+        infofn: str,
+        overwrite: bool | None = None,
+    ) -> bool | str | None:
         if not self.params.get('writeinfojson', False):
             return False
 
@@ -2683,7 +2705,7 @@ class YoutubeDL(object):
                 return True
             except (OSError, IOError):
                 self.report_error(msg('Cannot write %s to JSON file ', label) + infofn)
-                return
+                return None
 
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):

youtube_dl/__init__.py

@@ -287,6 +287,10 @@ def _real_main(argv=None):
         postprocessors.append({
             'key': 'FFmpegEmbedSubtitle',
         })
+    if opts.aacToMp3:
+        postprocessors.append({
+            'key': 'ConvertAACToMP3PP',
+        })
     if opts.embedthumbnail:
         already_have_thumbnail = opts.writethumbnail or opts.write_all_thumbnails
         postprocessors.append({

youtube_dl/downloader/external.py

@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+import logging
 import os
 import re
 import subprocess
@@ -496,20 +497,31 @@ class FFmpegFD(ExternalFD):
         # as a context manager (newer Python 3.x and compat)
         # Fixes "Resource Warning" in test/test_downloader_external.py
         # [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
-        with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
+        _proc = compat_subprocess_Popen(
+            args,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            text=True,
+            universal_newlines=True,
+            bufsize=1,
+            env=env,
+        )
+        ffmpeg_logger = logging.getLogger('ffmpeg')
+        with _proc as proc:
             try:
+                for line in iter(proc.stdout.readline, ''):
+                    ffmpeg_logger.debug(line.strip())
+
+                proc.stdout.close()
                 retval = proc.wait()
             except BaseException as e:
-                # subprocess.run would send the SIGKILL signal to ffmpeg and the
+                if isinstance(e, KeyError) and (sys.platform != 'win32'):
-                # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+                    process_communicate_or_kill(proc, 'q')
-                # produces a file that is playable (this is mostly useful for live
-                # streams). Note that Windows is not affected and produces playable
-                # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-                if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
-                    process_communicate_or_kill(proc, b'q')
                 else:
                     proc.kill()
                 raise
+
         return retval
 
 

youtube_dl/options.py

@@ -818,6 +818,11 @@ def parseOpts(overrideArguments=None):
         '--no-post-overwrites',
         action='store_true', dest='nopostoverwrites', default=False,
         help='Do not overwrite post-processed files; the post-processed files are overwritten by default')
+    postproc.add_option(
+        '--aac-to-mp3',
+        action='store_true', dest='aacToMp3', default=False,
+        help='Convert AAC files to MP3',
+    )
     postproc.add_option(
         '--embed-subs',
         action='store_true', dest='embedsubtitles', default=False,

youtube_dl/postprocessor/__init__.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 from .embedthumbnail import EmbedThumbnailPP
 from .ffmpeg import (
+    ConvertAACToMP3PP,
     FFmpegPostProcessor,
     FFmpegEmbedSubtitlePP,
     FFmpegExtractAudioPP,
@@ -23,6 +24,7 @@ def get_postprocessor(key):
 
 
 __all__ = [
+    'ConvertAACToMP3PP',
     'EmbedThumbnailPP',
     'ExecAfterDownloadPP',
     'FFmpegEmbedSubtitlePP',

youtube_dl/postprocessor/embedthumbnail.py

@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 
+import logging
 import os
 import subprocess
 
@@ -21,6 +22,9 @@ from ..utils import (
 from ..compat import compat_open as open
 
 
+logger = logging.getLogger('soundcloudutil.downloader')
+
+
 class EmbedThumbnailPPError(PostProcessingError):
     pass
 
@@ -128,6 +132,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 os.remove(encodeFilename(filename))
                 os.rename(encodeFilename(temp_filename), encodeFilename(filename))
         else:
-            raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
+            logger.warning('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
+            # raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.')
 
         return [], info

youtube_dl/postprocessor/ffmpeg.py

@@ -4,7 +4,8 @@ import os
 import subprocess
 import time
 import re
-
+from pathlib import Path
+from typing import Any
 
 from .common import AudioConversionError, PostProcessor
 
@@ -651,3 +652,26 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 }
 
         return sub_filenames, info
+
+
+class ConvertAACToMP3PP(FFmpegPostProcessor):
+    """
+    Custom post processor that converts .aac files to .mp3 files
+    """
+    def run(self, info: dict[str, Any]) -> tuple[list[str], dict[str, Any]]:
+        if info['ext'] == 'aac':
+            aac_path = Path(info['filepath'])
+            mp3_path = aac_path.with_suffix('.mp3')
+
+            self._downloader.to_screen('[ffmpeg] Converting .aac to .mp3')
+            options: list[str] = [
+                '-codec:a', 'libmp3lame',
+                '-qscale:a', '0',
+            ]
+            self.run_ffmpeg(str(aac_path), str(mp3_path), options)
+            aac_path.unlink()
+
+            info['filepath'] = str(mp3_path)
+            info['ext'] = 'mp3'
+
+        return [], info