From ab6ade9c4c0c0ef3f3c5ceb3f3c03619d1899309 Mon Sep 17 00:00:00 2001 From: "kerry.nich" Date: Tue, 22 Mar 2022 19:19:38 -0400 Subject: [PATCH] Added progress hook updates for postprocessing --- youtube_dl/YoutubeDL.py | 13 +++- youtube_dl/postprocessor/common.py | 10 +++ youtube_dl/postprocessor/embedthumbnail.py | 8 +++ youtube_dl/postprocessor/execafterdownload.py | 4 ++ youtube_dl/postprocessor/ffmpeg.py | 63 +++++++++++++++++++ youtube_dl/postprocessor/metadatafromtitle.py | 4 ++ youtube_dl/postprocessor/xattrpp.py | 5 ++ 7 files changed, 104 insertions(+), 3 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 69736acff..3c11b88c9 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -234,11 +234,13 @@ class YoutubeDL(object): postprocessor. progress_hooks: A list of functions that get called on download progress, with a dictionary with the entries - * status: One of "downloading", "error", or "finished". + * status: One of "downloading", "error", "finished", + or "postprocessed". Check this first and ignore unknown values. - If status is one of "downloading", or "finished", the - following properties may also be present: + If status is one of "downloading", "finished", or + "postprocessed", the following properties may also be + present: * filename: The final filename (always present) * tmpfilename: The filename we're currently writing to * downloaded_bytes: Bytes on disk @@ -253,6 +255,9 @@ class YoutubeDL(object): downloaded video fragment. * fragment_count: The number of fragments (= individual files that will be merged) + * postprocessor: The specific postprocessor that ran. + See youtube_dl/postprocessor/__init__.py + for a list of possibilities. Progress hooks are guaranteed to be called at least once (with status "finished") if the download is successful. @@ -2110,6 +2115,8 @@ class YoutubeDL(object): pps_chain.extend(ie_info['__postprocessors']) pps_chain.extend(self._pps) for pp in pps_chain: + for ph in self._progress_hooks: + pp.add_progress_hook(ph) files_to_delete = [] try: files_to_delete, info = pp.run(info) diff --git a/youtube_dl/postprocessor/common.py b/youtube_dl/postprocessor/common.py index 599dd1df2..22a27127f 100644 --- a/youtube_dl/postprocessor/common.py +++ b/youtube_dl/postprocessor/common.py @@ -33,6 +33,7 @@ class PostProcessor(object): def __init__(self, downloader=None): self._downloader = downloader + self._progress_hooks = [] def set_downloader(self, downloader): """Sets the downloader for this PP.""" @@ -64,6 +65,15 @@ class PostProcessor(object): def _configuration_args(self, default=[]): return cli_configuration_args(self._downloader.params, 'postprocessor_args', default) + def _hook_progress(self, status): + for ph in self._progress_hooks: + ph(status) + + def add_progress_hook(self, ph): + # See YoutubeDl.py (search for progress_hooks) for a description of + # this interface + self._progress_hooks.append(ph) + class AudioConversionError(PostProcessingError): pass diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py index 3990908b6..20371baaa 100644 --- a/youtube_dl/postprocessor/embedthumbnail.py +++ b/youtube_dl/postprocessor/embedthumbnail.py @@ -127,4 +127,12 @@ class EmbedThumbnailPP(FFmpegPostProcessor): else: raise EmbedThumbnailPPError('Only mp3 and m4a/mp4 are supported for thumbnail embedding for now.') + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) + return [], info diff --git a/youtube_dl/postprocessor/execafterdownload.py b/youtube_dl/postprocessor/execafterdownload.py index 64dabe790..a546d51c9 100644 --- a/youtube_dl/postprocessor/execafterdownload.py +++ b/youtube_dl/postprocessor/execafterdownload.py @@ -27,5 +27,9 @@ class ExecAfterDownloadPP(PostProcessor): if retCode != 0: raise PostProcessingError( 'Command returned error code %d' % retCode) + self._hook_progress({ + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], information diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py index 9f76c9d4e..3f358f2a1 100644 --- a/youtube_dl/postprocessor/ffmpeg.py +++ b/youtube_dl/postprocessor/ffmpeg.py @@ -349,6 +349,14 @@ class FFmpegExtractAudioPP(FFmpegPostProcessor): new_path, time.time(), information['filetime'], errnote='Cannot update utime of audio file') + fsize = os.path.getsize(new_path) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': new_path, + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) + return [path], information @@ -372,6 +380,13 @@ class FFmpegVideoConvertorPP(FFmpegPostProcessor): information['filepath'] = outpath information['format'] = self._preferedformat information['ext'] = self._preferedformat + fsize = os.path.getsize(outpath) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': outpath, + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [path], information @@ -429,6 +444,13 @@ class FFmpegEmbedSubtitlePP(FFmpegPostProcessor): self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return sub_filenames, information @@ -514,6 +536,13 @@ class FFmpegMetadataPP(FFmpegPostProcessor): os.remove(metadata_filename) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], info @@ -525,6 +554,13 @@ class FFmpegMergerPP(FFmpegPostProcessor): self._downloader.to_screen('[ffmpeg] Merging formats into "%s"' % filename) self.run_ffmpeg_multiple_files(info['__files_to_merge'], temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return info['__files_to_merge'], info def can_merge(self): @@ -560,6 +596,13 @@ class FFmpegFixupStretchedPP(FFmpegPostProcessor): os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], info @@ -578,6 +621,13 @@ class FFmpegFixupM4aPP(FFmpegPostProcessor): os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], info @@ -594,6 +644,14 @@ class FFmpegFixupM3u8PP(FFmpegPostProcessor): os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) + + fsize = os.path.getsize(encodeFilename(filename)) + self._hook_progress({ + 'total_bytes': fsize, + 'filename': encodeFilename(filename), + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], info @@ -657,4 +715,9 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor): 'data': f.read(), } + self._hook_progress({ + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) + return sub_filenames, info diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py index f5c14d974..285664f86 100644 --- a/youtube_dl/postprocessor/metadatafromtitle.py +++ b/youtube_dl/postprocessor/metadatafromtitle.py @@ -44,5 +44,9 @@ class MetadataFromTitlePP(PostProcessor): self._downloader.to_screen( '[fromtitle] parsed %s: %s' % (attribute, value if value is not None else 'NA')) + self._hook_progress({ + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) return [], info diff --git a/youtube_dl/postprocessor/xattrpp.py b/youtube_dl/postprocessor/xattrpp.py index 814dabecf..8ffdd1661 100644 --- a/youtube_dl/postprocessor/xattrpp.py +++ b/youtube_dl/postprocessor/xattrpp.py @@ -55,6 +55,11 @@ class XAttrMetadataPP(PostProcessor): write_xattr(filename, xattrname, byte_value) num_written += 1 + self._hook_progress({ + 'status': 'postprocessed', + 'postprocessor': self.__class__.__name__ + }) + return [], info except XAttrUnavailableError as e: