From 7c6630bfdd861e7e044a1f4bc7c31201996727dc Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 05:44:33 +0100 Subject: [PATCH 01/43] [YouTube] Miscellaneous clean-ups --- youtube_dl/extractor/youtube.py | 76 +++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index b31798729..c5024146a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1,5 +1,4 @@ # coding: utf-8 - from __future__ import unicode_literals import collections @@ -130,6 +129,15 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, 'SUPPORTS_COOKIES': True, }, + 'tv_simply': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'TVHTML5_SIMPLY', + 'clientVersion': '1.0', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 75, + }, 'web': { 'INNERTUBE_CONTEXT': { 'client': { @@ -140,6 +148,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, 'REQUIRE_PO_TOKEN': True, 'SUPPORTS_COOKIES': True, + 'PLAYER_PARAMS': '8AEB', }, } @@ -419,10 +428,15 @@ class YoutubeBaseInfoExtractor(InfoExtractor): T(compat_str))) def _extract_ytcfg(self, video_id, webpage): - return self._parse_json( - self._search_regex( - r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg', - default='{}'), video_id, fatal=False) or {} + ytcfg = self._search_json( + r'ytcfg\.set\s*\(', webpage, 'ytcfg', video_id, + end_pattern=r'\)\s*;', default={}) + + traverse_obj(ytcfg, ( + 'INNERTUBE_CONTEXT', 'client', 'configInfo', + T(lambda x: x.pop('appInstallData', None)))) + + return ytcfg def _extract_video(self, renderer): video_id = renderer['videoId'] @@ -1587,7 +1601,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): _PLAYER_JS_VARIANT_MAP = ( ('main', 'player_ias.vflset/en_US/base.js'), + ('tcc', 'player_ias_tcc.vflset/en_US/base.js'), ('tce', 'player_ias_tce.vflset/en_US/base.js'), + ('es5', 'player_es5.vflset/en_US/base.js'), + ('es6', 'player_es6.vflset/en_US/base.js'), ('tv', 'tv-player-ias.vflset/tv-player-ias.js'), ('tv_es6', 'tv-player-es6.vflset/tv-player-es6.js'), ('phone', 'player-plasma-ias-phone-en_US.vflset/base.js'), @@ -2257,13 +2274,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): player_response['videoDetails'] = video_details def is_agegated(playability): - if not isinstance(playability, dict): - return + # playability: dict + if not playability: + return False if playability.get('desktopLegacyAgeGateReason'): return True - reasons = filter(None, (playability.get(r) for r in ('status', 'reason'))) + reasons = traverse_obj(playability, (('status', 'reason'),)) AGE_GATE_REASONS = ( 'confirm your age', 'age-restricted', 'inappropriate', # reason 'age_verification_required', 'age_check_required', # status @@ -2321,15 +2339,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): trailer_video_id, self.ie_key(), trailer_video_id) def get_text(x): - if not x: - return - text = x.get('simpleText') - if text and isinstance(text, compat_str): - return text - runs = x.get('runs') - if not isinstance(runs, list): - return - return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)]) + return ''.join(traverse_obj( + x, (('simpleText',),), ('runs', Ellipsis, 'text'), + expected_type=compat_str)) search_meta = ( (lambda x: self._html_search_meta(x, webpage, default=None)) @@ -2541,15 +2553,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor): hls_manifest_url = streaming_data.get('hlsManifestUrl') if hls_manifest_url: - for f in self._extract_m3u8_formats( + formats.extend( + f for f in self._extract_m3u8_formats( hls_manifest_url, video_id, 'mp4', - entry_protocol='m3u8_native', live=is_live, fatal=False): + entry_protocol='m3u8_native', live=is_live, fatal=False) if process_manifest_format( - f, 'hls', None, self._search_regex( - r'/itag/(\d+)', f['url'], 'itag', default=None)): - formats.append(f) + f, 'hls', None, self._search_regex( + r'/itag/(\d+)', f['url'], 'itag', default=None))) - if self._downloader.params.get('youtube_include_dash_manifest', True): + if self.get_param('youtube_include_dash_manifest', True): dash_manifest_url = streaming_data.get('dashManifestUrl') if dash_manifest_url: for f in self._extract_mpd_formats( @@ -2576,7 +2588,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): playability_status, lambda x: x['errorScreen']['playerErrorMessageRenderer'], dict) or {} - reason = get_text(pemr.get('reason')) or playability_status.get('reason') + reason = get_text(pemr.get('reason')) or playability_status.get('reason') or '' subreason = pemr.get('subreason') if subreason: subreason = clean_html(get_text(subreason)) @@ -2732,7 +2744,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]: d_k += '_time' if d_k not in info and k in s_ks: - info[d_k] = parse_duration(query[k][0]) + info[d_k] = parse_duration(v[0]) if video_description: # Youtube Music Auto-generated description @@ -2780,9 +2792,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): for next_num, content in enumerate(contents, start=1): mmlir = content.get('macroMarkersListItemRenderer') or {} start_time = chapter_time(mmlir) - end_time = chapter_time(try_get( - contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \ - if next_num < len(contents) else duration + end_time = (traverse_obj( + contents, (next_num, 'macroMarkersListItemRenderer', T(chapter_time))) + if next_num < len(contents) else duration) if start_time is None or end_time is None: continue chapters.append({ @@ -3446,20 +3458,17 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): content_id = traverse_obj(view_model, ( 'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId', T(lambda v: v if YoutubeIE.suitable(v) else None))) - if not content_id: - return return merge_dicts(self.url_result( content_id, ie=YoutubeIE.ie_key(), video_id=content_id), { 'title': traverse_obj(view_model, ( 'overlayMetadata', 'primaryText', 'content', T(compat_str))), 'thumbnails': self._extract_thumbnails( view_model, 'thumbnail', final_key='sources'), - }) + }) if content_id else None def _video_entry(self, video_renderer): video_id = video_renderer.get('videoId') - if video_id: - return self._extract_video(video_renderer) + return self._extract_video(video_renderer) if video_id else None def _post_thread_entries(self, post_thread_renderer): post_renderer = try_get( @@ -4119,6 +4128,7 @@ class YoutubeFeedsInfoExtractor(YoutubeTabIE): Subclasses must define the _FEED_NAME property. """ + _LOGIN_REQUIRED = True @property From 0c41b03114df2591a9c0c087a4e1f9d521cb99fc Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 05:48:52 +0100 Subject: [PATCH 02/43] [YouTube] Update player client details --- youtube_dl/extractor/youtube.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c5024146a..81801f5bd 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -109,7 +109,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'MWEB', - 'clientVersion': '2.20250311.03.00', + 'clientVersion': '2.2.20250925.01.00', # mweb previously did not require PO Token with this UA 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', }, @@ -123,32 +123,35 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'client': { 'clientName': 'TVHTML5', 'clientVersion': '7.20250312.16.00', - 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version', + # See: https://github.com/youtube/cobalt/blob/main/cobalt/browser/user_agent/user_agent_platform_info.cc#L506 + 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/25.lts.30.1034943-gold (unlike Gecko), Unknown_TV_Unknown_0/Unknown (Unknown, Unknown)', }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, 'SUPPORTS_COOKIES': True, }, - 'tv_simply': { - 'INNERTUBE_CONTEXT': { - 'client': { - 'clientName': 'TVHTML5_SIMPLY', - 'clientVersion': '1.0', - }, - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 75, - }, + 'web': { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'WEB', - 'clientVersion': '2.20250312.04.00', + 'clientVersion': '2.20250925.01.00', + 'userAgent': 'Mozilla/5.0', }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, 'REQUIRE_PO_TOKEN': True, 'SUPPORTS_COOKIES': True, - 'PLAYER_PARAMS': '8AEB', + }, + # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats + 'web_safari': { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'WEB', + 'clientVersion': '2.20250925.01.00', + 'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)', + }, + }, }, } From 7f7b3881aacc5f68a43a14e0588087d986c59b14 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:01:02 +0100 Subject: [PATCH 03/43] [YouTube] Handle Web Safari formats From yt-dlp/yt-dlp#14168, thx bashonly. --- youtube_dl/extractor/youtube.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 81801f5bd..a6f60f9bf 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2540,6 +2540,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if f.get('source_preference') is None: f['source_preference'] = -1 + # Deprioritize since its pre-merged m3u8 formats may have lower quality audio streams + if client_name == 'web_safari' and proto == 'hls' and not is_live: + f['source_preference'] -= 1 + if itag in ('616', '235'): f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ') f['source_preference'] += 100 From aac0148b899d534c64b8bb2952a0f3c8abdc519c Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:06:32 +0100 Subject: [PATCH 04/43] [YouTube] Force `WEB` user agent for video page download Fixes #33142, until default UAs work. --- youtube_dl/extractor/youtube.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index a6f60f9bf..c349a123d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2183,8 +2183,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_id = self._match_id(url) base_url = self.http_scheme() + '//www.youtube.com/' webpage_url = base_url + 'watch?v=' + video_id + ua = traverse_obj(self._INNERTUBE_CLIENTS, ( + 'web', 'INNERTUBE_CONTEXT', 'client', 'userAgent')) + headers = {'User-Agent': ua} if ua else None webpage = self._download_webpage( - webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False) + webpage_url + '&bpctr=9999999999&has_verified=1', video_id, + headers=headers, fatal=False) player_response = None player_url = None From 0739f58f9024a086c7d492a47368225f1c2e51fa Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:09:21 +0100 Subject: [PATCH 05/43] [YouTube] Implement player JS override for player `0004de42` * based on yt-dlp/yt-dlp#14398, thx seproDev * adds --youtube-player-js-variant option * adds --youtube-player-js-version option * sets defaults to main variant of player `0004de42` * fixes #33187, for now --- youtube_dl/__init__.py | 2 ++ youtube_dl/extractor/youtube.py | 53 +++++++++++++++++++++++++++++---- youtube_dl/options.py | 11 +++++++ 3 files changed, 61 insertions(+), 5 deletions(-) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3c1272e7b..202f2c9b9 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -409,6 +409,8 @@ def _real_main(argv=None): 'include_ads': opts.include_ads, 'default_search': opts.default_search, 'youtube_include_dash_manifest': opts.youtube_include_dash_manifest, + 'youtube_player_js_version': opts.youtube_player_js_version, + 'youtube_player_js_variant': opts.youtube_player_js_variant, 'encoding': opts.encoding, 'extract_flat': opts.extract_flat, 'mark_watched': opts.mark_watched, diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c349a123d..830f2d502 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1625,6 +1625,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self._code_cache = {} self._player_cache = {} + def _get_player_js_version(self): + player_js_version = self.get_param('youtube_player_js_version') or '20348@0004de42' + sts_hash = self._search_regex( + ('^actual$(^)?(^)?', r'^([0-9]{5,})@([0-9a-f]{8,})$'), + player_js_version, 'player_js_version', group=(1, 2), default=None) + if sts_hash: + return sts_hash + self.report_warning( + 'Invalid player JS version "{0}" specified. ' + 'It should be "{1}" or in the format of {2}'.format( + player_js_version, 'actual', 'SignatureTimeStamp@Hash'), only_once=True) + return None, None + # *ytcfgs, webpage=None def _extract_player_url(self, *ytcfgs, **kw_webpage): if ytcfgs and not isinstance(ytcfgs[0], dict): @@ -1635,19 +1648,43 @@ class YoutubeIE(YoutubeBaseInfoExtractor): webpage or '', 'player URL', fatal=False) if player_url: ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},) - return traverse_obj( + player_url = traverse_obj( ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'), get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u)) + player_id_override = self._get_player_js_version()[1] + + requested_js_variant = self.get_param('youtube_player_js_variant') or 'main' + variant_js = next( + (v for k, v in self._PLAYER_JS_VARIANT_MAP if k == requested_js_variant), + None) + if variant_js: + player_id = player_id_override or self._extract_player_info(player_url) + original_url = player_url + player_url = '/s/player/{0}/{1}'.format(player_id, variant_js) + if original_url != player_url: + self.write_debug( + 'Forcing "{0}" player JS variant for player {1}\n' + ' original url = {2}'.format( + requested_js_variant, player_id, original_url), + only_once=True) + elif requested_js_variant != 'actual': + self.report_warning( + 'Invalid player JS variant name "{0}" requested. ' + 'Valid choices are: {1}'.format( + requested_js_variant, ','.join(k for k, _ in self._PLAYER_JS_VARIANT_MAP)), + only_once=True) + + return urljoin('https://www.youtube.com', player_url) + def _download_player_url(self, video_id, fatal=False): res = self._download_webpage( 'https://www.youtube.com/iframe_api', note='Downloading iframe API JS', video_id=video_id, fatal=fatal) player_version = self._search_regex( r'player\\?/([0-9a-fA-F]{8})\\?/', res or '', 'player version', fatal=fatal, - default=NO_DEFAULT if res else None) - if player_version: - return 'https://www.youtube.com/s/player/{0}/player_ias.vflset/en_US/base.js'.format(player_version) + default=NO_DEFAULT if res else None) or None + return player_version and 'https://www.youtube.com/s/player/{0}/player_ias.vflset/en_US/base.js'.format(player_version) def _signature_cache_id(self, example_sig): """ Return a string representation of a signature """ @@ -2034,9 +2071,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False): """ Extract signatureTimestamp (sts) + Required to tell API what sig/player version is in use. """ - sts = traverse_obj(ytcfg, 'STS', expected_type=int) + sts = traverse_obj( + (self._get_player_js_version(), ytcfg), + (0, 0), + (1, 'STS'), + expected_type=int_or_none) + if sts: return sts diff --git a/youtube_dl/options.py b/youtube_dl/options.py index 61705d1f0..e3360da89 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -412,6 +412,17 @@ def parseOpts(overrideArguments=None): '--youtube-skip-dash-manifest', action='store_false', dest='youtube_include_dash_manifest', help='Do not download the DASH manifests and related data on YouTube videos') + video_format.add_option( + '--youtube-player-js-variant', + action='store', dest='youtube_player_js_variant', + help='For YouTube, the player javascript variant to use for n/sig deciphering; `actual` to follow the site; default `%default`.', + choices=('actual', 'main', 'tcc', 'tce', 'es5', 'es6', 'tv', 'tv_es6', 'phone', 'tablet'), + default='main', metavar='VARIANT') + video_format.add_option( + '--youtube-player-js-version', + action='store', dest='youtube_player_js_version', + help='For YouTube, the player javascript version to use for n/sig deciphering, specified as `signature_timestamp@hash`, or `actual` to follow the site; default `%default`', + default='20348@0004de42', metavar='STS@HASH') video_format.add_option( '--merge-output-format', action='store', dest='merge_output_format', metavar='FORMAT', default=None, From 40ab920354e4ec5ec153c29fddd6bb0688b324e9 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:52:48 +0100 Subject: [PATCH 06/43] [downloader] Delay download according to `available_at` format key --- youtube_dl/downloader/common.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py index 91e691776..8354030a9 100644 --- a/youtube_dl/downloader/common.py +++ b/youtube_dl/downloader/common.py @@ -11,6 +11,7 @@ from ..utils import ( decodeArgument, encodeFilename, error_to_compat_str, + float_or_none, format_bytes, shell_quote, timeconvert, @@ -367,14 +368,27 @@ class FileDownloader(object): }) return True - min_sleep_interval = self.params.get('sleep_interval') - if min_sleep_interval: - max_sleep_interval = self.params.get('max_sleep_interval', min_sleep_interval) - sleep_interval = random.uniform(min_sleep_interval, max_sleep_interval) + min_sleep_interval, max_sleep_interval = ( + float_or_none(self.params.get(interval), default=0) + for interval in ('sleep_interval', 'max_sleep_interval')) + + sleep_note = '' + available_at = info_dict.get('available_at') + if available_at: + forced_sleep_interval = available_at - int(time.time()) + if forced_sleep_interval > min_sleep_interval: + sleep_note = 'as required by the site' + min_sleep_interval = forced_sleep_interval + if forced_sleep_interval > max_sleep_interval: + max_sleep_interval = forced_sleep_interval + + sleep_interval = random.uniform( + min_sleep_interval, max_sleep_interval or min_sleep_interval) + + if sleep_interval > 0: self.to_screen( - '[download] Sleeping %s seconds...' % ( - int(sleep_interval) if sleep_interval.is_integer() - else '%.2f' % sleep_interval)) + '[download] Sleeping %.2f seconds %s...' % ( + sleep_interval, sleep_note)) time.sleep(sleep_interval) return self.real_download(filename, info_dict) From 92680b127f933e35327e071de2b4a5f2d67ec661 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:20:48 +0100 Subject: [PATCH 07/43] [YouTube] Handle required preroll waiting period * Based on yt-dlp/yt-dlp#14081, thx bashonly * Uses internal `youtube_preroll_sleep` param, default 6s --- youtube_dl/extractor/youtube.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 830f2d502..1aca69b93 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2241,12 +2241,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): video_id, 'initial player response') is_live = traverse_obj(player_response, ('videoDetails', 'isLive')) + fetched_timestamp = None if False and not player_response: player_response = self._call_api( 'player', {'videoId': video_id}, video_id) if True or not player_response: origin = 'https://www.youtube.com' pb_context = {'html5Preference': 'HTML5_PREF_WANTS'} + fetched_timestamp = int(time.time()) player_url = self._extract_player_url(webpage) ytcfg = self._extract_ytcfg(video_id, webpage or '') @@ -2313,6 +2315,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): hls = traverse_obj( (player_response, api_player_response), (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none))) + fetched_timestamp = int(time.time()) if len(hls) == 2 and not hls[0] and hls[1]: player_response['streamingData']['hlsManifestUrl'] = hls[1] else: @@ -2474,6 +2477,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor): lower = lambda s: s.lower() + if is_live: + fetched_timestamp = None + elif fetched_timestamp is not None: + # Handle preroll waiting period + preroll_sleep = self.get_param('youtube_preroll_sleep') + preroll_sleep = int_or_none(preroll_sleep, default=6) + fetched_timestamp += preroll_sleep + for fmt in streaming_formats: if fmt.get('targetDurationSec'): continue @@ -2570,6 +2581,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'downloader_options': {'http_chunk_size': CHUNK_SIZE}, # No longer useful? }) + if fetched_timestamp: + dct['available_at'] = fetched_timestamp + formats.append(dct) def process_manifest_format(f, proto, client_name, itag, all_formats=False): From f2a774cb9d661b1f19df4ee0e91f402a15c4d413 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 07:03:16 +0100 Subject: [PATCH 08/43] [YouTube] Fix subtitles extraction From yt-dlp/yt-dlp#13659, thx bashonly --- youtube_dl/extractor/youtube.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1aca69b93..1228dc8a3 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2668,7 +2668,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.raise_geo_restricted( subreason, countries) reason += '\n' + subreason + if reason: + if 'sign in' in reason.lower(): + self.raise_login_required(remove_end(reason, 'This helps protect our community. Learn more')) + elif traverse_obj(playability_status, ('errorScreen', 'playerCaptchaViewModel', T(dict))): + reason += '. YouTube is requiring a captcha challenge before playback' raise ExtractorError(reason, expected=True) self._sort_formats(formats) @@ -2771,6 +2776,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): for fmt in self._SUBTITLE_FORMATS: query.update({ 'fmt': fmt, + # xosf=1 causes undesirable text position data for vtt, json3 & srv* subtitles + # See: https://github.com/yt-dlp/yt-dlp/issues/13654 + 'xosf': [] }) lang_subs.append({ 'ext': fmt, From 2735d1bf1d1f947891776330cb58d792d03cf436 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 06:16:50 +0100 Subject: [PATCH 09/43] [YouTube] Extract srt subtitles From yt-dlp/yt-dlp#13411, thx gamer191 --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 1228dc8a3..d32c9df99 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -711,7 +711,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): r'/(?P[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', r'\b(?Pvfl[a-zA-Z0-9_-]{6,})\b.*?\.js$', ) - _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt') + _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'srt', 'vtt') _GEO_BYPASS = False From 4222c6d78b63440849ad6a886a2ff5b607f3feae Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 07:09:31 +0100 Subject: [PATCH 10/43] [YouTube] Extract fallback title and description from initial data Based on yt-dlp/yt-dlp#14078, thx bashonly --- youtube_dl/extractor/youtube.py | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index d32c9df99..dea109eae 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2849,6 +2849,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): initial_data = self._call_api( 'next', {'videoId': video_id}, video_id, fatal=False) + initial_sdcr = None if initial_data: chapters = self._extract_chapters_from_json( initial_data, video_id, duration) @@ -2976,12 +2977,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor): info['track'] = mrr_contents_text # this is not extraction but spelunking! - carousel_lockups = traverse_obj( - initial_data, - ('engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer', - 'content', 'structuredDescriptionContentRenderer', 'items', Ellipsis, - 'videoDescriptionMusicSectionRenderer', 'carouselLockups', Ellipsis), - expected_type=dict) or [] + initial_sdcr = traverse_obj(initial_data, ( + 'engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer', + 'content', 'structuredDescriptionContentRenderer', T(dict)), + get_all=False) + carousel_lockups = traverse_obj(initial_sdcr, ( + 'items', Ellipsis, 'videoDescriptionMusicSectionRenderer', + 'carouselLockups', Ellipsis, T(dict))) or [] # try to reproduce logic from metadataRowContainerRenderer above (if it still is) fields = (('ALBUM', 'album'), ('ARTIST', 'artist'), ('SONG', 'track'), ('LICENSES', 'license')) # multiple_songs ? @@ -3006,6 +3008,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.mark_watched(video_id, player_response) + # Fallbacks for missing metadata + if initial_sdcr: + if info.get('description') is None: + info['description'] = traverse_obj(initial_sdcr, ( + 'items', Ellipsis, 'expandableVideoDescriptionBodyRenderer', + 'attributedDescriptionBodyText', 'content', T(compat_str)), + get_all=False) + # videoDescriptionHeaderRenderer also has publishDate/channel/handle/ucid, but not needed + if info.get('title') is None: + info['title'] = traverse_obj( + (initial_sdcr, initial_data), + (0, 'items', Ellipsis, 'videoDescriptionHeaderRenderer', T(dict)), + (1, 'playerOverlays', 'playerOverlayRenderer', 'videoDetails', + 'playerOverlayVideoDetailsRenderer', T(dict)), + expected_type=lambda x: self._get_text(x, 'title'), + get_all=False) + return merge_dicts( info, { 'uploader_id': self._extract_uploader_id(owner_profile_url), From 9223fcc48a498835a0c1f82a65d1d70fb17e303e Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 07:28:11 +0100 Subject: [PATCH 11/43] [YouTube] Support `LOCKUP_CONTENT_TYPE_VIDEO` in subscriptions feed extraction From yt-dlp/yt-dlp#13665), thx bashonly --- youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index dea109eae..0b802351d 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3535,18 +3535,29 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if not content_id: return content_type = view_model.get('contentType') - if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): + if content_type == 'LOCKUP_CONTENT_TYPE_VIDEO': + ie = YoutubeIE + url = update_url_query( + 'https://www.youtube.com/watch', {'v': content_id}), + thumb_keys = (None,) + elif content_type in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): + ie = YoutubeTabIE + url = update_url_query( + 'https://www.youtube.com/playlist', {'list': content_id}), + thumb_keys = ('collectionThumbnailViewModel', 'primaryThumbnail') + else: self.report_warning( - 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True) + 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), + only_once=True) return + thumb_keys = ('contentImage',) + thumb_keys + ('thumbnailViewModel', 'image') return merge_dicts(self.url_result( - update_url_query('https://www.youtube.com/playlist', {'list': content_id}), - ie=YoutubeTabIE.ie_key(), video_id=content_id), { + url, ie=ie.ie_key(), video_id=content_id), { 'title': traverse_obj(view_model, ( - 'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))), - 'thumbnails': self._extract_thumbnails(view_model, ( - 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', - 'thumbnailViewModel', 'image'), final_key='sources'), + 'metadata', 'lockupMetadataViewModel', 'title', + 'content', T(compat_str))), + 'thumbnails': self._extract_thumbnails( + view_model, thumb_keys, final_key='sources'), }) def _extract_shorts_lockup_view_model(self, view_model): From 617d4e646634f035727cae917455ad3cb898dabf Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 07:34:15 +0100 Subject: [PATCH 12/43] [core] Support explicit `--no-list-formats` option --- youtube_dl/options.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/youtube_dl/options.py b/youtube_dl/options.py index e3360da89..ce3633c41 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -404,6 +404,10 @@ def parseOpts(overrideArguments=None): '-F', '--list-formats', action='store_true', dest='listformats', help='List all available formats of requested videos') + video_format.add_option( + '--no-list-formats', + action='store_false', dest='listformats', + help='Do not list available formats of requested videos (default)') video_format.add_option( '--youtube-include-dash-manifest', action='store_true', dest='youtube_include_dash_manifest', default=True, From 82552faba6f29ced4914cd670ccd80836364fd45 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sun, 28 Sep 2025 13:41:27 +0100 Subject: [PATCH 13/43] [workflows/ci] Update to windows-2022 runner FFS --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8234e0ccb..c7a8fff84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -122,12 +122,12 @@ jobs: ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }} run-tests-ext: [sh] include: - - os: windows-2019 + - os: windows-2022 python-version: 3.4 python-impl: cpython ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} run-tests-ext: bat - - os: windows-2019 + - os: windows-2022 python-version: 3.4 python-impl: cpython ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} From e21ff28f6fb9a9dbb8097b951f6c709af33221f3 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 17 Oct 2025 06:20:42 +0100 Subject: [PATCH 14/43] [YouTube] Misc clean-ups from linter, etc --- youtube_dl/extractor/youtube.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 0b802351d..04d0881a8 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -100,8 +100,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, + 'REQUIRE_PO_TOKEN': False, 'REQUIRE_JS_PLAYER': False, - 'REQUIRE_PO_TOKEN': True, }, # mweb has 'ultralow' formats # See: https://github.com/yt-dlp/yt-dlp/pull/557 @@ -478,6 +478,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): def _extract_thumbnails(data, *path_list, **kw_final_key): """ Extract thumbnails from thumbnails dict + @param path_list: path list to level that contains 'thumbnails' key """ final_key = kw_final_key.get('final_key', 'thumbnails') @@ -2176,7 +2177,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): raise ExtractorError('Invalid URL: %s' % url) return mobj.group(2) - def _extract_chapters_from_json(self, data, video_id, duration): + @staticmethod + def _extract_chapters_from_json(data, video_id, duration): chapters_list = try_get( data, lambda x: x['playerOverlays'] @@ -2472,7 +2474,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return LazyList({ 'url': update_url_query(f['url'], { 'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize'])), - }) + }), } for range_start in range(0, f['filesize'], CHUNK_SIZE)) lower = lambda s: s.lower() @@ -2778,7 +2780,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'fmt': fmt, # xosf=1 causes undesirable text position data for vtt, json3 & srv* subtitles # See: https://github.com/yt-dlp/yt-dlp/issues/13654 - 'xosf': [] + 'xosf': [], }) lang_subs.append({ 'ext': fmt, @@ -3515,8 +3517,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): shelf_renderer, lambda x: x['title']['runs'][0]['text'], compat_str) yield self.url_result(shelf_url, video_title=title) # Shelf may not contain shelf URL, fallback to extraction from content - for entry in self._shelf_entries_from_content(shelf_renderer): - yield entry + for from_ in self._shelf_entries_from_content(shelf_renderer): + yield from_ def _playlist_entries(self, video_list_renderer): for content in video_list_renderer['contents']: @@ -3538,12 +3540,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if content_type == 'LOCKUP_CONTENT_TYPE_VIDEO': ie = YoutubeIE url = update_url_query( - 'https://www.youtube.com/watch', {'v': content_id}), + 'https://www.youtube.com/watch', {'v': content_id}) thumb_keys = (None,) elif content_type in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): ie = YoutubeTabIE url = update_url_query( - 'https://www.youtube.com/playlist', {'list': content_id}), + 'https://www.youtube.com/playlist', {'list': content_id}) thumb_keys = ('collectionThumbnailViewModel', 'primaryThumbnail') else: self.report_warning( @@ -4162,7 +4164,7 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor): 'only_matching': True, }] - def _real_extract(self, url): + def _real_extract(self, _): return self.url_result( 'https://www.youtube.com/playlist?list=LL', ie=YoutubeTabIE.ie_key()) @@ -4244,7 +4246,7 @@ class YoutubeFeedsInfoExtractor(YoutubeTabIE): def _real_initialize(self): self._login() - def _real_extract(self, url): + def _real_extract(self, _): return self.url_result( 'https://www.youtube.com/feed/%s' % self._FEED_NAME, ie=YoutubeTabIE.ie_key()) @@ -4259,7 +4261,7 @@ class YoutubeWatchLaterIE(InfoExtractor): 'only_matching': True, }] - def _real_extract(self, url): + def _real_extract(self, _): return self.url_result( 'https://www.youtube.com/playlist?list=WL', ie=YoutubeTabIE.ie_key()) @@ -4339,7 +4341,7 @@ class YoutubeTruncatedURLIE(InfoExtractor): 'only_matching': True, }] - def _real_extract(self, url): + def _real_extract(self, _): raise ExtractorError( 'Did you forget to quote the URL? Remember that & is a meta ' 'character in most shells, so you want to put the URL in quotes, ' From c1f5c3274a0ebc3181fc7094b53815445c154782 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 17 Oct 2025 06:22:53 +0100 Subject: [PATCH 15/43] [YouTube] Improve some traversals Pending full alignment with yt-dlp ... --- youtube_dl/extractor/youtube.py | 78 ++++++++++++--------------------- 1 file changed, 27 insertions(+), 51 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 04d0881a8..d56985420 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -521,34 +521,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor): headers={'content-type': 'application/json'}) if not search: break - slr_contents = try_get( + slr_contents = traverse_obj( search, - (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'], - lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']), - list) + ('contents', 'twoColumnSearchResultsRenderer', 'primaryContents', + 'sectionListRenderer', 'contents'), + ('onResponseReceivedCommands', 0, 'appendContinuationItemsAction', + 'continuationItems'), + expected_type=list) if not slr_contents: break - for slr_content in slr_contents: - isr_contents = try_get( - slr_content, - lambda x: x['itemSectionRenderer']['contents'], - list) - if not isr_contents: - continue - for content in isr_contents: - if not isinstance(content, dict): - continue - video = content.get('videoRenderer') - if not isinstance(video, dict): - continue - video_id = video.get('videoId') - if not video_id: - continue - yield self._extract_video(video) - token = try_get( + for video in traverse_obj( + slr_contents, + (Ellipsis, 'itemSectionRenderer', 'contents', + Ellipsis, 'videoRenderer', + T(lambda v: v if v.get('videoId') else None))): + yield self._extract_video(video) + + token = traverse_obj( slr_contents, - lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'], - compat_str) + (-1, 'continuationItemRenderer', 'continuationEndpoint', + 'continuationCommand', 'token', T(compat_str))) if not token: break data['continuation'] = token @@ -3428,13 +3420,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): @staticmethod def _extract_grid_item_renderer(item): - assert isinstance(item, dict) - for key, renderer in item.items(): - if not key.startswith('grid') or not key.endswith('Renderer'): - continue - if not isinstance(renderer, dict): - continue - return renderer + return traverse_obj(item, ( + T(dict.items), lambda _, k_v: k_v[0].startswith('grid') and k_v[0].endswith('Renderer'), + 1, T(dict)), get_all=False) @staticmethod def _get_text(r, k): @@ -3608,15 +3596,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): yield self.url_result(ep_url, ie=YoutubeIE.ie_key(), video_id=video_id) def _post_thread_continuation_entries(self, post_thread_continuation): - contents = post_thread_continuation.get('contents') - if not isinstance(contents, list): - return - for content in contents: - renderer = content.get('backstagePostThreadRenderer') - if not isinstance(renderer, dict): - continue - for entry in self._post_thread_entries(renderer): - yield entry + for renderer in traverse_obj(post_thread_continuation, ( + 'contents', Ellipsis, 'backstagePostThreadRenderer', T(dict))): + for from_ in self._post_thread_entries(renderer): + yield from_ def _rich_grid_entries(self, contents): for content in traverse_obj( @@ -3691,17 +3674,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if slr_renderer: is_channels_tab = tab.get('title') == 'Channels' continuation = None - slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or [] - for slr_content in slr_contents: - if not isinstance(slr_content, dict): - continue - is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict) - if not is_renderer: - continue - isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or [] - for isr_content in isr_contents: - if not isinstance(isr_content, dict): - continue + for is_renderer in traverse_obj(slr_renderer, ( + 'contents', Ellipsis, 'itemSectionRenderer', T(dict))): + for isr_content in traverse_obj(slr_renderer, ( + 'contents', Ellipsis, T(dict))): renderer = isr_content.get('playlistVideoListRenderer') if renderer: for entry in self._playlist_entries(renderer): From efb4011211f4cfb97894e8d30eace79e90e33c72 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 17 Oct 2025 06:24:43 +0100 Subject: [PATCH 16/43] [YouTube] Introduce `_extract_and_report_alerts()` per yt-dlp Fixes #33196. Also removing previous `_extract_alerts()` method. --- youtube_dl/extractor/youtube.py | 46 ++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index d56985420..c045bc8bc 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3872,18 +3872,34 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): uploader['channel'] = uploader['uploader'] return uploader - @classmethod - def _extract_alert(cls, data): - alerts = [] - for alert in traverse_obj(data, ('alerts', Ellipsis), expected_type=dict): - alert_text = traverse_obj( - alert, (None, lambda x: x['alertRenderer']['text']), get_all=False) - if not alert_text: - continue - text = cls._get_text(alert_text, 'text') - if text: - alerts.append(text) - return '\n'.join(alerts) + def _extract_and_report_alerts(self, data, expected=True, fatal=True, only_once=False): + + def alerts(): + for alert in traverse_obj(data, ('alerts', Ellipsis), expected_type=dict): + alert_dict = traverse_obj( + alert, 'alertRenderer', None, expected_type=dict, get_all=False) + alert_type = traverse_obj(alert_dict, 'type') + if not alert_type: + continue + message = self._get_text(alert_dict, 'text') + if message: + yield alert_type, message + + errors, warnings = [], [] + _IGNORED_WARNINGS = T('Unavailable videos will be hidden during playback') + for alert_type, alert_message in alerts(): + if alert_type.lower() == 'error' and fatal: + errors.append([alert_type, alert_message]) + elif alert_message not in _IGNORED_WARNINGS: + warnings.append([alert_type, alert_message]) + + for alert_type, alert_message in itertools.chain(warnings, errors[:-1]): + self.report_warning( + 'YouTube said: %s - %s' % (alert_type, alert_message), + only_once=only_once) + if errors: + raise ExtractorError( + 'YouTube said: %s' % (errors[-1][1],), expected=expected) def _extract_from_tabs(self, item_id, webpage, data, tabs): selected_tab = self._extract_selected_tab(tabs) @@ -3983,10 +3999,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): compat_str) or video_id if video_id: return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id) + # Capture and output alerts - alert = self._extract_alert(data) - if alert: - raise ExtractorError(alert, expected=True) + self._extract_and_report_alerts(data) + # Failed to recognize raise ExtractorError('Unable to recognize tab page') From 1e109aaee13a30e2a23f982410ffb3e4f73913df Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 17 Oct 2025 06:55:27 +0100 Subject: [PATCH 17/43] [workflows/ci] Avoid installing wheel and setuptools with pip Works around dependent wheel installation failure with Py 3.4 from 2025-10 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7a8fff84..073c4458c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -365,7 +365,7 @@ jobs: python -m ensurepip || python -m pip --version || { \ get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \ curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \ - python get-pip.py; } + python get-pip.py --no-setuptools --no-wheel; } - name: Set up Python 2.6 pip if: ${{ matrix.python-version == '2.6' }} shell: bash From 014ae63a11d6f8647e262ed25d68f1e4aab9ae20 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 30 Oct 2025 16:36:45 +0000 Subject: [PATCH 18/43] [test] Support additional args and kwargs in report_warning() mocks --- test/helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/helper.py b/test/helper.py index 6f2129eff..fac069d25 100644 --- a/test/helper.py +++ b/test/helper.py @@ -85,10 +85,10 @@ class FakeYDL(YoutubeDL): # Silence an expected warning matching a regex old_report_warning = self.report_warning - def report_warning(self, message): + def report_warning(self, message, *args, **kwargs): if re.match(regex, message): return - old_report_warning(message) + old_report_warning(message, *args, **kwargs) self.report_warning = types.MethodType(report_warning, self) @@ -265,11 +265,11 @@ def assertRegexpMatches(self, text, regexp, msg=None): def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning - def _report_warning(w): + def _report_warning(self, w, *args, **kwargs): if not any(re.search(w_re, w) for w_re in warnings_re): real_warning(w) - ydl.report_warning = _report_warning + ydl.report_warning = types.MethodType(_report_warning, ydl) def http_server_port(httpd): From bc39e5e6787579b3cb6d1fef3ff5e05e6db0ce71 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 31 Oct 2025 12:08:08 +0000 Subject: [PATCH 19/43] [test] Fix test_traversal_morsel for Py 3.14+ Thx: yt-dlp/yt-dlp#13471 --- test/test_traversal.py | 38 ++++++++++++++++---------------------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/test/test_traversal.py b/test/test_traversal.py index 00a428edb..101bb57b1 100644 --- a/test/test_traversal.py +++ b/test/test_traversal.py @@ -9,6 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import itertools import re from youtube_dl.traversal import ( @@ -18,9 +19,12 @@ from youtube_dl.traversal import ( traverse_obj, ) from youtube_dl.compat import ( + compat_chr as chr, compat_etree_fromstring, compat_http_cookies, + compat_map as map, compat_str, + compat_zip as zip, ) from youtube_dl.utils import ( int_or_none, @@ -446,36 +450,26 @@ class TestTraversal(_TestCase): msg='`any` should allow further branching') def test_traversal_morsel(self): - values = { - 'expires': 'a', - 'path': 'b', - 'comment': 'c', - 'domain': 'd', - 'max-age': 'e', - 'secure': 'f', - 'httponly': 'g', - 'version': 'h', - 'samesite': 'i', - } - # SameSite added in Py3.8, breaks .update for 3.5-3.7 - if sys.version_info < (3, 8): - del values['samesite'] morsel = compat_http_cookies.Morsel() + # SameSite added in Py3.8, breaks .update for 3.5-3.7 + # Similarly Partitioned, Py3.14, thx Grub4k + values = dict(zip(morsel, map(chr, itertools.count(ord('a'))))) morsel.set(str('item_key'), 'item_value', 'coded_value') morsel.update(values) - values['key'] = str('item_key') - values['value'] = 'item_value' + values.update({ + 'key': str('item_key'), + 'value': 'item_value', + }), values = dict((str(k), v) for k, v in values.items()) - # make test pass even without ordered dict - value_set = set(values.values()) for key, value in values.items(): self.assertEqual(traverse_obj(morsel, key), value, msg='Morsel should provide access to all values') - self.assertEqual(set(traverse_obj(morsel, Ellipsis)), value_set, - msg='`...` should yield all values') - self.assertEqual(set(traverse_obj(morsel, lambda k, v: True)), value_set, - msg='function key should yield all values') + values = list(values.values()) + self.assertMaybeCountEqual(traverse_obj(morsel, Ellipsis), values, + msg='`...` should yield all values') + self.assertMaybeCountEqual(traverse_obj(morsel, lambda k, v: True), values, + msg='function key should yield all values') self.assertIs(traverse_obj(morsel, [(None,), any]), morsel, msg='Morsel should not be implicitly changed to dict on usage') From cca41c9d2ca51fbfdc9a8c16f2f7b049b577300b Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 31 Oct 2025 12:09:14 +0000 Subject: [PATCH 20/43] [test] Move dict_get() traversal test to its own class Matches yt-dlp/yt-dlp#9426 --- test/test_traversal.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_traversal.py b/test/test_traversal.py index 101bb57b1..2987970ba 100644 --- a/test/test_traversal.py +++ b/test/test_traversal.py @@ -476,6 +476,8 @@ class TestTraversal(_TestCase): def test_get_first(self): self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam') + +class TestDictGet(_TestCase): def test_dict_get(self): FALSE_VALUES = { 'none': None, From 96419fa7064c7f77ccb1909e23150fde603f9f36 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 31 Oct 2025 12:20:26 +0000 Subject: [PATCH 21/43] [utils] Support `filter` traversal key Thx yt-dlp/yt-dlp#10653 --- test/test_traversal.py | 8 ++++++++ youtube_dl/compat.py | 6 ++++++ youtube_dl/utils.py | 8 ++++++++ 3 files changed, 22 insertions(+) diff --git a/test/test_traversal.py b/test/test_traversal.py index 2987970ba..21f81136f 100644 --- a/test/test_traversal.py +++ b/test/test_traversal.py @@ -473,6 +473,14 @@ class TestTraversal(_TestCase): self.assertIs(traverse_obj(morsel, [(None,), any]), morsel, msg='Morsel should not be implicitly changed to dict on usage') + def test_traversal_filter(self): + data = [None, False, True, 0, 1, 0.0, 1.1, '', 'str', {}, {0: 0}, [], [1]] + + self.assertEqual( + traverse_obj(data, (Ellipsis, filter)), + [True, 1, 1.1, 'str', {0: 0}, [1]], + '`filter` should filter falsy values') + def test_get_first(self): self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam') diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index ebe22bdf9..96b099a58 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -3452,6 +3452,8 @@ except ImportError: except ImportError: compat_map = map + +# compat_filter, compat_filter_fns try: from future_builtins import filter as compat_filter except ImportError: @@ -3459,6 +3461,9 @@ except ImportError: from itertools import ifilter as compat_filter except ImportError: compat_filter = filter +# "Is this function one or maybe the other filter()?" +compat_filter_fns = tuple(set((filter, compat_filter))) + # compat_zip try: @@ -3675,6 +3680,7 @@ __all__ = [ 'compat_etree_fromstring', 'compat_etree_iterfind', 'compat_filter', + 'compat_filter_fns', 'compat_get_terminal_size', 'compat_getenv', 'compat_getpass_getpass', diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index c4262936e..29d62130a 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -53,6 +53,8 @@ from .compat import ( compat_etree_fromstring, compat_etree_iterfind, compat_expanduser, + compat_filter as filter, + compat_filter_fns, compat_html_entities, compat_html_entities_html5, compat_http_client, @@ -6283,6 +6285,7 @@ def traverse_obj(obj, *paths, **kwargs): Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`. - `any`-builtin: Take the first matching object and return it, resetting branching. - `all`-builtin: Take all matching objects and return them as a list, resetting branching. + - `filter`-builtin: Return the value if it is truthy, `None` otherwise. `tuple`, `list`, and `dict` all support nested paths and branches. @@ -6497,6 +6500,11 @@ def traverse_obj(obj, *paths, **kwargs): objs = (list(filtered_objs),) continue + # filter might be from __builtin__, future_builtins, or itertools.ifilter + if key in compat_filter_fns: + objs = filter(None, objs) + continue + if __debug__ and callable(key): # Verify function signature _try_bind_args(key, None, None) From 68fe8c1781f6bbfef3aa75cf7746f6508d7ef75c Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 31 Oct 2025 13:36:55 +0000 Subject: [PATCH 22/43] [utils] Support traversal helper functions `require`, `value`, `unpack` Thx: yt-dlp/yt-dlp#10653 --- test/test_traversal.py | 39 +++++++++++++++++++++++++++++++++++---- youtube_dl/traversal.py | 3 +++ youtube_dl/utils.py | 25 +++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 4 deletions(-) diff --git a/test/test_traversal.py b/test/test_traversal.py index 21f81136f..5d08b8dbb 100644 --- a/test/test_traversal.py +++ b/test/test_traversal.py @@ -15,8 +15,11 @@ import re from youtube_dl.traversal import ( dict_get, get_first, + require, T, traverse_obj, + unpack, + value, ) from youtube_dl.compat import ( compat_chr as chr, @@ -27,7 +30,9 @@ from youtube_dl.compat import ( compat_zip as zip, ) from youtube_dl.utils import ( + ExtractorError, int_or_none, + join_nonempty, str_or_none, ) @@ -462,8 +467,8 @@ class TestTraversal(_TestCase): }), values = dict((str(k), v) for k, v in values.items()) - for key, value in values.items(): - self.assertEqual(traverse_obj(morsel, key), value, + for key, val in values.items(): + self.assertEqual(traverse_obj(morsel, key), val, msg='Morsel should provide access to all values') values = list(values.values()) self.assertMaybeCountEqual(traverse_obj(morsel, Ellipsis), values, @@ -481,8 +486,31 @@ class TestTraversal(_TestCase): [True, 1, 1.1, 'str', {0: 0}, [1]], '`filter` should filter falsy values') - def test_get_first(self): - self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam') + +class TestTraversalHelpers(_TestCase): + def test_traversal_require(self): + with self.assertRaises(ExtractorError, msg='Missing `value` should raise'): + traverse_obj(_TEST_DATA, ('None', T(require('value')))) + self.assertEqual( + traverse_obj(_TEST_DATA, ('str', T(require('value')))), 'str', + '`require` should pass through non-`None` values') + + def test_unpack(self): + self.assertEqual( + unpack(lambda *x: ''.join(map(compat_str, x)))([1, 2, 3]), '123') + self.assertEqual( + unpack(join_nonempty)([1, 2, 3]), '1-2-3') + self.assertEqual( + unpack(join_nonempty, delim=' ')([1, 2, 3]), '1 2 3') + with self.assertRaises(TypeError): + unpack(join_nonempty)() + with self.assertRaises(TypeError): + unpack() + + def test_value(self): + self.assertEqual( + traverse_obj(_TEST_DATA, ('str', T(value('other')))), 'other', + '`value` should substitute specified value') class TestDictGet(_TestCase): @@ -508,6 +536,9 @@ class TestDictGet(_TestCase): self.assertEqual(dict_get(d, ('b', 'c', key, )), None) self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value) + def test_get_first(self): + self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/traversal.py b/youtube_dl/traversal.py index 834cfef7f..e4e8758c6 100644 --- a/youtube_dl/traversal.py +++ b/youtube_dl/traversal.py @@ -5,6 +5,9 @@ from .utils import ( dict_get, get_first, + require, T, traverse_obj, + unpack, + value, ) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 29d62130a..437257f5b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -6543,6 +6543,31 @@ def traverse_obj(obj, *paths, **kwargs): return None if default is NO_DEFAULT else default +def value(value): + return lambda _: value + + +class require(ExtractorError): + def __init__(self, name, expected=False): + super(require, self).__init__( + 'Unable to extract {0}'.format(name), expected=expected) + + def __call__(self, value): + if value is None: + raise self + + return value + + +def unpack(func, **kwargs): + """Make a function that applies `partial(func, **kwargs)` to its argument as *args""" + @functools.wraps(func) + def inner(items): + return func(*items, **kwargs) + + return inner + + def T(*x): """ For use in yt-dl instead of {type, ...} or set((type, ...)) """ return set(x) From a96a77875023407233bae4111a36d113b756a4e3 Mon Sep 17 00:00:00 2001 From: dirkf Date: Fri, 31 Oct 2025 14:27:33 +0000 Subject: [PATCH 23/43] [core] Fix housekeeping for `available_at` --- youtube_dl/YoutubeDL.py | 2 +- youtube_dl/extractor/common.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 8367b6e53..ec1d35c3a 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -357,7 +357,7 @@ class YoutubeDL(object): _NUMERIC_FIELDS = set(( 'width', 'height', 'tbr', 'abr', 'asr', 'vbr', 'fps', 'filesize', 'filesize_approx', - 'timestamp', 'upload_year', 'upload_month', 'upload_day', + 'timestamp', 'upload_year', 'upload_month', 'upload_day', 'available_at', 'duration', 'view_count', 'like_count', 'dislike_count', 'repost_count', 'average_rating', 'comment_count', 'age_limit', 'start_time', 'end_time', diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index a64fcfccc..a0901dab5 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -214,6 +214,7 @@ class InfoExtractor(object): width : height ratio as float. * no_resume The server does not support resuming the (HTTP or RTMP) download. Boolean. + * available_at Unix timestamp of when a format will be available to download * downloader_options A dictionary of downloader options as described in FileDownloader From 23a848c3141ad2ba1e7bb62708f5ed72ef81c98a Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 1 Nov 2025 20:24:43 +0000 Subject: [PATCH 24/43] [utils] Add `partial_application` decorator function Thx: yt-dlp/yt-dlp#10653 --- test/test_utils.py | 16 ++++++++++++++++ youtube_dl/utils.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/test/test_utils.py b/test/test_utils.py index 2947cce7e..9aca4df63 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -69,6 +69,7 @@ from youtube_dl.utils import ( parse_iso8601, parse_resolution, parse_qs, + partial_application, pkcs1pad, prepend_extension, read_batch_urls, @@ -1723,6 +1724,21 @@ Line 1 'a', 'b', 'c', 'd', from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d') + def test_partial_application(self): + test_fn = partial_application(lambda x, kwarg=None: '{0}, kwarg={1!r}'.format(x, kwarg)) + self.assertTrue( + callable(test_fn(kwarg=10)), + 'missing positional parameter should apply partially') + self.assertEqual( + test_fn(10, kwarg=0.1), '10, kwarg=0.1', + 'positionally passed argument should call function') + self.assertEqual( + test_fn(x=10), '10, kwarg=None', + 'keyword passed positional should call function') + self.assertEqual( + test_fn(kwarg=0.1)(10), '10, kwarg=0.1', + 'call after partial application should call the function') + if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 437257f5b..f2c02829b 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -1861,6 +1861,39 @@ def write_json_file(obj, fn): raise +class partial_application(object): + """Allow a function to use pre-set argument values""" + + # see _try_bind_args() + try: + inspect.signature + + @staticmethod + def required_args(fn): + return [ + param.name for param in inspect.signature(fn).parameters.values() + if (param.kind in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) + and param.default is inspect.Parameter.empty)] + + except AttributeError: + + # Py < 3.3 + @staticmethod + def required_args(fn): + fn_args = inspect.getargspec(fn) + n_defaults = len(fn_args.defaults or []) + return (fn_args.args or [])[:-n_defaults if n_defaults > 0 else None] + + def __new__(cls, func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + if set(cls.required_args(func)[len(args):]).difference(kwargs): + return functools.partial(func, *args, **kwargs) + return func(*args, **kwargs) + + return wrapped + + if sys.version_info >= (2, 7): def find_xpath_attr(node, xpath, key, val=None): """ Find the xpath xpath[@key=val] """ From a9b4649d928b397a0b9f60fc7f9311e2be57d4b0 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 1 Nov 2025 20:35:11 +0000 Subject: [PATCH 25/43] [utils] Apply `partial_application` decorator to existing functions Thx: yt-dlp/yt-dlp#10653 (etc) --- test/test_utils.py | 4 ++-- youtube_dl/utils.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index 9aca4df63..baadbb5fa 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1730,13 +1730,13 @@ Line 1 callable(test_fn(kwarg=10)), 'missing positional parameter should apply partially') self.assertEqual( - test_fn(10, kwarg=0.1), '10, kwarg=0.1', + test_fn(10, kwarg=42), '10, kwarg=42', 'positionally passed argument should call function') self.assertEqual( test_fn(x=10), '10, kwarg=None', 'keyword passed positional should call function') self.assertEqual( - test_fn(kwarg=0.1)(10), '10, kwarg=0.1', + test_fn(kwarg=42)(10), '10, kwarg=42', 'call after partial application should call the function') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index f2c02829b..c88d02d35 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3187,6 +3187,7 @@ def extract_timezone(date_str): return timezone, date_str +@partial_application def parse_iso8601(date_str, delimiter='T', timezone=None): """ Return a UNIX timestamp from the given date """ @@ -3264,6 +3265,7 @@ def unified_timestamp(date_str, day_first=True): return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone) +@partial_application def determine_ext(url, default_ext='unknown_video'): if url is None or '.' not in url: return default_ext @@ -3842,6 +3844,7 @@ def base_url(url): return re.match(r'https?://[^?#&]+/', url).group() +@partial_application def urljoin(base, path): path = _decode_compat_str(path, encoding='utf-8', or_none=True) if not path: @@ -3866,6 +3869,7 @@ class PUTRequest(compat_urllib_request.Request): return 'PUT' +@partial_application def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1, base=None): if get_attr: if v is not None: @@ -3892,6 +3896,7 @@ def str_to_int(int_str): return int_or_none(int_str) +@partial_application def float_or_none(v, scale=1, invscale=1, default=None): if v is None: return default @@ -4286,6 +4291,7 @@ def urlencode_postdata(*args, **kargs): return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii') +@partial_application def update_url(url, **kwargs): """Replace URL components specified by kwargs url: compat_str or parsed URL tuple @@ -4307,6 +4313,7 @@ def update_url(url, **kwargs): return compat_urllib_parse.urlunparse(url._replace(**kwargs)) +@partial_application def update_url_query(url, query): return update_url(url, query_update=query) From 70b40dd1efeb7d4afc7f860ddcaea28455a64d11 Mon Sep 17 00:00:00 2001 From: dirkf Date: Sat, 1 Nov 2025 20:37:55 +0000 Subject: [PATCH 26/43] [utils] Add `subs_list_to_dict()` traversal helper Thx: yt-dlp/yt-dlp#10653, etc --- test/test_traversal.py | 101 ++++++++++++++++++++++++++++++++++++++++ youtube_dl/traversal.py | 1 + youtube_dl/utils.py | 45 ++++++++++++++++++ 3 files changed, 147 insertions(+) diff --git a/test/test_traversal.py b/test/test_traversal.py index 5d08b8dbb..504cdee37 100644 --- a/test/test_traversal.py +++ b/test/test_traversal.py @@ -16,6 +16,7 @@ from youtube_dl.traversal import ( dict_get, get_first, require, + subs_list_to_dict, T, traverse_obj, unpack, @@ -30,6 +31,7 @@ from youtube_dl.compat import ( compat_zip as zip, ) from youtube_dl.utils import ( + determine_ext, ExtractorError, int_or_none, join_nonempty, @@ -495,6 +497,105 @@ class TestTraversalHelpers(_TestCase): traverse_obj(_TEST_DATA, ('str', T(require('value')))), 'str', '`require` should pass through non-`None` values') + def test_subs_list_to_dict(self): + self.assertEqual(traverse_obj([ + {'name': 'de', 'url': 'https://example.com/subs/de.vtt'}, + {'name': 'en', 'url': 'https://example.com/subs/en1.ass'}, + {'name': 'en', 'url': 'https://example.com/subs/en2.ass'}, + ], [Ellipsis, { + 'id': 'name', + 'url': 'url', + }, all, T(subs_list_to_dict)]), { + 'de': [{'url': 'https://example.com/subs/de.vtt'}], + 'en': [ + {'url': 'https://example.com/subs/en1.ass'}, + {'url': 'https://example.com/subs/en2.ass'}, + ], + }, 'function should build subtitle dict from list of subtitles') + self.assertEqual(traverse_obj([ + {'name': 'de', 'url': 'https://example.com/subs/de.ass'}, + {'name': 'de'}, + {'name': 'en', 'content': 'content'}, + {'url': 'https://example.com/subs/en'}, + ], [Ellipsis, { + 'id': 'name', + 'data': 'content', + 'url': 'url', + }, all, T(subs_list_to_dict(lang=None))]), { + 'de': [{'url': 'https://example.com/subs/de.ass'}], + 'en': [{'data': 'content'}], + }, 'subs with mandatory items missing should be filtered') + self.assertEqual(traverse_obj([ + {'url': 'https://example.com/subs/de.ass', 'name': 'de'}, + {'url': 'https://example.com/subs/en', 'name': 'en'}, + ], [Ellipsis, { + 'id': 'name', + 'ext': ['url', T(determine_ext(default_ext=None))], + 'url': 'url', + }, all, T(subs_list_to_dict(ext='ext'))]), { + 'de': [{'url': 'https://example.com/subs/de.ass', 'ext': 'ass'}], + 'en': [{'url': 'https://example.com/subs/en', 'ext': 'ext'}], + }, '`ext` should set default ext but leave existing value untouched') + self.assertEqual(traverse_obj([ + {'name': 'en', 'url': 'https://example.com/subs/en2', 'prio': True}, + {'name': 'en', 'url': 'https://example.com/subs/en1', 'prio': False}, + ], [Ellipsis, { + 'id': 'name', + 'quality': ['prio', T(int)], + 'url': 'url', + }, all, T(subs_list_to_dict(ext='ext'))]), {'en': [ + {'url': 'https://example.com/subs/en1', 'ext': 'ext'}, + {'url': 'https://example.com/subs/en2', 'ext': 'ext'}, + ]}, '`quality` key should sort subtitle list accordingly') + self.assertEqual(traverse_obj([ + {'name': 'de', 'url': 'https://example.com/subs/de.ass'}, + {'name': 'de'}, + {'name': 'en', 'content': 'content'}, + {'url': 'https://example.com/subs/en'}, + ], [Ellipsis, { + 'id': 'name', + 'url': 'url', + 'data': 'content', + }, all, T(subs_list_to_dict(lang='en'))]), { + 'de': [{'url': 'https://example.com/subs/de.ass'}], + 'en': [ + {'data': 'content'}, + {'url': 'https://example.com/subs/en'}, + ], + }, 'optionally provided lang should be used if no id available') + self.assertEqual(traverse_obj([ + {'name': 1, 'url': 'https://example.com/subs/de1'}, + {'name': {}, 'url': 'https://example.com/subs/de2'}, + {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'}, + {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'}, + ], [Ellipsis, { + 'id': 'name', + 'url': 'url', + 'ext': 'ext', + }, all, T(subs_list_to_dict(lang=None))]), { + 'de': [ + {'url': 'https://example.com/subs/de3'}, + {'url': 'https://example.com/subs/de4'}, + ], + }, 'non str types should be ignored for id and ext') + self.assertEqual(traverse_obj([ + {'name': 1, 'url': 'https://example.com/subs/de1'}, + {'name': {}, 'url': 'https://example.com/subs/de2'}, + {'name': 'de', 'ext': 1, 'url': 'https://example.com/subs/de3'}, + {'name': 'de', 'ext': {}, 'url': 'https://example.com/subs/de4'}, + ], [Ellipsis, { + 'id': 'name', + 'url': 'url', + 'ext': 'ext', + }, all, T(subs_list_to_dict(lang='de'))]), { + 'de': [ + {'url': 'https://example.com/subs/de1'}, + {'url': 'https://example.com/subs/de2'}, + {'url': 'https://example.com/subs/de3'}, + {'url': 'https://example.com/subs/de4'}, + ], + }, 'non str types should be replaced by default id') + def test_unpack(self): self.assertEqual( unpack(lambda *x: ''.join(map(compat_str, x)))([1, 2, 3]), '123') diff --git a/youtube_dl/traversal.py b/youtube_dl/traversal.py index e4e8758c6..1de48b145 100644 --- a/youtube_dl/traversal.py +++ b/youtube_dl/traversal.py @@ -6,6 +6,7 @@ from .utils import ( dict_get, get_first, require, + subs_list_to_dict, T, traverse_obj, unpack, diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index c88d02d35..bd8d62572 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -6599,6 +6599,51 @@ class require(ExtractorError): return value +@partial_application +# typing: (subs: list[dict], /, *, lang='und', ext=None) -> dict[str, list[dict] +def subs_list_to_dict(subs, lang='und', ext=None): + """ + Convert subtitles from a traversal into a subtitle dict. + The path should have an `all` immediately before this function. + + Arguments: + `lang` The default language tag for subtitle dicts with no + `lang` (`und`: undefined) + `ext` The default value for `ext` in the subtitle dicts + + In the dict you can set the following additional items: + `id` The language tag to which the subtitle dict should be added + `quality` The sort order for each subtitle dict + """ + + result = collections.defaultdict(list) + + for sub in subs: + tn_url = url_or_none(sub.pop('url', None)) + if tn_url: + sub['url'] = tn_url + elif not sub.get('data'): + continue + sub_lang = sub.pop('id', None) + if not isinstance(sub_lang, compat_str): + if not lang: + continue + sub_lang = lang + sub_ext = sub.get('ext') + if not isinstance(sub_ext, compat_str): + if not ext: + sub.pop('ext', None) + else: + sub['ext'] = ext + result[sub_lang].append(sub) + result = dict(result) + + for subs in result.values(): + subs.sort(key=lambda x: x.pop('quality', 0) or 0) + + return result + + def unpack(func, **kwargs): """Make a function that applies `partial(func, **kwargs)` to its argument as *args""" @functools.wraps(func) From 27867cc814b60b7a9e95de4e971a3743f33bbd96 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 3 Nov 2025 20:41:04 +0000 Subject: [PATCH 27/43] [compat] Add `compat_thread` --- youtube_dl/compat.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 96b099a58..25fbc8edd 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -3637,6 +3637,16 @@ except ImportError: compat_zstandard = None +# compat_thread +try: + import _thread as compat_thread +except ImportError: + try: + import thread as compat_thread + except ImportError: + import dummy_thread as compat_thread + + legacy = [ 'compat_HTMLParseError', 'compat_HTMLParser', @@ -3722,6 +3732,7 @@ __all__ = [ 'compat_struct_unpack', 'compat_subprocess_get_DEVNULL', 'compat_subprocess_Popen', + 'compat_thread', 'compat_tokenize_tokenize', 'compat_urllib_error', 'compat_urllib_parse', From 931e15621cadbdeea4e16c533177d749f159b6e3 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 23:48:37 +0000 Subject: [PATCH 28/43] [compat] Add `compat_abc_ABC` Base class for abstract classes --- youtube_dl/compat.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 25fbc8edd..8ebd7f742 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -55,7 +55,7 @@ except AttributeError: try: import collections.abc as compat_collections_abc except ImportError: - import collections as compat_collections_abc + compat_collections_abc = collections # compat_urllib_request @@ -3483,6 +3483,15 @@ except ImportError: from itertools import izip_longest as compat_itertools_zip_longest +# compat_abc_ABC +try: + from abc import ABC as compat_abc_ABC +except ImportError: + # Py < 3.4 + from abc import ABCMeta as _ABCMeta + compat_abc_ABC = _ABCMeta(str('ABC'), (object,), {}) + + # compat_collections_chain_map # collections.ChainMap: new class try: @@ -3677,6 +3686,7 @@ legacy = [ __all__ = [ 'compat_Struct', + 'compat_abc_ABC', 'compat_base64_b64decode', 'compat_basestring', 'compat_brotli', From 5585d76da68931ca39b3edd137382bed78746251 Mon Sep 17 00:00:00 2001 From: dirkf Date: Mon, 3 Nov 2025 20:45:53 +0000 Subject: [PATCH 29/43] [compat] Add `compat_dict` A dict that preserves insertion order and otherwise resembles the dict builtin (if it isn't it) rather than `collections.OrderedDict`. Also: * compat_builtins_dict: the built-in definition in case `compat_dict` was imported as `dict` * compat_dict_items: use instead of `dict.items` to get items from a `compat_dict` in insertion order, if you didn't define `dict` as `compat_dict`. --- youtube_dl/compat.py | 141 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index 8ebd7f742..a985cb03e 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -3492,6 +3492,31 @@ except ImportError: compat_abc_ABC = _ABCMeta(str('ABC'), (object,), {}) +# dict mixin used here +# like UserDict.DictMixin, without methods created by MutableMapping +class _DictMixin(compat_abc_ABC): + def has_key(self, key): + return key in self + + # get(), clear(), setdefault() in MM + + def iterkeys(self): + return (k for k in self) + + def itervalues(self): + return (self[k] for k in self) + + def iteritems(self): + return ((k, self[k]) for k in self) + + # pop(), popitem() in MM + + def copy(self): + return type(self)(self) + + # update() in MM + + # compat_collections_chain_map # collections.ChainMap: new class try: @@ -3656,6 +3681,119 @@ except ImportError: import dummy_thread as compat_thread +# compat_dict +# compat_builtins_dict +# compat_dict_items +if sys.version_info >= (3, 6): + compat_dict = compat_builtins_dict = dict + compat_dict_items = dict.items +else: + _get_ident = compat_thread.get_ident + + class compat_dict(compat_collections_abc.MutableMapping, _DictMixin, dict): + """`dict` that preserves insertion order with interface like Py3.7+""" + + _order = [] # default that should never be used + + def __init__(self, *mappings_or_iterables, **kwargs): + # order an unordered dict using a list of keys: actual Py 2.7+ + # OrderedDict uses a doubly linked list for better performance + self._order = [] + for arg in mappings_or_iterables: + self.__update(arg) + if kwargs: + self.__update(kwargs) + + def __getitem__(self, key): + return dict.__getitem__(self, key) + + def __setitem__(self, key, value): + try: + if key not in self._order: + self._order.append(key) + dict.__setitem__(self, key, value) + except Exception: + if key in self._order[-1:] and key not in self: + del self._order[-1] + raise + + def __len__(self): + return dict.__len__(self) + + def __delitem__(self, key): + dict.__delitem__(self, key) + try: + # expected case, O(len(self)), but who dels anyway? + self._order.remove(key) + except ValueError: + pass + + def __iter__(self): + for from_ in self._order: + if from_ in self: + yield from_ + + def __del__(self): + for attr in ('_order',): + try: + delattr(self, attr) + except Exception: + pass + + def __repr__(self, _repr_running={}): + # skip recursive items ... + call_key = id(self), _get_ident() + if _repr_running.get(call_key): + return '...' + _repr_running[call_key] = True + try: + return '%s({%s})' % ( + type(self).__name__, + ','.join('%r: %r' % k_v for k_v in self.items())) + finally: + del _repr_running[call_key] + + # merge/update (PEP 584) + + def __or__(self, other): + if not isinstance(other, compat_collections_abc.Mapping): + return NotImplemented + new = type(self)(self) + new.update(other) + return new + + def __ror__(self, other): + if not isinstance(other, compat_collections_abc.Mapping): + return NotImplemented + new = type(other)(other) + new.update(self) + return new + + def __ior__(self, other): + self.update(other) + return self + + # optimisations + + def __reversed__(self): + for from_ in reversed(self._order): + if from_ in self: + yield from_ + + def __contains__(self, item): + return dict.__contains__(self, item) + + # allow overriding update without breaking __init__ + def __update(self, *args, **kwargs): + super(compat_dict, self).update(*args, **kwargs) + + compat_builtins_dict = dict + # Using the object's method, not dict's: + # an ordered dict's items can be returned unstably by unordered + # dict.items as if the method was not ((k, self[k]) for k in self) + compat_dict_items = lambda d: d.items() + + legacy = [ 'compat_HTMLParseError', 'compat_HTMLParser', @@ -3690,6 +3828,7 @@ __all__ = [ 'compat_base64_b64decode', 'compat_basestring', 'compat_brotli', + 'compat_builtins_dict', 'compat_casefold', 'compat_chr', 'compat_collections_abc', @@ -3697,6 +3836,8 @@ __all__ = [ 'compat_contextlib_suppress', 'compat_ctypes_WINFUNCTYPE', 'compat_datetime_timedelta_total_seconds', + 'compat_dict', + 'compat_dict_items', 'compat_etree_fromstring', 'compat_etree_iterfind', 'compat_filter', From 7a488f7faef0dcb60cab77304313d9eced03987e Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 06:22:02 +0000 Subject: [PATCH 30/43] [utils] Stabilise traversal results using `compat_dict` In `traverse_obj()`, use `compat_dict` to construct dicts, ensuring insertion order sort, but`compat_builtin_dict` to test for dict-iness... --- youtube_dl/utils.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index bd8d62572..edac2456d 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -6367,6 +6367,11 @@ def traverse_obj(obj, *paths, **kwargs): # instant compat str = compat_str + from .compat import ( + compat_builtins_dict as dict_, # the basic dict type + compat_dict as dict, # dict preserving imsertion order + ) + casefold = lambda k: compat_casefold(k) if isinstance(k, str) else k if isinstance(expected_type, type): @@ -6449,7 +6454,7 @@ def traverse_obj(obj, *paths, **kwargs): if not branching: # string traversal result = ''.join(result) - elif isinstance(key, dict): + elif isinstance(key, dict_): iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items()) result = dict((k, v if v is not None else default) for k, v in iter_obj if v is not None or default is not NO_DEFAULT) or None @@ -6527,7 +6532,7 @@ def traverse_obj(obj, *paths, **kwargs): has_branched = False key = None - for last, key in lazy_last(variadic(path, (str, bytes, dict, set))): + for last, key in lazy_last(variadic(path, (str, bytes, dict_, set))): if not casesense and isinstance(key, str): key = compat_casefold(key) @@ -6557,10 +6562,10 @@ def traverse_obj(obj, *paths, **kwargs): objs = from_iterable(new_objs) - if test_type and not isinstance(key, (dict, list, tuple)): + if test_type and not isinstance(key, (dict_, list, tuple)): objs = map(type_test, objs) - return objs, has_branched, isinstance(key, dict) + return objs, has_branched, isinstance(key, dict_) def _traverse_obj(obj, path, allow_empty, test_type): results, has_branched, is_dict = apply_path(obj, path, test_type) From 43e3121020c95b6b31e8fe629ef170b448c2ff70 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 20:13:19 +0000 Subject: [PATCH 31/43] [utils] Align `parse_duration()` behaviour with yt-dlp * handle comma-separated long-form durations * support : as millisecond separator. --- test/test_utils.py | 6 ++++++ youtube_dl/utils.py | 46 ++++++++++++++++++++++++--------------------- 2 files changed, 31 insertions(+), 21 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index baadbb5fa..b9db2d45a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -665,6 +665,8 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_duration('3h 11m 53s'), 11513) self.assertEqual(parse_duration('3 hours 11 minutes 53 seconds'), 11513) self.assertEqual(parse_duration('3 hours 11 mins 53 secs'), 11513) + self.assertEqual(parse_duration('3 hours, 11 minutes, 53 seconds'), 11513) + self.assertEqual(parse_duration('3 hours, 11 mins, 53 secs'), 11513) self.assertEqual(parse_duration('62m45s'), 3765) self.assertEqual(parse_duration('6m59s'), 419) self.assertEqual(parse_duration('49s'), 49) @@ -683,6 +685,10 @@ class TestUtil(unittest.TestCase): self.assertEqual(parse_duration('PT1H0.040S'), 3600.04) self.assertEqual(parse_duration('PT00H03M30SZ'), 210) self.assertEqual(parse_duration('P0Y0M0DT0H4M20.880S'), 260.88) + self.assertEqual(parse_duration('01:02:03:050'), 3723.05) + self.assertEqual(parse_duration('103:050'), 103.05) + self.assertEqual(parse_duration('1HR 3MIN'), 3780) + self.assertEqual(parse_duration('2hrs 3mins'), 7380) def test_fix_xml_ampersands(self): self.assertEqual( diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index edac2456d..b93f4be5c 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -3931,38 +3931,46 @@ def parse_duration(s): return None s = s.strip() + if not s: + return None days, hours, mins, secs, ms = [None] * 5 - m = re.match(r'(?:(?:(?:(?P[0-9]+):)?(?P[0-9]+):)?(?P[0-9]+):)?(?P[0-9]+)(?P\.[0-9]+)?Z?$', s) + m = re.match(r'''(?x) + (?P + (?:(?:(?P[0-9]+):)?(?P[0-9]+):)? + (?P[0-9]+):)? + (?P(?(before_secs)[0-9]{1,2}|[0-9]+)) + (?:[.:](?P[0-9]+))?Z?$ + ''', s) if m: - days, hours, mins, secs, ms = m.groups() + days, hours, mins, secs, ms = m.group('days', 'hours', 'mins', 'secs', 'ms') else: m = re.match( r'''(?ix)(?:P? (?: - [0-9]+\s*y(?:ears?)?\s* + [0-9]+\s*y(?:ears?)?,?\s* )? (?: - [0-9]+\s*m(?:onths?)?\s* + [0-9]+\s*m(?:onths?)?,?\s* )? (?: - [0-9]+\s*w(?:eeks?)?\s* + [0-9]+\s*w(?:eeks?)?,?\s* )? (?: - (?P[0-9]+)\s*d(?:ays?)?\s* + (?P[0-9]+)\s*d(?:ays?)?,?\s* )? T)? (?: - (?P[0-9]+)\s*h(?:ours?)?\s* + (?P[0-9]+)\s*h(?:(?:ou)?rs?)?,?\s* )? (?: - (?P[0-9]+)\s*m(?:in(?:ute)?s?)?\s* + (?P[0-9]+)\s*m(?:in(?:ute)?s?)?,?\s* )? (?: - (?P[0-9]+)(?P\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s* + (?P[0-9]+)(?:\.(?P[0-9]+))?\s*s(?:ec(?:ond)?s?)?\s* )?Z?$''', s) if m: - days, hours, mins, secs, ms = m.groups() + days, hours, mins, secs, ms = m.group('days', 'hours', 'mins', 'secs', 'ms') else: m = re.match(r'(?i)(?:(?P[0-9.]+)\s*(?:hours?)|(?P[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s) if m: @@ -3970,17 +3978,13 @@ def parse_duration(s): else: return None - duration = 0 - if secs: - duration += float(secs) - if mins: - duration += float(mins) * 60 - if hours: - duration += float(hours) * 60 * 60 - if days: - duration += float(days) * 24 * 60 * 60 - if ms: - duration += float(ms) + duration = ( + ((((float(days) * 24) if days else 0) + + (float(hours) if hours else 0)) * 60 + + (float(mins) if mins else 0)) * 60 + + (float(secs) if secs else 0) + + (float(ms) / 10 ** len(ms) if ms else 0)) + return duration From c55ace3c503c7bddb9c25112bd19bea282bbae59 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 20:51:08 +0000 Subject: [PATCH 32/43] [YouTube] Use insertion-order-preserving dict for InnerTube client data --- youtube_dl/extractor/youtube.py | 45 ++++++++++++++++++++++----------- 1 file changed, 30 insertions(+), 15 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c045bc8bc..ce23b39bb 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -17,6 +17,8 @@ from ..compat import ( compat_chr, compat_HTTPError, compat_map as map, + compat_dict as o_dict, + compat_dict_items as dict_items, compat_str, compat_urllib_parse, compat_urllib_parse_parse_qs as compat_parse_qs, @@ -86,8 +88,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor): _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)' - _INNERTUBE_CLIENTS = { - 'ios': { + # priority order for now + _INNERTUBE_CLIENTS = o_dict(( + ('ios', { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'IOS', @@ -100,12 +103,13 @@ class YoutubeBaseInfoExtractor(InfoExtractor): }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, - 'REQUIRE_PO_TOKEN': False, + 'REQUIRE_PO_TOKEN': True, 'REQUIRE_JS_PLAYER': False, - }, + 'WITH_COOKIES': False, + }), # mweb has 'ultralow' formats # See: https://github.com/yt-dlp/yt-dlp/pull/557 - 'mweb': { + ('mweb', { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'MWEB', @@ -117,8 +121,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'INNERTUBE_CONTEXT_CLIENT_NAME': 2, 'REQUIRE_PO_TOKEN': True, 'SUPPORTS_COOKIES': True, - }, - 'tv': { + }), + ('tv', { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'TVHTML5', @@ -128,10 +132,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor): }, }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, - 'SUPPORTS_COOKIES': True, - }, - - 'web': { + }), + ('web', { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'WEB', @@ -141,10 +143,20 @@ class YoutubeBaseInfoExtractor(InfoExtractor): }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, 'REQUIRE_PO_TOKEN': True, + }), + ('web_embedded', { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'WEB_EMBEDDED_PLAYER', + 'clientVersion': '1.20250923.21.00', + 'embedUrl': 'https://www.youtube.com/', # Can be any valid URL + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 56, 'SUPPORTS_COOKIES': True, - }, + }), # Safari UA returns pre-merged video+audio 144p/240p/360p/720p/1080p HLS formats - 'web_safari': { + ('web_safari', { 'INNERTUBE_CONTEXT': { 'client': { 'clientName': 'WEB', @@ -152,8 +164,11 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'userAgent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15,gzip(gfe)', }, }, - }, - } + 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, + 'SUPPORTS_COOKIES': True, + 'REQUIRE_PO': True, + }), + )) def _login(self): """ From a1e2c7d90b2a5e67acf489483d5e583e588a272a Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 20:52:15 +0000 Subject: [PATCH 33/43] [YouTube] Add further InnerTube clients FWIW: android-sdkless, tv_downgraded, web_creator Thx yt-dlp passim --- youtube_dl/extractor/youtube.py | 38 +++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index ce23b39bb..3d89d5319 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -90,6 +90,21 @@ class YoutubeBaseInfoExtractor(InfoExtractor): # priority order for now _INNERTUBE_CLIENTS = o_dict(( + # Doesn't require a PoToken for some reason: thx yt-dlp/yt-dlp#14693 + ('android_sdkless', { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'ANDROID', + 'clientVersion': '20.10.38', + 'userAgent': 'com.google.android.youtube/20.10.38 (Linux; U; Android 11) gzip', + 'osName': 'Android', + 'osVersion': '11', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 3, + 'REQUIRE_JS_PLAYER': False, + 'WITH_COOKIES': False, + }), ('ios', { 'INNERTUBE_CONTEXT': { 'client': { @@ -120,6 +135,16 @@ class YoutubeBaseInfoExtractor(InfoExtractor): }, 'INNERTUBE_CONTEXT_CLIENT_NAME': 2, 'REQUIRE_PO_TOKEN': True, + }), + ('tv_downgraded', { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'TVHTML5', + 'clientVersion': '4', # avoids SABR formats, thx yt-dlp/yt-dlp#14887 + 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, 'SUPPORTS_COOKIES': True, }), ('tv', { @@ -168,6 +193,19 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'SUPPORTS_COOKIES': True, 'REQUIRE_PO': True, }), + # This client now requires sign-in for every video + ('web_creator', { + 'INNERTUBE_CONTEXT': { + 'client': { + 'clientName': 'WEB_CREATOR', + 'clientVersion': '1.20250922.03.00', + }, + }, + 'INNERTUBE_CONTEXT_CLIENT_NAME': 62, + 'REQUIRE_AUTH': True, + 'SUPPORTS_COOKIES': True, + 'WITH_COOKIES': True, + }), )) def _login(self): From 5d445f8c5fff7d2cf495d4b20ca41526350b3513 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 20:58:12 +0000 Subject: [PATCH 34/43] [YouTube] Re-work client selection * use `android_sdkless` by default * use `web_safari` (HLS only) if logged in * skip any non-HLS format with n-challenge --- youtube_dl/extractor/youtube.py | 74 ++++++++++++++++++++++----------- 1 file changed, 50 insertions(+), 24 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 3d89d5319..a095e1db0 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -483,6 +483,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor): ('responseContext', 'visitorData')), T(compat_str))) + # @functools.cached_property + def is_authenticated(self, _cache={}): + if self not in _cache: + _cache[self] = bool(self._generate_sapisidhash_header()) + return _cache[self] + def _extract_ytcfg(self, video_id, webpage): ytcfg = self._search_json( r'ytcfg\.set\s*\(', webpage, 'ytcfg', video_id, @@ -2101,8 +2107,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return self._cached(self._decrypt_nsig, 'nsig', n, player_url) for fmt in formats: - parsed_fmt_url = compat_urllib_parse.urlparse(fmt['url']) - n_param = compat_parse_qs(parsed_fmt_url.query).get('n') + n_param = parse_qs(fmt['url']).get('n') if not n_param: continue n_param = n_param[-1] @@ -2268,6 +2273,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor): (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE), regex), webpage, name, default='{}'), video_id, fatal=False) + def _is_premium_subscriber(self, initial_data): + if not self.is_authenticated or not initial_data: + return False + + tlr = traverse_obj( + initial_data, ('topbar', 'desktopTopbarRenderer', 'logo', 'topbarLogoRenderer')) + return ( + traverse_obj(tlr, ('iconImage', 'iconType')) == 'YOUTUBE_PREMIUM_LOGO' + or 'premium' in (self._get_text(tlr, 'tooltipText') or '').lower() + ) + def _real_extract(self, url): url, smuggled_data = unsmuggle_url(url, {}) video_id = self._match_id(url) @@ -2303,24 +2319,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if sts: pb_context['signatureTimestamp'] = sts - client_names = traverse_obj(self._INNERTUBE_CLIENTS, ( - T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'), - 0))[:1] + auth = self._generate_sapisidhash_header(origin) + + client_names = [] + if auth or self._is_premium_subscriber(player_response): + client_names = traverse_obj(self._INNERTUBE_CLIENTS, ( + T(dict_items), lambda _, k_v: k_v[0] == 'web_safari', 0))[:1] + if not client_names: + client_names = traverse_obj(self._INNERTUBE_CLIENTS, ( + T(dict_items), lambda _, k_v: not ( + k_v[1].get('REQUIRE_PO_TOKEN') + or (bool(k_v[1].get('WITH_COOKIES', auth)) ^ bool(auth)) + ), 0))[:1] if 'web' not in client_names: - # webpage links won't download: ignore links and playability + # only live HLS webpage links will download: ignore playability player_response = filter_dict( player_response or {}, - lambda k, _: k not in ('streamingData', 'playabilityStatus')) - - if is_live and 'ios' not in client_names: - client_names.append('ios') + lambda k, _: k != 'playabilityStatus') headers = { 'Sec-Fetch-Mode': 'navigate', 'Origin': origin, 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '', } - auth = self._generate_sapisidhash_header(origin) + if auth is not None: headers['Authorization'] = auth headers['X-Origin'] = origin @@ -2350,7 +2372,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'INNERTUBE_CONTEXT', 'client', 'clientVersion'), 'User-Agent': ( 'INNERTUBE_CONTEXT', 'client', 'userAgent'), - })) + }) or {}) api_player_response = self._call_api( 'player', query, video_id, fatal=False, headers=api_headers, @@ -2359,19 +2381,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'context', 'client', 'clientName')), 'API JSON', delim=' ')) - hls = traverse_obj( - (player_response, api_player_response), - (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none))) + # be sure to find HLS in case of is_live + hls = traverse_obj(player_response, ( + 'streamingData', 'hlsManifestUrl', T(url_or_none))) fetched_timestamp = int(time.time()) - if len(hls) == 2 and not hls[0] and hls[1]: - player_response['streamingData']['hlsManifestUrl'] = hls[1] - else: - video_details = merge_dicts(*traverse_obj( - (player_response, api_player_response), - (Ellipsis, 'videoDetails', T(dict)))) - player_response.update(filter_dict( - api_player_response or {}, cndn=lambda k, _: k != 'captions')) - player_response['videoDetails'] = video_details + video_details = merge_dicts(*traverse_obj( + (player_response, api_player_response), + (Ellipsis, 'videoDetails', T(dict)))) + player_response.update(filter_dict( + api_player_response or {}, cndn=lambda k, _: k != 'captions')) + player_response['videoDetails'] = video_details + if hls and not traverse_obj(player_response, ( + 'streamingData', 'hlsManifestUrl', T(url_or_none))): + player_response['streamingData']['hlsManifestUrl'] = hls def is_agegated(playability): # playability: dict @@ -2575,6 +2597,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self.write_debug(error_to_compat_str(e), only_once=True) continue + if parse_qs(fmt_url).get('n'): + # this and (we assume) all the formats here are n-scrambled + break + language_preference = ( 10 if audio_track.get('audioIsDefault') else -10 if 'descriptive' in (traverse_obj(audio_track, ('displayName', T(lower))) or '') From 6f5d4c32897090d7222df451d09e00cf2e9994e1 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 21:28:46 +0000 Subject: [PATCH 35/43] [YouTube] Improve targeting of pre-roll wait Experimental for now. Thx: yt-dlp/yt-dlp#14646 --- youtube_dl/extractor/youtube.py | 39 ++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index a095e1db0..442b9fac4 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2273,6 +2273,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor): (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE), regex), webpage, name, default='{}'), video_id, fatal=False) + def _get_preroll_length(self, ad_slot_lists): + + def parse_instream_ad_renderer(instream_renderer): + for skippable, path in ( + ('', ('skipOffsetMilliseconds', T(int))), + ('non-', ('playerVars', T(compat_parse_qs), + 'length_seconds', -1, T(int_or_none(invscale=1000))))): + length_ms = traverse_obj(instream_renderer, path) + if length_ms is not None: + self.write_debug('Detected a %ds %sskippable ad' % ( + length_ms // 1000, skippable)) + break + return length_ms + + for slot_renderer in traverse_obj(ad_slot_lists, ('adSlots', Ellipsis, 'adSlotRenderer', T(dict))): + if traverse_obj(slot_renderer, ('adSlotMetadata', 'triggerEvent')) != 'SLOT_TRIGGER_EVENT_BEFORE_CONTENT': + continue + rendering_content = traverse_obj(slot_renderer, ( + 'fulfillmentContent', 'fulfilledLayout', 'playerBytesAdLayoutRenderer', + 'renderingContent', 'instreamVideoAdRenderer', T(dict))) + length_ms = parse_instream_ad_renderer(rendering_content) + if length_ms is not None: + return length_ms + times = traverse_obj(rendering_content, (( + ('playerBytesSequentialLayoutRenderer', 'sequentialLayouts'), + None), any, Ellipsis, 'playerBytesAdLayoutRenderer', + 'renderingContent', 'instreamVideoAdRenderer', + T(parse_instream_ad_renderer))) + if times: + return sum(times) + return 0 + def _is_premium_subscriber(self, initial_data): if not self.is_authenticated or not initial_data: return False @@ -2311,8 +2343,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): if True or not player_response: origin = 'https://www.youtube.com' pb_context = {'html5Preference': 'HTML5_PREF_WANTS'} - fetched_timestamp = int(time.time()) - player_url = self._extract_player_url(webpage) ytcfg = self._extract_ytcfg(video_id, webpage or '') sts = self._extract_signature_timestamp(video_id, player_url, ytcfg) @@ -2385,6 +2415,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): hls = traverse_obj(player_response, ( 'streamingData', 'hlsManifestUrl', T(url_or_none))) fetched_timestamp = int(time.time()) + preroll_length_ms = ( + self._get_preroll_length(api_player_response) + or self._get_preroll_length(player_response)) video_details = merge_dicts(*traverse_obj( (player_response, api_player_response), (Ellipsis, 'videoDetails', T(dict)))) @@ -2551,7 +2584,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): elif fetched_timestamp is not None: # Handle preroll waiting period preroll_sleep = self.get_param('youtube_preroll_sleep') - preroll_sleep = int_or_none(preroll_sleep, default=6) + preroll_sleep = min(6, int_or_none(preroll_sleep, default=preroll_length_ms / 1000)) fetched_timestamp += preroll_sleep for fmt in streaming_formats: From 39378f7b5cdd6e11ff0ba6ec0f6b4a7788cfea9a Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 21:32:06 +0000 Subject: [PATCH 36/43] [YouTube] Fix incorrect chapter extraction * align `_get_text()` with yt-dlp (thx, passim) at last --- youtube_dl/extractor/youtube.py | 55 +++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 23 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 442b9fac4..c0b097fbe 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -533,6 +533,27 @@ class YoutubeBaseInfoExtractor(InfoExtractor): 'uploader': uploader, } + @staticmethod + def _get_text(data, *path_list, **kw_max_runs): + max_runs = kw_max_runs.get('max_runs') + + for path in path_list or [None]: + if path is None: + obj = [data] # shortcut + else: + obj = traverse_obj(data, tuple(variadic(path) + (all,))) + for runs in traverse_obj( + obj, ('simpleText', {'text': T(compat_str)}, all, filter), + ('runs', lambda _, r: isinstance(r.get('text'), compat_str), all, filter), + (T(list), lambda _, r: isinstance(r.get('text'), compat_str)), + default=[]): + max_runs = int_or_none(max_runs, default=len(runs)) + if max_runs < len(runs): + runs = runs[:max_runs] + text = ''.join(traverse_obj(runs, (Ellipsis, 'text'))) + if text: + return text + @staticmethod def _extract_thumbnails(data, *path_list, **kw_final_key): """ @@ -2493,10 +2514,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return self.url_result( trailer_video_id, self.ie_key(), trailer_video_id) - def get_text(x): - return ''.join(traverse_obj( - x, (('simpleText',),), ('runs', Ellipsis, 'text'), - expected_type=compat_str)) + get_text = lambda x: self._get_text(x) or '' search_meta = ( (lambda x: self._html_search_meta(x, webpage, default=None)) @@ -2960,24 +2978,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor): chapters = self._extract_chapters_from_json( initial_data, video_id, duration) if not chapters: - for engagment_pannel in (initial_data.get('engagementPanels') or []): - contents = try_get( - engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'], - list) - if not contents: - continue + def chapter_time(mmlir): + return parse_duration( + get_text(mmlir.get('timeDescription'))) - def chapter_time(mmlir): - return parse_duration( - get_text(mmlir.get('timeDescription'))) + for markers in traverse_obj(initial_data, ( + 'engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer', + 'content', 'macroMarkersListRenderer', 'contents', T(list))): chapters = [] - for next_num, content in enumerate(contents, start=1): + for next_num, content in enumerate(markers, start=1): mmlir = content.get('macroMarkersListItemRenderer') or {} start_time = chapter_time(mmlir) - end_time = (traverse_obj( - contents, (next_num, 'macroMarkersListItemRenderer', T(chapter_time))) - if next_num < len(contents) else duration) + end_time = (traverse_obj(markers, ( + next_num, 'macroMarkersListItemRenderer', T(chapter_time))) + if next_num < len(markers) else duration) if start_time is None or end_time is None: continue chapters.append({ @@ -3536,12 +3551,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): T(dict.items), lambda _, k_v: k_v[0].startswith('grid') and k_v[0].endswith('Renderer'), 1, T(dict)), get_all=False) - @staticmethod - def _get_text(r, k): - return traverse_obj( - r, (k, 'runs', 0, 'text'), (k, 'simpleText'), - expected_type=txt_or_none) - def _grid_entries(self, grid_renderer): for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))): lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict))) From d65882a0220b5c7dd25b16b90ed6d273aac264d9 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 21:43:43 +0000 Subject: [PATCH 37/43] [YouTube] Improve mark_watched() Thx: Brett824, yt-dlp/yt-dlp#4146 --- youtube_dl/extractor/youtube.py | 41 ++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index c0b097fbe..4a61abcc5 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2177,32 +2177,35 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return sts def _mark_watched(self, video_id, player_response): - playback_url = url_or_none(try_get( - player_response, - lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl'])) - if not playback_url: - return - # cpn generation algorithm is reverse engineered from base.js. # In fact it works even with dummy cpn. CPN_ALPHABET = string.ascii_letters + string.digits + '-_' cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(16)) - # more consistent results setting it to right before the end - qs = parse_qs(playback_url) - video_length = '{0}'.format(float((qs.get('len') or ['1.5'])[0]) - 1) + for is_full, key in enumerate(('videostatsPlaybackUrl', 'videostatsWatchtimeUrl')): + label = 'fully ' if is_full > 0 else '' - playback_url = update_url_query( - playback_url, { - 'ver': '2', - 'cpn': cpn, - 'cmt': video_length, - 'el': 'detailpage', # otherwise defaults to "shorts" - }) + playback_url = traverse_obj(player_response, ( + 'playbackTracking'. key, 'baseUrl', T(url_or_none))) + if not playback_url: + self.report_warning('Unable to mark {0}watched'.format(label)) + continue - self._download_webpage( - playback_url, video_id, 'Marking watched', - 'Unable to mark watched', fatal=False) + # more consistent results setting it to right before the end + qs = parse_qs(playback_url) + video_length = '{0}'.format(float((qs.get('len') or ['1.5'])[0]) - 1) + + playback_url = update_url_query( + playback_url, { + 'ver': '2', + 'cpn': cpn, + 'cmt': video_length, + 'el': 'detailpage', # otherwise defaults to "shorts" + }) + + self._download_webpage( + playback_url, video_id, 'Marking {0}watched'.format(label), + 'Unable to mark watched', fatal=False) @staticmethod def _extract_urls(webpage): From 25890f2ad102f9bfaaea7b725e4977c521908680 Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 21:45:12 +0000 Subject: [PATCH 38/43] [YouTube] Improve detection of geo-restriction Thx yt-dlp --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 4a61abcc5..9a1b87304 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -2787,7 +2787,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): subreason = pemr.get('subreason') if subreason: subreason = clean_html(get_text(subreason)) - if subreason == 'The uploader has not made this video available in your country.': + if subreason.startswith('The uploader has not made this video available in your country'): countries = microformat.get('availableCountries') if not countries: regions_allowed = search_meta('regionsAllowed') From aeb1254fcf89ea43876522371ca6d1e3c2bff25f Mon Sep 17 00:00:00 2001 From: dirkf Date: Tue, 4 Nov 2025 21:52:43 +0000 Subject: [PATCH 39/43] [YouTube] Fix playlist thumbnail extraction Thx seproDev, yt-dlp/yt-dlp#11615 --- youtube_dl/extractor/youtube.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 9a1b87304..044796018 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3664,15 +3664,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True) return + thumb_keys = ('contentImage',) + thumb_keys + ('thumbnailViewModel', 'image') + return merge_dicts(self.url_result( - url, ie=ie.ie_key(), video_id=content_id), { - 'title': traverse_obj(view_model, ( - 'metadata', 'lockupMetadataViewModel', 'title', - 'content', T(compat_str))), - 'thumbnails': self._extract_thumbnails( - view_model, thumb_keys, final_key='sources'), - }) + url, ie=ie.ie_key(), video_id=content_id), + traverse_obj(view_model, { + 'title': ('metadata', 'lockupMetadataViewModel', 'title', + 'content', T(compat_str)), + 'thumbnails': T(lambda vm: self._extract_thumbnails( + vm, thumb_keys, final_key='sources')), + 'duration': ( + 'contentImage', 'thumbnailViewModel', 'overlays', + Ellipsis, ( + ('thumbnailBottomOverlayViewModel', 'badges'), + ('thumbnailOverlayBadgeViewModel', 'thumbnailBadges') + ), Ellipsis, 'thumbnailBadgeViewModel', 'text', + T(parse_duration), any), + }) + ) def _extract_shorts_lockup_view_model(self, view_model): content_id = traverse_obj(view_model, ( From 6315f4b1dfaa735f3ce07ddbcde5e1e2d3b8cef8 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 19 Nov 2025 20:38:14 +0000 Subject: [PATCH 40/43] [utils] Support additional codecs and dynamic_range --- test/test_utils.py | 24 +++++++++++++++++++++ youtube_dl/utils.py | 51 +++++++++++++++++++++++++++++---------------- 2 files changed, 57 insertions(+), 18 deletions(-) diff --git a/test/test_utils.py b/test/test_utils.py index b9db2d45a..1106f2819 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -902,6 +902,30 @@ class TestUtil(unittest.TestCase): 'vcodec': 'av01.0.05M.08', 'acodec': 'none', }) + self.assertEqual(parse_codecs('vp9.2'), { + 'vcodec': 'vp9.2', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('vp09.02.50.10.01.09.18.09.00'), { + 'vcodec': 'vp09.02.50.10.01.09.18.09.00', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('av01.0.12M.10.0.110.09.16.09.0'), { + 'vcodec': 'av01.0.12M.10.0.110.09.16.09.0', + 'acodec': 'none', + 'dynamic_range': 'HDR10', + }) + self.assertEqual(parse_codecs('dvhe'), { + 'vcodec': 'dvhe', + 'acodec': 'none', + 'dynamic_range': 'DV', + }) + self.assertEqual(parse_codecs('fLaC'), { + 'vcodec': 'none', + 'acodec': 'flac', + }) self.assertEqual(parse_codecs('theora, vorbis'), { 'vcodec': 'theora', 'acodec': 'vorbis', diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index b93f4be5c..02a49ff49 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -4744,30 +4744,45 @@ def parse_codecs(codecs_str): if not codecs_str: return {} split_codecs = list(filter(None, map( - lambda str: str.strip(), codecs_str.strip().strip(',').split(',')))) - vcodec, acodec = None, None + lambda s: s.strip(), codecs_str.strip().split(',')))) + vcodec, acodec, hdr = None, None, None for full_codec in split_codecs: - codec = full_codec.split('.')[0] - if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1', 'av01', 'theora'): - if not vcodec: - vcodec = full_codec - elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'): + codec, rest = full_codec.partition('.')[::2] + codec = codec.lower() + full_codec = '.'.join((codec, rest)) if rest else codec + codec = re.sub(r'0+(?=\d)', '', codec) + if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', + 'h263', 'h264', 'mp4v', 'hvc1', 'av1', 'theora', 'dvh1', 'dvhe'): + if vcodec: + continue + vcodec = full_codec + if codec in ('dvh1', 'dvhe'): + hdr = 'DV' + elif codec in ('av1', 'vp9'): + n, m = { + 'av1': (2, '10'), + 'vp9': (0, '2'), + }[codec] + if (rest.split('.', n + 1)[n:] or [''])[0].lstrip('0') == m: + hdr = 'HDR10' + elif codec in ('flac', 'mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-4', + 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'): if not acodec: acodec = full_codec else: - write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr) - if not vcodec and not acodec: - if len(split_codecs) == 2: - return { - 'vcodec': split_codecs[0], - 'acodec': split_codecs[1], - } - else: - return { + write_string('WARNING: Unknown codec %s\n' % (full_codec,), sys.stderr) + + return ( + filter_dict({ 'vcodec': vcodec or 'none', 'acodec': acodec or 'none', - } - return {} + 'dynamic_range': hdr, + }) if vcodec or acodec + else { + 'vcodec': split_codecs[0], + 'acodec': split_codecs[1], + } if len(split_codecs) == 2 + else {}) def urlhandle_detect_ext(url_handle): From d0283f5385acd21dd51afa2102844c8cb2fe6b62 Mon Sep 17 00:00:00 2001 From: dirkf Date: Thu, 20 Nov 2025 17:29:25 +0000 Subject: [PATCH 41/43] [YouTube] Revert forcing player JS by default * still leaving the parameters in place thx bashonly for confirming this suggestion --- youtube_dl/extractor/youtube.py | 33 +++++++++++++++++---------------- youtube_dl/options.py | 4 ++-- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 044796018..7965fa08a 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1699,16 +1699,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor): self._player_cache = {} def _get_player_js_version(self): - player_js_version = self.get_param('youtube_player_js_version') or '20348@0004de42' - sts_hash = self._search_regex( - ('^actual$(^)?(^)?', r'^([0-9]{5,})@([0-9a-f]{8,})$'), - player_js_version, 'player_js_version', group=(1, 2), default=None) - if sts_hash: - return sts_hash - self.report_warning( - 'Invalid player JS version "{0}" specified. ' - 'It should be "{1}" or in the format of {2}'.format( - player_js_version, 'actual', 'SignatureTimeStamp@Hash'), only_once=True) + player_js_version = self.get_param('youtube_player_js_version') + if player_js_version: + sts_hash = self._search_regex( + ('^actual$(^)?(^)?', r'^([0-9]{5,})@([0-9a-f]{8,})$'), + player_js_version, 'player_js_version', group=(1, 2), default=None) + if sts_hash: + return sts_hash + self.report_warning( + 'Invalid player JS version "{0}" specified. ' + 'It should be "{1}" or in the format of {2}'.format( + player_js_version, 'actual', 'SignatureTimeStamp@Hash'), only_once=True) return None, None # *ytcfgs, webpage=None @@ -1723,18 +1724,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},) player_url = traverse_obj( ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'), - get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u)) + get_all=False, expected_type=self._yt_urljoin) - player_id_override = self._get_player_js_version()[1] - - requested_js_variant = self.get_param('youtube_player_js_variant') or 'main' + requested_js_variant = self.get_param('youtube_player_js_variant') variant_js = next( (v for k, v in self._PLAYER_JS_VARIANT_MAP if k == requested_js_variant), None) if variant_js: + player_id_override = self._get_player_js_version()[1] player_id = player_id_override or self._extract_player_info(player_url) original_url = player_url - player_url = '/s/player/{0}/{1}'.format(player_id, variant_js) + player_url = self._yt_urljoin( + '/s/player/{0}/{1}'.format(player_id, variant_js)) if original_url != player_url: self.write_debug( 'Forcing "{0}" player JS variant for player {1}\n' @@ -1748,7 +1749,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): requested_js_variant, ','.join(k for k, _ in self._PLAYER_JS_VARIANT_MAP)), only_once=True) - return urljoin('https://www.youtube.com', player_url) + return player_url def _download_player_url(self, video_id, fatal=False): res = self._download_webpage( diff --git a/youtube_dl/options.py b/youtube_dl/options.py index ce3633c41..9b0d77a23 100644 --- a/youtube_dl/options.py +++ b/youtube_dl/options.py @@ -421,12 +421,12 @@ def parseOpts(overrideArguments=None): action='store', dest='youtube_player_js_variant', help='For YouTube, the player javascript variant to use for n/sig deciphering; `actual` to follow the site; default `%default`.', choices=('actual', 'main', 'tcc', 'tce', 'es5', 'es6', 'tv', 'tv_es6', 'phone', 'tablet'), - default='main', metavar='VARIANT') + default='actual', metavar='VARIANT') video_format.add_option( '--youtube-player-js-version', action='store', dest='youtube_player_js_version', help='For YouTube, the player javascript version to use for n/sig deciphering, specified as `signature_timestamp@hash`, or `actual` to follow the site; default `%default`', - default='20348@0004de42', metavar='STS@HASH') + default='actual', metavar='STS@HASH') video_format.add_option( '--merge-output-format', action='store', dest='merge_output_format', metavar='FORMAT', default=None, From d5f561166b9decff97ad6657cc992c7b0fd1aba2 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 26 Nov 2025 01:16:35 +0000 Subject: [PATCH 42/43] [core] Re-work format_note display in format list with abbreviated codec name --- youtube_dl/YoutubeDL.py | 90 +++++++++++++++------------------ youtube_dl/extractor/youtube.py | 10 ++-- 2 files changed, 47 insertions(+), 53 deletions(-) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index ec1d35c3a..4c762bf2c 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -2404,60 +2404,52 @@ class YoutubeDL(object): return res def _format_note(self, fdict): - res = '' - if fdict.get('ext') in ['f4f', 'f4m']: - res += '(unsupported) ' - if fdict.get('language'): - if res: - res += ' ' - res += '[%s] ' % fdict['language'] - if fdict.get('format_note') is not None: - res += fdict['format_note'] + ' ' - if fdict.get('tbr') is not None: - res += '%4dk ' % fdict['tbr'] + + def simplified_codec(f, field): + assert field in ('acodec', 'vcodec') + codec = f.get(field) + return ( + 'unknown' if not codec + else '.'.join(codec.split('.')[:4]) if codec != 'none' + else 'images' if field == 'vcodec' and f.get('acodec') == 'none' + else None if field == 'acodec' and f.get('vcodec') == 'none' + else 'audio only' if field == 'vcodec' + else 'video only') + + res = join_nonempty( + fdict.get('ext') in ('f4f', 'f4m') and '(unsupported)', + fdict.get('language') and ('[%s]' % (fdict['language'],)), + fdict.get('format_note') is not None and fdict['format_note'], + fdict.get('tbr') is not None and ('%4dk' % fdict['tbr']), + delim=' ') + res = [res] if res else [] if fdict.get('container') is not None: - if res: - res += ', ' - res += '%s container' % fdict['container'] - if (fdict.get('vcodec') is not None - and fdict.get('vcodec') != 'none'): - if res: - res += ', ' - res += fdict['vcodec'] - if fdict.get('vbr') is not None: - res += '@' + res.append('%s container' % (fdict['container'],)) + if fdict.get('vcodec') not in (None, 'none'): + codec = simplified_codec(fdict, 'vcodec') + if codec and fdict.get('vbr') is not None: + codec += '@' elif fdict.get('vbr') is not None and fdict.get('abr') is not None: - res += 'video@' - if fdict.get('vbr') is not None: - res += '%4dk' % fdict['vbr'] + codec = 'video@' + else: + codec = None + codec = join_nonempty(codec, fdict.get('vbr') is not None and ('%4dk' % fdict['vbr'])) + if codec: + res.append(codec) if fdict.get('fps') is not None: - if res: - res += ', ' - res += '%sfps' % fdict['fps'] - if fdict.get('acodec') is not None: - if res: - res += ', ' - if fdict['acodec'] == 'none': - res += 'video only' - else: - res += '%-5s' % fdict['acodec'] - elif fdict.get('abr') is not None: - if res: - res += ', ' - res += 'audio' - if fdict.get('abr') is not None: - res += '@%3dk' % fdict['abr'] - if fdict.get('asr') is not None: - res += ' (%5dHz)' % fdict['asr'] + res.append('%sfps' % (fdict['fps'],)) + codec = ( + simplified_codec(fdict, 'acodec') if fdict.get('acodec') is not None + else 'audio' if fdict.get('abr') is not None else None) + if codec: + res.append(join_nonempty( + '%-4s' % (codec + (('@%3dk' % fdict['abr']) if fdict.get('abr') else ''),), + fdict.get('asr') and '(%5dHz)' % fdict['asr'], delim=' ')) if fdict.get('filesize') is not None: - if res: - res += ', ' - res += format_bytes(fdict['filesize']) + res.append(format_bytes(fdict['filesize'])) elif fdict.get('filesize_approx') is not None: - if res: - res += ', ' - res += '~' + format_bytes(fdict['filesize_approx']) - return res + res.append('~' + format_bytes(fdict['filesize_approx'])) + return ', '.join(res) def list_formats(self, info_dict): formats = info_dict.get('formats', [info_dict]) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index 7965fa08a..e5d218d17 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -1669,10 +1669,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '_rtmp': {'protocol': 'rtmp'}, # av01 video only formats sometimes served with "unknown" codecs - '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, - '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, - '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, - '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, + '394': {'acodec': 'none', 'vcodec': 'av01.0.00M.08'}, + '395': {'acodec': 'none', 'vcodec': 'av01.0.00M.08'}, + '396': {'acodec': 'none', 'vcodec': 'av01.0.01M.08'}, + '397': {'acodec': 'none', 'vcodec': 'av01.0.04M.08'}, + '398': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, + '399': {'acodec': 'none', 'vcodec': 'av01.0.08M.08'}, } _PLAYER_JS_VARIANT_MAP = ( From 956b8c585591b401a543e409accb163eeaaa1193 Mon Sep 17 00:00:00 2001 From: dirkf Date: Wed, 26 Nov 2025 01:29:22 +0000 Subject: [PATCH 43/43] [YouTube] Bug-fix for `c1f5c3274a` --- youtube_dl/extractor/youtube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index e5d218d17..81a019143 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3813,7 +3813,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): continuation = None for is_renderer in traverse_obj(slr_renderer, ( 'contents', Ellipsis, 'itemSectionRenderer', T(dict))): - for isr_content in traverse_obj(slr_renderer, ( + for isr_content in traverse_obj(is_renderer, ( 'contents', Ellipsis, T(dict))): renderer = isr_content.get('playlistVideoListRenderer') if renderer: